Commit 9600dd2c authored by Andy Buckley's avatar Andy Buckley
Browse files

Merge branch '21.6' of ssh://gitlab.cern.ch:7999/atlas/athena into 21.6

parents 399bfceb 6fe78584
......@@ -15,7 +15,6 @@ endif()
#set( PYTHIA8_ROOT
# /cvmfs/sft.cern.ch/lcg/releases/LCG_88/MCGenerators/pythia8/${PYTHIA8_VERSION}/${LCG_PLATFORM} )
# Find Pythia8:
find_package( Pythia8 )
......@@ -28,7 +27,7 @@ endif()
# Install the files from the xmldoc directory into the offline release.
set( xmldoc_dir "${PYTHIA8_LCGROOT}/share/Pythia8/xmldoc" )
if( NOT IS_DIRECTORY ${xmldoc_dir} )
message( WARNING "Can't access ${xmldoc_dir}" )
message( WARNING " Can't access ${xmldoc_dir}" )
return()
endif()
atlas_install_generic( ${xmldoc_dir}/*.xml ${xmldoc_dir}/*.dat
......@@ -36,3 +35,14 @@ atlas_install_generic( ${xmldoc_dir}/*.xml ${xmldoc_dir}/*.dat
${xmldoc_dir}/*.tbl ${xmldoc_dir}/*.eps
DESTINATION share/Pythia8/xmldoc
TYPENAME XmlDoc )
# Install the files from the pdfdata directory into the offline release. New in Pythia8.3
set( pdfdata_dir "${PYTHIA8_LCGROOT}/share/Pythia8/pdfdata" )
if( NOT IS_DIRECTORY ${pdfdata_dir} )
message( INFO " Can't access ${pdfdata_dir} (Needed only for Pythia8.3)" )
return()
endif()
atlas_install_generic( ${pdfdata_dir}/*.*
DESTINATION share/Pythia8/pdfdata
TYPENAME XmlDoc )
......@@ -15,6 +15,7 @@ atlas_depends_on_subdirs( PUBLIC
# Install files from the package:
atlas_install_python_modules( python/*.py )
atlas_install_joboptions( share/*.py )
atlas_install_joboptions( share/file/*.py )
atlas_install_joboptions( test/test_*.py )
atlas_install_runtime( scripts/*.py )
atlas_install_runtime( test/EvgenJobTransforms_TestConfiguration.xml )
......@@ -7,7 +7,7 @@ lhefGenerators = ["Lhef", # generic name: prefer to use the names below
"aMcAtNlo", "McAtNlo", "Powheg", "PowHel", "MadGraph", "CompHep", "CalcHep","Geneva",
"Whizard", "MCFM", "JHU", "MEtop", "Charybdis", "Charybdis2", "BCVEGPY", "Dire4Pythia8",
"BlackMax", "QBH", "gg2ww", "gg2zz", "gg2vv", "HvyN", "VBFNLO", "FPMC", "ProtosLHEF",
"BCVEGPY", "STRINGS"]
"BCVEGPY", "STRINGS", "Phantom"]
## A more general list of generators which provide partonic input, including non-LHEF ones
inputGenerators = lhefGenerators + ["Alpgen", "Protos"]
......@@ -31,7 +31,7 @@ mainGenerators += ["AMPT","Superchic","Starlight", "Hijing", "Hydjet", "Reldis",
## Misc generators
mainGenerators += ["AcerMC", "TopRex", "LPair"]
## Reading in fully-formed events
mainGenerators += ["HepMCAscii"]
mainGenerators += ["HepMCAscii", "ReadMcAscii"]
## Special QED and decay afterburners
afterburnerGenerators = ["Photos", "Photospp", "Tauola", "TauolaPP", "Tauolapp", "EvtGen", "ParticleDecayer"]
......@@ -123,6 +123,8 @@ class EvgenConfig(TransformConfig):
extraSaveItems = ListOfStrings("List of extra StreamEVGEN items to save in output file - note occurs AFTER doNotSaveItems are removed")
inputFilesPerJob = Integer("number of input files per job",0, AllowedExpression("value >= 0"))
nEventsPerJob = Integer("number of input events per job",0, AllowedExpression("value >= 0"))
obsolete = Boolean("Are JOs/common fragment obsolete", False)
PDGparams = Boolean("Do we use the standard PDG values for masses, widths etc. ", False)
def __init__(self, name="evgenConfig"):
TransformConfig.__init__(self, name)
......@@ -132,7 +134,7 @@ class EvgenConfig(TransformConfig):
self.maxeventsstrategy = "ABORT"
self.specialConfig = "NONE"
# for the sake of Generate_tf leave minevents for a while
self.minevents = 5000
self.minevents = 0
## Explicitly block MC11/12 settings of efficiency, input*base, or weighting attrs
def __setattr__(self, name, value):
......
......@@ -185,6 +185,9 @@ def getTransform():
if "--outputEVNTFile" in str(sys.argv[1:]):
exeSet.add(EvgenExecutor(name="generate", skeleton="EvgenJobTransforms/skel.GENtoEVGEN.py", inData=["inNULL"], outData=["EVNT", "EVNT_Pre", "TXT" ]))
msg.info("Output EVNT file")
elif "--outputYODAFile" in str(sys.argv[1:]):
exeSet.add(EvgenExecutor(name="generate", skeleton="EvgenJobTransforms/skel.GENtoEVGEN.py", inData=["inNULL"], outData=["outNULL", "TXT" ]))
msg.info("Output EVNT file")
elif "--outputTXTFile" in str(sys.argv[1:]):
exeSet.add(EvgenExecutor(name="generate", skeleton="EvgenJobTransforms/skel.GENtoTXT.py", inData=["inNULL"], outData=["TXT"]))
msg.info("Output TXT file")
......
#! /usr/bin/env python
"""
Run event simulation and produce an EVNT file.
"""
import os, sys, time, shutil
from PyJobTransforms.trfLogger import msg
from PyJobTransforms.transform import transform
from PyJobTransforms.trfExe import athenaExecutor, scriptExecutor
from PyJobTransforms.trfArgs import addAthenaArguments
from PyJobTransforms.trfDecorators import stdTrfExceptionHandler, sigUsrStackTrace
from EvgenJobTransforms.evgenTrfArgs import addStdEvgenArgs
import PyJobTransforms.trfArgClasses as trfArgClasses
## Prodsys1 hack...
# TODO: Remove!
ListOfDefaultPositionalKeys=['--AMIConfig', '--AMITag', '--argJSON', '--asetup', '--athena', '--athenaMPMergeTargetSize', '--athenaopts', '--attempt', '--checkEventCount', '--command', '--dumpJSON', '--dumpPickle', '--ecmEnergy', '--env', '--eventAcceptanceEfficiency', '--evgenJobOpts', '--execOnly', '--fileValidation', '--firstEvent', '--ignoreErrors', '--ignoreFiles', '--ignorePatterns', '--imf', '--inputEVNT_PreFile', '--inputFileValidation', '--inputGenConfFile', '--inputGeneratorFile', '--jobConfig', '--jobid', '--maxEvents', '--orphanKiller', '--outputEVNTFile', '--outputEVNT_PreFile', '--outputFileValidation', '--outputNTUP_TRUTHFile', '--outputTXTFile', '--parallelFileValidation', '--postExec', '--postInclude', '--preExec', '--preInclude', '--printEvts', '--randomSeed', '--reportName', '--reportType', '--rivetAnas', '--runNumber', '--showGraph', '--showPath', '--showSteps', '--skipEvents', '--skipFileValidation', '--skipInputFileValidation', '--skipOutputFileValidation', '--steering', '--taskid', '--tcmalloc', '--valgrind', '--valgrindbasicopts', '--valgrindextraopts', '--lheOnly', '--localPath', '--cleanOut', '--saveList']
class EvgenExecutor(athenaExecutor):
"Specialised trf executor class for matrix element generation jobs"
def __init__(self, name="generate", skeleton=None, substep=None, inData=set(), outData=set()):
athenaExecutor.__init__(self, name=name, skeletonFile=skeleton, substep=substep, tryDropAndReload=False, inData=inData, outData=outData)
# def __init__(self, name="generate", inData=["inNULL"], outData=["TXT"], exe='ls'):
# athenaExecutor.__init__(self, name=name, inData=inData, outData=outData, exe=exe)
print "jestem w init !!!"
def preExecute(self, input=set(), output=set()):
"Get input tarball, unpack and set up env if an evgenJobOpts arg was provided."
## First call the base class preExecute
super(EvgenExecutor, self).preExecute(input, output)
print "jestem w pre !!!"
def expand_if_archive(filename):
"Function to expand a file if it is a zip archive or tarball"
if ".tar" in filename:
import tarfile
tf = tarfile.open(filename)
tf.extractall()
tf.close()
elif filename.endswith(".zip"):
import zipfile
zf = zipfile.ZipFile(filename)
zf.extractall()
zf.close()
def get_immediate_subdirectories(a_dir):
return [name for name in os.listdir(a_dir)
if os.path.isdir(os.path.join(a_dir, name))]
# read the JO directory
os.environ['LOCAL_INSTALL_DIR'] = (os.environ['JOBOPTSEARCHPATH']).split(":")[0]
CommonCvmfsDir = '/cvmfs/atlas.cern.ch/repo/sw/Generators/MCJobOptions/common'
os.environ["JOBOPTSEARCHPATH"] = CommonCvmfsDir+":"+os.environ["JOBOPTSEARCHPATH"]
# msg.info("Using JOBOPTSEARCHPATH = '%s'" % os.environ["LOCAL_INSTALL_DIR"])
dsidparam = (self._trf.argdict["jobConfig"].value).values()[0][0]
# Adding cvmfs path to JOBOPTSEARCHPATH
BaseCvmfsPath = "/cvmfs/atlas.cern.ch/repo/sw/Generators/MCJobOptions/"
msg.info("!! Base cvmfs path = '%s'" % BaseCvmfsPath)
if len(dsidparam)==6 and dsidparam.isdigit(): #only dsid is provided, add cvmfs folder like 123xxx to JOBOPTSEARCHPATH
Jodir = dsidparam[:3]+'xxx'
cwdir = os.getcwd()
cwd_ful = os.path.join(cwdir, dsidparam)
msg.info("!! cwd full = '%s'" % cwd_ful)
if (os.path.isdir(cwd_ful)):
os.environ["JOBOPTSEARCHPATH"] = cwd_ful+":"+os.environ["JOBOPTSEARCHPATH"]
else:
cwd_Jodir = os.path.join(cwdir,Jodir)
cwd_Jodir_ful = os.path.join(cwd_Jodir,dsidparam)
if (os.path.isdir(cwd_Jodir_ful)):
os.environ["JOBOPTSEARCHPATH"] = cwd_Jodir_ful+":"+os.environ["JOBOPTSEARCHPATH"]
else:
JoCvmfsPath = os.path.join(BaseCvmfsPath, Jodir)
JoCvmfsPath_ful = os.path.join(JoCvmfsPath, dsidparam)
os.environ["JOBOPTSEARCHPATH"] = JoCvmfsPath_ful+":"+os.environ["JOBOPTSEARCHPATH"]
print '!! JoCvmfsPath_ful ',JoCvmfsPath_ful
msg.info("Using JOBOPTSEARCHPATH! = '%s'" % os.environ["JOBOPTSEARCHPATH"])
#os.environ["JOBOPTSEARCHPATH"] = os.environ['LOCAL_INSTALL_DIR']+":"+os.environ["JOBOPTSEARCHPATH"]
print "!! Jodir ",Jodir
else: #Suppose full path of dsid folder is provided(/afs/.../123xxx/123456), add cvmfs floder and local path(/afs/.../123xxx) to JOBOPTSEARCHPATH
# dsid_part=os.path.basename(dsidparam)
if (os.path.isdir(dsidparam)):
os.environ["JOBOPTSEARCHPATH"] = dsidparam+":"+os.environ["JOBOPTSEARCHPATH"]
else:
msg.error("JOs not found, please check = '%s'" % dsidparam)
# Jodir = dsidparam[:3]+'xxx'
# JoCvmfsPath = os.path.join(BaseCvmfsPath, Jodir)
# JoLocPath = os.path.dirname()
# print "!! JoLocPath ",JoLocPath
# os.environ["JOBOPTSEARCHPATH"] = JoLocPath+":"+os.environ["JOBOPTSEARCHPATH"]
# os.environ["JOBOPTSEARCHPATH"] = JoCvmfsPath+":"+os.environ["JOBOPTSEARCHPATH"]
# msg.info("Using JOBOPTSEARCHPATH!! = '%s'" % os.environ["JOBOPTSEARCHPATH"])
# os.environ["JOBOPTSEARCHPATH"] = os.environ['LOCAL_INSTALL_DIR']+":"+os.environ["JOBOPTSEARCHPATH"]
# msg.info("Using JOBOPTSEARCHPATH!! = '%s'" % os.environ["JOBOPTSEARCHPATH"])
# else: #wrong JOoption format
# msg.info("Incorrect JO, Please check")
msg.info("Using JOBOPTSEARCHPATH = '%s'" % os.environ["JOBOPTSEARCHPATH"])
if "evgenJobOpts" in self._trf.argdict: ## Use a specified JO tarball
tarball = self._trf.argdict["evgenJobOpts"].value
## Prepend the standard tarball URL prefix if the arg is not a full URL
# TODO: also allow a copy from a local filesystem location if it starts with a / (or can be found with os.path)?
if tarball.startswith("http"):
url = tarball
tarball = os.basename(tarball)
else:
url = "http://cern.ch/atlas-computing/links/kitsDirectory/EvgenJobOpts/" + tarball
## Download the tarball in a Grid-safe way (if it doesn't already exist)
if not os.path.exists(tarball):
from EvgenJobTransforms.download import downloadUsingProxy
status, output = downloadUsingProxy(url)
if status != 0:
raise EnvironmentError('Error downloading tarball %s. Downloader reports: %s' % (tarball, output))
msg.info('Evgen tarball download success: %s' % output)
## Expand tarball
expand_if_archive(tarball)
## Expand tarball input event and generator conf files, if provided
if "inputGeneratorFile" in self._trf.argdict:
# expand_if_archive(self._trf.argdict["inputGeneratorFile"].value)
myinputfiles = self._trf.argdict["inputGeneratorFile"].value
genInputFiles = myinputfiles.split(',')
for file in genInputFiles:
expand_if_archive(file)
if "inputGenConfFile" in self._trf.argdict:
expand_if_archive(self._trf.argdict["inputGenConfFile"].value)
def move_files(main_dir,tmp_dir,whitelist):
files = os.listdir(tmp_dir)
print("list of files ",files)
print("white list ",whitelist)
files.sort()
for f in files:
for i in whitelist:
if i in f:
src = tmp_dir+"/"+f
dest = main_dir+"/"+f
# print("src ",src)
# print("dest ",dest)
os.rename(src,dest)
break
def getTransform():
exeSet = set()
#exeSet.add(EvgenScriptExecutor())
exeSet.add(EvgenExecutor(name="generate", skeleton="EvgenJobTransforms/skel.GENtoTXT.py", inData=["inNULL"], outData=["TXT"]))
# exeSet.add(EvgenExecutor(name="afterburn", skeleton="EvgenJobTransforms/skeleton.ABtoEVGEN.py", inData=["EVNT_Pre"], outData=["EVNT"]))
exeSet.add(athenaExecutor(name = "AODtoDPD", skeletonFile = "PATJobTransforms/skeleton.AODtoDPD_tf.py",
substep = "a2d", inData = ["EVNT"], outData = ["NTUP_TRUTH"], perfMonFile = "ntuple_AODtoDPD.pmon.gz"))
trf = transform(executor=exeSet)
addAthenaArguments(trf.parser, maxEventsDefaultSubstep='all')
addStdEvgenArgs(trf.parser)
return trf
@stdTrfExceptionHandler
@sigUsrStackTrace
def main():
msg.info("This is %s" % sys.argv[0])
main_dir = os.getcwd()
trf = getTransform()
trf.parseCmdLineArgs(sys.argv[1:])
print("main ",main_dir)
if (("cleanOut" in trf.argdict) and (trf.argdict["cleanOut"].value != 0)):
name_tmpdir = "tmprun"
tmp_dir = os.path.join(main_dir, name_tmpdir)
# print("tmpdir ",tmp_dir)
if os.path.isdir(tmp_dir):
shutil.rmtree(tmp_dir, ignore_errors=True)
os.mkdir("tmprun")
os.chdir("tmprun")
tmp_dir = os.getcwd()
whitelist_in = ['MC','group','TXT']
move_files(tmp_dir,main_dir,whitelist_in)
# trf = getTransform()
# trf.parseCmdLineArgs(sys.argv[1:])
trf.execute()
trf.generateReport()
msg.info("%s stopped at %s, trf exit code %d" % (sys.argv[0], time.asctime(), trf.exitCode))
# read files/dirs that should be saved and if present in cwd - remove
if (("cleanOut" in trf.argdict) and (trf.argdict["cleanOut"].value!=0)):
# print("mamy cleanOut !!!",trf.argdict["cleanOut"].value)
whitelist_out = ['log.generate','.root']
if "outputTXTFile" in trf.argdict:
whitelist_out.append('TXT')
if "saveList" in trf.argdict:
saveList_dic= trf.argdict["saveList"].value
saveList_str= str(saveList_dic)
saveList_str=saveList_str[10:-3]
saveList= saveList_str.split(",")
for item in saveList:
test_ex = os.path.join(main_dir,str(item))
if os.path.isdir(test_ex):
shutil.rmtree(test_ex, ignore_errors=True)
# print("directory removed ", test_ex)
elif os.path.isfile(test_ex):
os.remove(test_ex)
# print("file removed ", test_ex)
# new_list = list(list(int(a) for a in b) for b in T1 if a.isdigit())
if not saveList[0].isdigit():
whitelist_out=whitelist_out+saveList
move_files(main_dir,tmp_dir,whitelist_out)
os.chdir(main_dir)
if "saveList" not in trf.argdict:
shutil.rmtree(tmp_dir, ignore_errors=True)
# elif firstSaveList!=1:
elif not saveList[0].isdigit():
shutil.rmtree(tmp_dir, ignore_errors=True)
# if cleanOut is not defined and multipleinput preset, remove the merged file
elif ("inputGeneratorFile" in trf.argdict):
myinputfiles = trf.argdict["inputGeneratorFile"].value
genInputFiles = myinputfiles.split(',')
numberOfFiles = len(genInputFiles)
merge_file = 'merged_lhef._0.events'
if((numberOfFiles>1) and (os.path.exists(merge_file))):
os.remove(merge_file)
#
sys.exit(trf.exitCode)
# TODO: Open resulting EVNT file to extract cross-section, generator names+versions, etc. from the HepMC::GenRun or whatever... in an executor postExecute?
if __name__ == "__main__":
main()
#! /usr/bin/env python
# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
#
"""
# Run event simulation and produce an EVNT file.
"""
import os, sys, time, shutil
from PyJobTransforms.trfLogger import msg
from PyJobTransforms.transform import transform
from PyJobTransforms.trfExe import athenaExecutor
from PyJobTransforms.trfArgs import addAthenaArguments
from PyJobTransforms.trfDecorators import stdTrfExceptionHandler, sigUsrStackTrace
from EvgenJobTransforms.evgenTrfArgs import addStdEvgenArgs
import PyJobTransforms.trfArgClasses as trfArgClasses
## Prodsys1 hack...
# TODO: Remove!
ListOfDefaultPositionalKeys=['--AMIConfig', '--AMITag', '--argJSON', '--asetup', '--athena', '--athenaMPMergeTargetSize', '--athenaopts', '--attempt', '--checkEventCount', '--command', '--dumpJSON', '--dumpPickle', '--ecmEnergy', '--env', '--eventAcceptanceEfficiency', '--evgenJobOpts', '--execOnly', '--fileValidation', '--firstEvent', '--ignoreErrors', '--ignoreFiles', '--ignorePatterns', '--imf', '--inputEVNT_PreFile', '--inputFileValidation', '--inputGenConfFile', '--inputGeneratorFile', '--jobConfig', '--jobid', '--maxEvents', '--orphanKiller', '--outputEVNTFile', '--outputEVNT_PreFile', '--outputFileValidation', '--outputNTUP_TRUTHFile', '--outputTXTFile', '--parallelFileValidation', '--postExec', '--postInclude', '--preExec', '--preInclude', '--printEvts', '--randomSeed', '--reportName', '--reportType', '--rivetAnas', '--runNumber', '--showGraph', '--showPath', '--showSteps', '--skipEvents', '--skipFileValidation', '--skipInputFileValidation', '--skipOutputFileValidation', '--steering', '--taskid', '--tcmalloc', '--valgrind', '--valgrindbasicopts', '--valgrindextraopts', '--lheOnly', '--localPath', '--cleanOut', '--saveList']
class EvgenExecutor(athenaExecutor):
"Specialised trf executor class for event generation jobs"
def __init__(self, name="generate", skeleton="EvgenJobTransforms/skeleton.GENtoEVGEN.py", substep=None, inData=["inNULL"], outData=["EVNT", "EVNT_Pre", "TXT"]):
athenaExecutor.__init__(self, name=name, skeletonFile=skeleton, substep=substep, tryDropAndReload=False, inData=inData, outData=outData)
def preExecute(self, input=set(), output=set()):
"Get input tarball, unpack and set up env if an evgenJobOpts arg was provided."
## First call the base class preExecute
super(EvgenExecutor, self).preExecute(input, output)
def expand_if_archive(filename):
"Function to expand a file if it is a zip archive or tarball"
if ".tar" in filename:
import tarfile
tf = tarfile.open(filename)
tf.extractall()
tf.close()
elif filename.endswith(".zip"):
import zipfile
zf = zipfile.ZipFile(filename)
zf.extractall()
zf.close()
def get_immediate_subdirectories(a_dir):
return [name for name in os.listdir(a_dir)
if os.path.isdir(os.path.join(a_dir, name))]
def mk_jo_proxy(targetbasepath, pkgname, proxypath, addtosearch=True):
"Make a JO proxy dir such that the MCxxJobOptions/dddd dirs contents are found via include(MCxxJobOptions/yyyy)"
if proxypath:
if os.path.exists(proxypath):
shutil.rmtree(proxypath)
os.mkdir(proxypath)
os.environ['LOCAL_INSTALL_DIR'] = (os.environ['JOBOPTSEARCHPATH']).split(":")[0]
os.environ['LOCAL_DATA_DIR'] = (os.environ['DATAPATH']).split(":")[0]
dirlist = get_immediate_subdirectories(targetbasepath)
subdirlist=dirlist;
for dd in (dirlist):
if (('.svn' not in dd) and ('cmt' not in dd) and ('_joproxy' not in dd)):
deepdir = os.path.join(targetbasepath, dd)
subdirlist1 = get_immediate_subdirectories(deepdir)
subdirlist = subdirlist+["%s" % dd+"/%s" % item for item in subdirlist1]
for d in (subdirlist):
# TODO: we could _maybe_ add the appropriate share/DSIDxxxx/ dir to the path based on the jobConfig arg... too much magic?
if (('.svn' not in d) and ('cmt' not in d) and ('_joproxy' not in d) and ('share/' not in d)):
dpath = os.path.join(proxypath, d)
if proxypath:
os.mkdir(dpath)
if ('nonStandard' in dpath):
dpath_ex = os.path.join(dpath,pkgname)
if proxypath:
os.mkdir(dpath_ex)
os.symlink(os.path.join(targetbasepath, d), os.path.join(dpath_ex, "nonStandard"))
else :
os.symlink(os.path.join(targetbasepath, d), os.path.join(dpath, pkgname))
if addtosearch:
os.environ["JOBOPTSEARCHPATH"] = dpath+":"+os.environ["JOBOPTSEARCHPATH"]
os.environ["DATAPATH"] =os.path.join(targetbasepath, d)+":"+os.environ["DATAPATH"]
os.environ["JOBOPTSEARCHPATH"] = os.environ['LOCAL_INSTALL_DIR']+":"+os.environ["JOBOPTSEARCHPATH"]
os.environ["DATAPATH"] = os.environ['LOCAL_DATA_DIR']+":"+os.environ["DATAPATH"]
## Handle locating of evgen job options / fragments, either from a tarball or CVMFS
# read the JO used
joparam = self._trf.argdict["jobConfig"].value
if "evgenJobOpts" in self._trf.argdict: ## Use a specified JO tarball
tarball = self._trf.argdict["evgenJobOpts"].value
## Prepend the standard tarball URL prefix if the arg is not a full URL
# TODO: also allow a copy from a local filesystem location if it starts with a / (or can be found with os.path)?
if tarball.startswith("http"):
url = tarball
tarball = os.basename(tarball)
else:
url = "http://cern.ch/atlas-computing/links/kitsDirectory/EvgenJobOpts/" + tarball
## Download the tarball in a Grid-safe way (if it doesn't already exist)
if not os.path.exists(tarball):
from EvgenJobTransforms.download import downloadUsingProxy
status, output = downloadUsingProxy(url)
if status != 0:
raise EnvironmentError('Error downloading tarball %s. Downloader reports: %s' % (tarball, output))
msg.info('Evgen tarball download success: %s' % output)
## Expand tarball
expand_if_archive(tarball)
if "MC14" in str(joparam):
mk_jo_proxy(os.getcwd(), "MC14JobOptions", "_joproxy14")
else :
mk_jo_proxy(os.getcwd(), "MC15JobOptions", "_joproxy15")
## Source setup script (requires some shenanigans to update the Python env robustly)
# TODO: trf framework now bans use of exec()...
#import subprocess
#pipe = subprocess.Popen(". ./setupevprod.sh; python -c 'import os; print \"newenv = %r\" % os.environ'", stdout=subprocess.PIPE, shell=True)
#exec(pipe.communicate()[0])
#os.environ.update(newenv)
else: ## Use the CVMFS copy of the latest MC14 JOs tag
sw_base = os.popen("echo $ATLAS_SW_BASE").read()
# print subprocess.Popen("echo Hellllo World", shell=True, stdout=subprocess.PIPE).stdout.read()
# sw_base = print subprocess.Popen("echo $ATLAS_LOCAL_ROOT_BASE", shell=True, stdout=subprocess.PIPE)
sw_base = sw_base.strip()
if (sw_base == ""):
msg.info('$ATLAS_SW_BASE not defined, trying explicite /cvmfs path')
sw_base = '/cvmfs'
# msg.info('sw_base path %s ' %sw_base)
else:
msg.info('ATLAS_SW_BASE path: %s' %sw_base)
local_path = None
if ("localPath" in self._trf.argdict ):
local_path = self._trf.argdict["localPath"].value
print("local path",local_path)
cvmfs_path = os.path.join(sw_base, "atlas.cern.ch")
if ((local_path is not None) and (os.path.exists(local_path))) :
mk_jo_proxy(local_path, "MC15JobOptions","_joproxy15")
print("JO fragments taken from local path i.e. ",local_path)
elif os.path.exists(cvmfs_path):
# TODO: Make the package name configurable
if "MC14" in str(joparam):
cvmfs_mc14 = os.path.join(cvmfs_path, "repo/sw/Generators/MC14JobOptions/latest/")
mk_jo_proxy(cvmfs_mc14, "MC14JobOptions","_joproxy14")
# if "MC15" in str(joparam):
else :
cvmfs_mc15 = os.path.join(cvmfs_path, "repo/sw/Generators/MC15JobOptions/latest/")
# msg.info('cvmfs_mc15 path: %s ' %cvmfs_mc15)
mk_jo_proxy(cvmfs_mc15, "MC15JobOptions","_joproxy15")
msg.info("No evgenJobOpts tarball specified, using JOBOPTSEARCHPATH = '%s'" % os.environ["JOBOPTSEARCHPATH"])
elif os.path.exists("/afs/cern.ch/atlas/groups/Generators"):
if "MC14" in str(joparam):
mk_jo_proxy("/afs/cern.ch/atlas/groups/Generators/MC14JobOptions/latest/", "MC14JobOptions","_joproxy14")
else :
mk_jo_proxy("/afs/cern.ch/atlas/groups/Generators/MC15JobOptions/latest/", "MC15JobOptions","_joproxy15")
msg.info("No evgenJobOpts tarball specified, no cvmfs, using JOBOPTSEARCHPATH = '%s'" % os.environ["JOBOPTSEARCHPATH"])
## Expand tarball input event and generator conf files, if provided
if "inputGeneratorFile" in self._trf.argdict:
# expand_if_archive(self._trf.argdict["inputGeneratorFile"].value)
myinputfiles = self._trf.argdict["inputGeneratorFile"].value
genInputFiles = myinputfiles.split(',')
for file in genInputFiles:
expand_if_archive(file)
if "inputGenConfFile" in self._trf.argdict:
expand_if_archive(self._trf.argdict["inputGenConfFile"].value)
def move_files(main_dir,tmp_dir,whitelist):
files = os.listdir(tmp_dir)
print("list of files ",files)
print("white list ",whitelist)
files.sort()
for f in files:
for i in whitelist:
if i in f:
src = tmp_dir+"/"+f
dest = main_dir+"/"+f
# print("src ",src)
# print("dest ",dest)
os.rename(src,dest)
break
def getTransform():
exeSet = set()
exeSet.add(EvgenExecutor(name="generate"))
exeSet.add(EvgenExecutor(name="afterburn", skeleton="EvgenJobTransforms/skeleton.ABtoEVGEN.py", inData=["EVNT_Pre"], outData=["EVNT"]))
exeSet.add(athenaExecutor(name = "AODtoDPD", skeletonFile = "PATJobTransforms/skeleton.AODtoDPD_tf.py",
substep = "a2d", inData = ["EVNT"], outData = ["NTUP_TRUTH"], perfMonFile = "ntuple_AODtoDPD.pmon.gz"))
trf = transform(executor=exeSet)
addAthenaArguments(trf.parser, maxEventsDefaultSubstep='all')
addStdEvgenArgs(trf.parser)
return trf
@stdTrfExceptionHandler
@sigUsrStackTrace
def main():
msg.info("This is %s" % sys.argv[0])
main_dir = os.getcwd()
trf = getTransform()
trf.parseCmdLineArgs(sys.argv[1:])
msg.info(' ######################################################################################################## ')
msg.info('THIS TRANSFORM SHOULD NOT BE USED IN PRODUCTION')
msg.info('PLEASE USE GEN_TF !!! ')
msg.info(' ######################################################################################################## ')
# print("main ",main_dir)
if (("cleanOut" in trf.argdict) and (trf.argdict["cleanOut"].value != 0)):
name_tmpdir = "tmprun"
tmp_dir = os.path.join(main_dir, name_tmpdir)
# print("tmpdir ",tmp_dir)
if os.path.isdir(tmp_dir):
shutil.rmtree(tmp_dir, ignore_errors=True)
os.mkdir("tmprun")
os.chdir("tmprun")
tmp_dir = os.getcwd()
whitelist_in = ['MC','group','TXT']
move_files(tmp_dir,main_dir,whitelist_in)
# trf = getTransform()
# trf.parseCmdLineArgs(sys.argv[1:])
trf.execute()
trf.generateReport()
msg.info("%s stopped at %s, trf exit code %d" % (sys.argv[0], time.asctime(), trf.exitCode))
# read files/dirs that should be saved and if present in cwd - remove
if (("cleanOut" in trf.argdict) and (trf.argdict["cleanOut"].value!=0)):
# print("mamy cleanOut !!!",trf.argdict["cleanOut"].value)
whitelist_out = ['log.generate','.root']
if "outputTXTFile" in trf.argdict:
whitelist_out.append('TXT')
if "saveList" in trf.argdict: