From 4b6d2fba3f623007648c184fcf75d71e4eb75b7c Mon Sep 17 00:00:00 2001
From: Ewelina Maria Lobodzinska <ewelina.maria.lobodzinska@cern.ch>
Date: Wed, 23 Dec 2020 15:50:32 +0000
Subject: [PATCH] update gen_tf.py and skel.GENtoEVGEN.py to include the most
recent modifications introduced for rel. 21.6
---
Generators/EvgenJobTransforms/CMakeLists.txt | 2 +-
.../EvgenJobTransforms/scripts/Gen_tf.py | 34 +++++-
.../share/skel.GENtoEVGEN.py | 104 +++++++++++++++---
.../EvgenJobTransforms/share/skel.GENtoTXT.py | 69 ++++++++----
4 files changed, 162 insertions(+), 47 deletions(-)
diff --git a/Generators/EvgenJobTransforms/CMakeLists.txt b/Generators/EvgenJobTransforms/CMakeLists.txt
index 8736b66f00e..4b9ece44306 100644
--- a/Generators/EvgenJobTransforms/CMakeLists.txt
+++ b/Generators/EvgenJobTransforms/CMakeLists.txt
@@ -6,4 +6,4 @@ atlas_subdir( EvgenJobTransforms )
# Install files from the package:
atlas_install_python_modules( python/*.py POST_BUILD_CMD ${ATLAS_FLAKE8} )
atlas_install_joboptions( share/*.py )
-atlas_install_runtime( scripts/*.py )
+atlas_install_runtime( scripts/*.py share/file/*.txt )
diff --git a/Generators/EvgenJobTransforms/scripts/Gen_tf.py b/Generators/EvgenJobTransforms/scripts/Gen_tf.py
index ee7ba47953f..91bd3d84c4f 100755
--- a/Generators/EvgenJobTransforms/scripts/Gen_tf.py
+++ b/Generators/EvgenJobTransforms/scripts/Gen_tf.py
@@ -118,19 +118,39 @@ class EvgenExecutor(athenaExecutor):
# copy config files to cwd
FIRST_DIR = (os.environ['JOBOPTSEARCHPATH']).split(":")[0]
configFiles = [f for f in os.listdir(FIRST_DIR) if ( "GRID" in f)]
+ confFile=None
if len(configFiles) == 1:
confFile = os.path.join(FIRST_DIR, configFiles[0])
expand_if_archive(confFile)
msg.info('Configuration input found: %s' % confFile)
elif len(configFiles) >1:
- msg.error("Too many *GRID* config files, please check = '%s'" % dsidparam)
+ msg.info("more then one gridpack ! ")
+ if "--ecmEnergy" in str(sys.argv[1:]):
+ ener=str(sys.argv[1:]).split("ecmEnergy",1)[1]
+ energy=str(ener)[:4].strip(" =0\']")
+ printfunc("Should be used gridpack for energy "+energy)
+ else:
+ energy="13"
+ for x in configFiles:
+ gridS="mc_"+energy+"TeV"
+ printfunc("Gridpack should start from "+gridS)
+ if x.startswith(gridS):
+ confFile = os.path.join(FIRST_DIR, x)
+ msg.info("using gridpack = "+confFile)
+ if confFile is None:
+ msg.error("No *GRID* config files, for requested energy = '%s' please check = '%s'" %(energy,dsidparam))
+
+ if confFile is not None:
+ expand_if_archive(confFile)
+# os.system("cp %s ." % confFile)
+ printfunc("Configuration input gridpack found " + confFile)
#Expand if a tarball is found in local directory
loc_files = os.listdir(os.getcwd())
- for loc_file in loc_files:
- expand_if_archive(loc_file)
-
-
+ for loc_file in loc_files:
+ if "GRID" not in loc_file:
+ expand_if_archive(loc_file)
+
## Expand tarball input event and generator conf files, if provided
if "inputGeneratorFile" in self._trf.argdict:
# expand_if_archive(self._trf.argdict["inputGeneratorFile"].value)
@@ -159,6 +179,10 @@ def getTransform():
if "--outputEVNTFile" in str(sys.argv[1:]):
exeSet.add(EvgenExecutor(name="generate", skeleton="EvgenJobTransforms/skel.GENtoEVGEN.py", inData=["inNULL"], outData=["EVNT", "EVNT_Pre", "TXT" ]))
msg.info("Output EVNT file")
+ elif "--outputYODAFile" in str(sys.argv[1:]):
+ exeSet.add(EvgenExecutor(name="generate", skeleton="EvgenJobTransforms/skel.GENtoEVGEN.py", inData=["inNULL"], outData=["outNULL", "TXT"
+]))
+ msg.info("Output EVNT file")
elif "--outputTXTFile" in str(sys.argv[1:]):
exeSet.add(EvgenExecutor(name="generate", skeleton="EvgenJobTransforms/skel.GENtoTXT.py", inData=["inNULL"], outData=["TXT"]))
msg.info("Output TXT file")
diff --git a/Generators/EvgenJobTransforms/share/skel.GENtoEVGEN.py b/Generators/EvgenJobTransforms/share/skel.GENtoEVGEN.py
index 6b5c098e200..beae3cff244 100644
--- a/Generators/EvgenJobTransforms/share/skel.GENtoEVGEN.py
+++ b/Generators/EvgenJobTransforms/share/skel.GENtoEVGEN.py
@@ -79,11 +79,21 @@ if hasattr(runArgs, "runNumber"):
if hasattr(runArgs, "inputGenConfFile"):
raise RuntimeError("inputGenConfFile is invalid !! Gridpacks and config. files/links to be put into DSID directory ")
+
+if hasattr(runArgs, "inputGeneratorFile"):
+ evgenLog.info("inputGeneratorFile used " + runArgs.inputGeneratorFile)
+
+if hasattr(runArgs, "outputYODAFile"):
+ evgenLog.info("outputYODAFile specified " + runArgs.outputYODAFile)
## Ensure that an output name has been given
# TODO: Allow generation without writing an output file (if outputEVNTFile is None)?
if not hasattr(runArgs, "outputEVNTFile") and not hasattr(runArgs, "outputEVNT_PreFile"):
- raise RuntimeError("No output evgen EVNT or EVNT_Pre file provided.")
+ if hasattr(runArgs, "outputYODAFile"):
+ evgenLog.info("No outputEVNTFile specified but outputYODAFile is used")
+ evgenLog.info("Will run GENtoEVGEN without saving the output EVNT file, asuming a valid outputYODAFile will be produced")
+ else:
+ raise RuntimeError("No output evgen EVNT or EVNT_Pre file provided.")
## Ensure that mandatory args have been supplied (complain before processing the includes)
if not hasattr(runArgs, "ecmEnergy"):
@@ -166,6 +176,8 @@ if hasattr(runArgs, "rivetAnas"):
anaSeq += Rivet_i()
anaSeq.Rivet_i.Analyses = runArgs.rivetAnas
anaSeq.Rivet_i.DoRootHistos = True
+ if hasattr(runArgs, "outputYODAFile"):
+ anaSeq.Rivet_i.HistoFile = runArgs.outputYODAFile
##==============================================================
## Pre- and main config parsing
@@ -249,6 +261,9 @@ include(jofile)
## Announce start of JO checking
evgenLog.debug("****************** CHECKING EVGEN CONFIGURATION *****************")
+if hasattr(runArgs,'inputGeneratorFile') and int(evgenConfig.inputFilesPerJob) == 0 :
+ evgenConfig.inputFilesPerJob = 1
+
## Print out options
for opt in str(evgenConfig).split(os.linesep):
evgenLog.info(opt)
@@ -311,15 +326,29 @@ rounding = 0
if hasattr(runArgs,'inputGeneratorFile') and ',' in runArgs.inputGeneratorFile: multiInput = runArgs.inputGeneratorFile.count(',')+1
else:
multiInput = 0
-
-if evgenConfig.nEventsPerJob < 1:
- raise RunTimeError("evgenConfig.nEventsPerJob must be at least 1")
+
+# check if default nEventsPerJob used
+if not evgenConfig.nEventsPerJob:
+ evgenLog.info('#############################################################')
+ evgenLog.info(' !!!! no nEventsPerJob set !!! The default 10000 used. !!! ')
+ evgenLog.info('#############################################################')
+else:
+ evgenLog.info(' nEventsPerJob set to ' + str(evgenConfig.nEventsPerJob) )
+
+if evgenConfig.minevents > 0 :
+ raise RuntimeError("evgenConfig.minevents is obsolete and should be removed from the JOs")
+elif evgenConfig.nEventsPerJob > 100000:
+ raise RuntimeError("evgenConfig.nEventsPerJob can be max. 100000")
else:
allowed_nEventsPerJob_lt1000 = [1, 2, 5, 10, 20, 25, 50, 100, 200, 500, 1000]
msg = "evgenConfig.nEventsPerJob = %d: " % evgenConfig.nEventsPerJob
- if evgenConfig.nEventsPerJob >= 1000 and evgenConfig.nEventsPerJob % 1000 != 0 and 10000 % evgenConfig.nEventsPerJob != 0 :
- msg += "nEventsPerJob in range >= 1K must be a multiple of 1K and a divisor of 10K"
+ if evgenConfig.nEventsPerJob >= 1000 and evgenConfig.nEventsPerJob <=10000 and (evgenConfig.nEventsPerJob % 1000 != 0 or 10000 %
+evgenConfig.nEventsPerJob != 0) :
+ msg += "nEventsPerJob in range [1K, 10K] must be a multiple of 1K and a divisor of 10K"
+ raise RuntimeError(msg)
+ elif evgenConfig.nEventsPerJob > 10000 and evgenConfig.nEventsPerJob % 10000 != 0:
+ msg += "nEventsPerJob >10K must be a multiple of 10K"
raise RuntimeError(msg)
elif evgenConfig.nEventsPerJob < 1000 and evgenConfig.nEventsPerJob not in allowed_nEventsPerJob_lt1000:
msg += "nEventsPerJob in range <= 1000 must be one of %s" % allowed_nEventsPerJob_lt1000
@@ -331,9 +360,9 @@ else:
if evgenConfig.keywords:
## Get the allowed keywords file from the JO package if possibe
# TODO: Make the package name configurable
- kwfile = "EvgenJobTransforms/evgenkeywords.txt"
+ kwfile = "evgenkeywords.txt"
kwpath = None
- for p in os.environ["JOBOPTSEARCHPATH"].split(":"):
+ for p in os.environ["DATAPATH"].split(":"):
kwpath = os.path.join(p, kwfile)
if os.path.exists(kwpath):
break
@@ -356,15 +385,15 @@ if evgenConfig.keywords:
if officialJO:
sys.exit(1)
else:
- evgenLog.warning("Could not find evgenkeywords.txt file %s in $JOBOPTSEARCHPATH" % kwfile)
+ evgenLog.warning("Could not find evgenkeywords.txt file %s in DATAPATH" % kwfile )
## Check that the L1 and L2 keywords pairs are in the list of allowed words pairs (and exit if processing an official JO)
if evgenConfig.categories:
## Get the allowed categories file from the JO package if possibe
# TODO: Make the package name configurable
- lkwfile = "EvgenJobTransforms/CategoryList.txt"
+ lkwfile = "CategoryList.txt"
lkwpath = None
- for p in os.environ["JOBOPTSEARCHPATH"].split(":"):
+ for p in os.environ["DATAPATH"].split(":"):
lkwpath = os.path.join(p, lkwfile)
if os.path.exists(lkwpath):
break
@@ -397,7 +426,7 @@ if evgenConfig.categories:
if officialJO:
sys.exit(1)
else:
- evgenLog.warning("Could not find CategoryList.txt file %s in $JOBOPTSEARCHPATH" % lkwfile)
+ evgenLog.warning("Could not find CategoryList.txt file %s in $DATAPATH" % lkwfile)
## Configure POOL streaming to the output EVNT format file
from AthenaPoolCnvSvc.WriteAthenaPool import AthenaPoolOutputStream
@@ -465,6 +494,38 @@ if hasattr(testSeq, "TestHepMC") and not gens_testhepmc(evgenConfig.generators):
evgenLog.info("Removing TestHepMC sanity checker")
del testSeq.TestHepMC
+##=============================================================
+## Check release number
+##=============================================================
+# Function to check blacklist (from Spyros'es logParser.py)
+def checkBlackList(relFlavour,cache,generatorName) :
+ isError = None
+ with open('/cvmfs/atlas.cern.ch/repo/sw/Generators/MC16JobOptions/common/BlackList_caches.txt') as bfile:
+ for line in bfile.readlines():
+ if not line.strip():
+ continue
+ # Blacklisted release flavours
+ badRelFlav=line.split(',')[0].strip()
+ # Blacklisted caches
+ badCache=line.split(',')[1].strip()
+ # Blacklisted generators
+ badGens=line.split(',')[2].strip()
+
+ used_gens = ','.join(generatorName)
+ #Match Generator and release type e.g. AtlasProduction, MCProd
+ if relFlavour==badRelFlav and cache==badCache and re.search(badGens,used_gens) is not None:
+ if badGens=="": badGens="all generators"
+ isError=relFlavour+","+cache+" is blacklisted for " + badGens
+ return isError
+ return isError
+## Announce start of JO checkingrelease number checking
+evgenLog.debug("****************** CHECKING RELEASE IS NOT BLACKLISTED *****************")
+rel = os.popen("echo $AtlasVersion").read()
+rel = rel.strip()
+errorBL = checkBlackList("AthGeneration",rel,gennames)
+if (errorBL):
+ raise RuntimeError("This run is blacklisted for this generator, please use a different one !! "+ errorBL)
+# evgenLog.warning("This run is blacklisted for this generator, please use a different one !! "+ errorBL )
##==============================================================
## Handling of a post-include/exec args at the end of standard configuration
@@ -659,12 +720,6 @@ else:
raise RuntimeError("evgenConfig.inputfilecheck specified in %s, but generators %s do not require an input file" %
(runArgs.jobConfig, str(gennames)))
-## Check conf files, as above but for a different command line arg, and with omission allowed
-if hasattr(runArgs, "inputGenConfFile") and runArgs.inputGenConfFile != "NONE":
- if evgenConfig.inputconfcheck and not re.search(evgenConfig.inputconfcheck, runArgs.inputGenConfFile):
- raise RuntimeError("inputGenConfFile=%s is incompatible with inputconfcheck (%s) in %s" %
- (runArgs.inputGenConfFile, evgenConfig.inputconfcheck, runArgs.jobConfig))
-
## Do the aux-file copying
if evgenConfig.auxfiles:
from PyJobTransformsCore.trfutil import get_files
@@ -683,6 +738,19 @@ def _checkattr(attr, required=False):
return False
return True
+if hasattr(runArgs, "outputTXTFile"):
+ # counting the number of events in LHE output
+ with open(eventsFile) as f:
+ contents = f.read()
+ count_ev = contents.count("<event>")
+ printfunc("MetaData: %s = %s" % ("Number of produced LHE events ", count_ev))
+elif hasattr(runArgs, "inputGeneratorFile"):
+ # counting the number of events in LHE output
+ with open(eventsFile) as f:
+ contents = f.read()
+ count_ev = contents.count("<event>")
+ printfunc("MetaData: %s = %s" % ("Number of input LHE events ", count_ev))
+
if _checkattr("description", required=True):
msg = evgenConfig.description
if _checkattr("notes"):
diff --git a/Generators/EvgenJobTransforms/share/skel.GENtoTXT.py b/Generators/EvgenJobTransforms/share/skel.GENtoTXT.py
index db20bad83f6..f213345143a 100644
--- a/Generators/EvgenJobTransforms/share/skel.GENtoTXT.py
+++ b/Generators/EvgenJobTransforms/share/skel.GENtoTXT.py
@@ -33,7 +33,7 @@ testSeq = acam.athMasterSeq.EvgenTestSeq
## NOTE: LogicalExpressionFilter is an algorithm, not a sequence
from EvgenProdTools.LogicalExpressionFilter import LogicalExpressionFilter
acam.athMasterSeq += LogicalExpressionFilter("EvgenFilterSeq")
-filtSeq = acam.athMsterSeq.EvgenFilterSeq
+filtSeq = acam.athMasterSeq.EvgenFilterSeq
topSeq = acas.AlgSequence()
anaSeq = topSeq
topSeq += acas.AlgSequence("EvgenPostSeq")
@@ -61,7 +61,7 @@ jobproperties.AthenaCommonFlags.AllowIgnoreConfigError = False
## Set up a standard logger
from AthenaCommon.Logging import logging
-evgenLog = logging.getLogger('Gen_tf_txt')
+evgenLog = logging.getLogger('Gen_tf')
##==============================================================
## Run arg handling
@@ -155,7 +155,7 @@ def OutputTXTFile():
## Only permit one jobConfig argument for evgen: does more than one _ever_ make sense?
if len(runArgs.jobConfig) != 1:
- evgenLog.info("runArgs.jobConfig %s ", % runArgs.jobConfig)
+ evgenLog.info("runArgs.jobConfig " + runArgs.jobConfig)
evgenLog.error("You must supply one and only one jobConfig file argument. It has to start from mc. and end with .py")
sys.exit(1)
@@ -169,7 +169,7 @@ evgenLog.info("dsid " + dsid)
jofiles = [f for f in os.listdir(FIRST_DIR) if (f.startswith('mc') and f.endswith('.py'))]
## Only permit one JO file in each dsid folder
if len(jofiles) !=1:
- evgenLog.info("runArgs.jobConfig wrong %s ", % runArgs.jobConfig)
+ evgenLog.info("runArgs.jobConfig wrong " + runArgs.jobConfig)
evgenLog.error("You must supply one and only one jobOption file in DSID directory. It has to start with mc. and end with .py")
sys.exit(1)
jofile = jofiles[0]
@@ -277,21 +277,14 @@ rounding = 0
if hasattr(runArgs,'inputGeneratorFile') and ',' in runArgs.inputGeneratorFile: multiInput = runArgs.inputGeneratorFile.count(',')+1
else:
multiInput = 0
+
+
if evgenConfig.nEventsPerJob < 1:
- raise RunTimeError("evgenConfig.nEventsPerJob must be at least 0")
-else:
- allowed_nEventsPerJob_lt1000 = [1, 2, 5, 10, 20, 25, 50, 100, 200, 500, 1000]
- msg = "evgenConfig.nEventsPerJob = %d: " % evgenConfig.nEventsPerJob
-
- if evgenConfig.nEventsPerJob >= 1000 and evgenConfig.nEventsPerJob % 1000 != 0 and 10000 % evgenConfig.nEventsPerJob != 0:
- msg += "nEventsPerJob in range >= 1000 must be a multiple of 1000 and a divisor of 10000"
- raise RuntimeError(msg)
- elif evgenConfig.nEventsPerJob < 1000 and evgenConfig.nEventsPerJob not in allowed_nEventsPerJob_lt1000:
- msg += "nEventsPerJob in range <= 1000 must be one of %s" % allowed_nEventsPerJob_lt1000
- raise RuntimeError(msg)
- postSeq.CountHepMC.RequestedOutput = evgenConfig.nEventsPerJob if runArgs.maxEvents == -1 else runArgs.maxEvents
- evgenLog.info('Requested output events '+str(postSeq.CountHepMC.RequestedOutput))
+ raise RunTimeError("evgenConfig.nEventsPerJob must be at least 1")
+else:
+ evgenLog.info("evgenConfig.nEventsPerJob = {}, but only {} (dummy) event(s) will be generated by Pythia8 for lhe-only production".format(evgenConfig.nEventsPerJob, postSeq.CountHepMC.RequestedOutput))
+### NB: postSeq.CountHepMC.RequestedOutput is set to 1 in LHEonly.py
## Check that the keywords list is not empty:
if not evgenConfig.keywords:
@@ -373,6 +366,9 @@ if evgenConfig.categories:
evgenLog.warning("Could not find CategoryList.txt file %s in $JOBOPTSEARCHPATH" % lkwfile)
## Set the run numbers
+dsid = os.path.basename(runArgs.jobConfig[0])
+if not dsid.isdigit():
+ dsid = "999999"
svcMgr.EventSelector.RunNumber = int(dsid)
## Include information about generators in metadata
@@ -398,6 +394,39 @@ include("EvgenJobTransforms/Generate_ecmenergies.py")
## Process random seed arg and pass to generators
include("EvgenJobTransforms/Generate_randomseeds.py")
+##=============================================================
+## Check release number
+##=============================================================
+# Function to check blacklist (from Spyros'es logParser.py)
+def checkBlackList(relFlavour,cache,generatorName) :
+ isError = None
+ with open('/cvmfs/atlas.cern.ch/repo/sw/Generators/MC16JobOptions/common/BlackList_caches.txt') as bfile:
+ for line in bfile.readlines():
+ if not line.strip():
+ continue
+ # Blacklisted release flavours
+ badRelFlav=line.split(',')[0].strip()
+ # Blacklisted caches
+ badCache=line.split(',')[1].strip()
+ # Blacklisted generators
+ badGens=line.split(',')[2].strip()
+
+ used_gens = ','.join(generatorName)
+ #Match Generator and release type e.g. AtlasProduction, MCProd
+ if relFlavour==badRelFlav and cache==badCache and re.search(badGens,used_gens) is not None:
+ if badGens=="": badGens="all generators"
+ isError=relFlavour+","+cache+" is blacklisted for " + badGens
+ return isError
+ return isError
+## Announce start of JO checkingrelease nimber checking
+evgenLog.debug("****************** CHECKING RELEASE IS NOT BLACKLISTED *****************")
+rel = os.popen("echo $AtlasVersion").read()
+rel = rel.strip()
+errorBL = checkBlackList("AthGeneration",rel,gennames)
+if (errorBL):
+# evgenLog.warning("This run is blacklisted for this generator, please use a different one !! "+ errorBL )
+ raise RuntimeError("This run is blacklisted for this generator, please use a different one !! "+ errorBL)
+
## Add special config option (extended model info for BSM scenarios)
svcMgr.TagInfoMgr.ExtraTagValuePairs += ["specialConfiguration", evgenConfig.specialConfig ]
@@ -595,12 +624,6 @@ else:
raise RuntimeError("evgenConfig.inputfilecheck specified in %s, but generators %s do not require an input file" %
(runArgs.jobConfig, str(gennames)))
-## Check conf files, as above but for a different command line arg, and with omission allowed
-if hasattr(runArgs, "inputGenConfFile") and runArgs.inputGenConfFile != "NONE":
- if evgenConfig.inputconfcheck and not re.search(evgenConfig.inputconfcheck, runArgs.inputGenConfFile):
- raise RuntimeError("inputGenConfFile=%s is incompatible with inputconfcheck (%s) in %s" %
- (runArgs.inputGenConfFile, evgenConfig.inputconfcheck, runArgs.jobConfig))
-
## Do the aux-file copying
if evgenConfig.auxfiles:
from PyJobTransformsCore.trfutil import get_files
--
GitLab