Revert "Merge branch 'revert-b3708b79' into '21.0'"

This reverts merge request !4774

Generators/BeamHaloGenerator/share/formatFlukaFile.pl

+#!/usr/bin/perl
+#
+# A.Sbrizzi
+#
+# A script to format FLUKA HL-LHC beam gas files from Anton Lechner format into Roderick Bruce format
+#
+use strict;
+
+die ("Usage: $0 <input file> <output prefix> <number of events>\n")
+    unless ($#ARGV==2);
+
+my $input_file     = $ARGV[0];
+my $output_prefix  = $ARGV[1];
+my $num_events     = $ARGV[2];
+my $file_counter   = 1; 
+my $event_counter  = 0;
+
+my $event_number   = 0;
+my $particle_gen   = 1;
+my $particle_type  = 0;
+my $stat_weight    = 0;
+my $x_position     = 0;
+my $y_position     = 0;
+my $x_cosine       = 0;
+my $y_cosine       = 0;
+my $t_start        = -1;
+my $total_energy   = 0;
+my $kinetic_energy = 0;
+my $t_primary      = 0;
+my $x_primary      = 0;
+my $y_primary      = 0;
+my $z_primary      = 0;
+
+die ("Cannot open $input_file for reading\n")
+    unless open(INPUTFILE, "< $input_file");
+
+my $output_file = sprintf("$output_prefix%05d.events", $file_counter);
+die ("Cannot open $output_file for writing\n")
+    unless open(OUTPUTFILE, '>', $output_file);
+
+my $last_event = -10;
+
+while (my $line = <INPUTFILE>) {
+    
+    # Remove the new line character
+    chomp $line;
+    
+    # If the string is not refined, skip this line.
+    if (defined $line) {
+	
+	# Split the string, using one or more spaces to denote the splitting.
+	my @frags = split(/\s+/,$line);
+
+	$event_number   = $frags[1];
+	$particle_type  = $frags[2];
+	$stat_weight    = $frags[4];
+	$x_position     = $frags[5];
+	$y_position     = $frags[6];
+	$x_cosine       = $frags[7];
+	$y_cosine       = $frags[8];
+	$total_energy   = $frags[9];
+	$kinetic_energy = $frags[10];
+	$t_primary      = $frags[11];
+	$x_primary      = $frags[12];
+	$y_primary      = $frags[13];
+	$z_primary      = $frags[14];
+
+	if ($event_number != $last_event) {
+
+	    $event_counter = $event_counter + 1;
+
+	    if ($event_counter > $num_events) {
+		
+		close(OUTPUTFILE);
+		
+		$file_counter++;
+		$event_counter = 1;
+
+		$output_file = sprintf("$output_prefix%05d.events", $file_counter);
+
+		die ("Cannot open $output_file for writing\n")
+		    unless open(OUTPUTFILE, '>', $output_file);
+	    }
+	}
+
+	$last_event = $event_number;
+    }
+
+    printf OUTPUTFILE "%10d %3d %6d %27.17e %27.17e %27.17e %27.17e %27.17e %27.17e %3d %27.17e %27.17e %27.17e %27.17e %27.17e \n", 
+    $event_counter, $particle_gen, $particle_type, $kinetic_energy, $stat_weight, $x_position, $y_position, $x_cosine, $y_cosine, $t_start, $total_energy, $x_primary, $y_primary, $z_primary, $t_primary;
+}
+
+close(INPUTFILE);
+close(OUTPUTFILE);

Generators/CosmicGenerator/python/CosmicGeneratorConfig.py

-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
 
 class CavernPropertyCalculator(object):
 
@@ -135,13 +137,13 @@ class CavernPropertyCalculator(object):
             else:
                 from AthenaCommon.AppMgr import theApp
                 GeoModelSvc = theApp.service('GeoModelSvc')
-            if (newSize <=  500000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-03-Bedrock500'
-            elif (newSize <= 1000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-03-Bedrock1000'
-            elif (newSize <= 1500000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-03-Bedrock1500'
-            elif (newSize <= 2000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-03-Bedrock2000'
-            elif (newSize <= 3000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-03-Bedrock3000'
-            elif (newSize <= 4000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-03-Bedrock4000'
-            elif (newSize <= 5000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-03-Bedrock5000'
+            if (newSize <=  500000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-04-Bedrock500'
+            elif (newSize <= 1000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-04-Bedrock1000'
+            elif (newSize <= 1500000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-04-Bedrock1500'
+            elif (newSize <= 2000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-04-Bedrock2000'
+            elif (newSize <= 3000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-04-Bedrock3000'
+            elif (newSize <= 4000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-04-Bedrock4000'
+            elif (newSize <= 5000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-04-Bedrock5000'
         else :
             print "No need to resize the bedrock for cosmic generation"
 

Generators/CosmicGenerator/share/CosmicSliceConfig.py

@@ -46,12 +46,12 @@ if (bedrockDX > 350000 or bedrockDZ > 350000) :
         GeoModelSvc = Configurable.allConfigurables.get('GeoModelSvc')
     else:
         GeoModelSvc = theApp.service('GeoModelSvc')
-    if (newSize <=  500000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-03-Bedrock500'
-    elif (newSize <= 1000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-03-Bedrock1000'
-    elif (newSize <= 1500000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-03-Bedrock1500'
-    elif (newSize <= 2000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-03-Bedrock2000'
-    elif (newSize <= 3000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-03-Bedrock3000'
-    elif (newSize <= 4000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-03-Bedrock4000'
-    elif (newSize <= 5000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-03-Bedrock5000'
+    if (newSize <=  500000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-04-Bedrock500'
+    elif (newSize <= 1000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-04-Bedrock1000'
+    elif (newSize <= 1500000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-04-Bedrock1500'
+    elif (newSize <= 2000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-04-Bedrock2000'
+    elif (newSize <= 3000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-04-Bedrock3000'
+    elif (newSize <= 4000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-04-Bedrock4000'
+    elif (newSize <= 5000000) : GeoModelSvc.CavernInfraVersionOverride = 'CavernInfra-04-Bedrock5000'
 else :
     print "No need to resize the bedrock for cosmic generation"

Generators/CosmicGenerator/src/CosmicGenerator.cxx

@@ -79,7 +79,10 @@ extern "C" float cosmicrndm_(int* /*dummy*/)
 
 //--------------------------------------------------------------------------
 CosmicGenerator::CosmicGenerator(const std::string& name,
-      ISvcLocator* pSvcLocator): GenModule(name,pSvcLocator)
+      ISvcLocator* pSvcLocator)
+  : GenModule(name,pSvcLocator)
+  , m_stopped_tminus(0.)
+  , m_stopped_tplus(0.)
 //--------------------------------------------------------------------------
 {
   //

Generators/Epos_i/src/Epos.cxx

@@ -209,7 +209,7 @@ StatusCode Epos::genInitialize()
   long int si1 = sip[0];
   long int si2 = sip[1];
 
-  int iSeed = si1;     // FIXME ?
+  int iSeed = si1%1000000000;     // FIXME ?
 
   // set up initial values
 

Generators/EvgenJobTransforms/python/EvgenConfig.py

-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
 
 ## Generators providing input events via the LHEF format (MC@NLO produces LHEF
 ## when used in connection with Herwig++, which is fine since this variable is

Generators/EvgenJobTransforms/python/evgenTrfArgs.py

-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
 
 import PyJobTransforms.trfArgClasses as trfArgClasses
 
@@ -99,4 +99,20 @@ def addStdEvgenArgs(parser):
                         default=trfArgClasses.argInt(0, runarg=True),
                         type=trfArgClasses.argFactory(trfArgClasses.argInt, runarg=True))
 
+    parser.add_argument('--lheOnly', '--lheOnly', group='Evgen',
+                        default=trfArgClasses.argInt(0, runarg=True),
+                        type=trfArgClasses.argFactory(trfArgClasses.argInt, runarg=True),
+                        help='Set to 1 if Evgen file should be removed')
+
+    parser.add_argument("--localPath", group="Evgen",
+                        help="Name of local path to JOs",
+                        type=trfArgClasses.argFactory(trfArgClasses.argString, runarg=True))
 
+    parser.add_argument('--saveList', '--saveList', group='Evgen',
+                        help="a comma-separated list of files to be saved in output or a single digit if the whole output directory is to be saved",
+                        type=trfArgClasses.argFactory(trfArgClasses.argSubstepList, runarg=True))
+
+    parser.add_argument('--cleanOut', '--cleanOut', group='Evgen',
+                        default=trfArgClasses.argInt(0, runarg=True),
+                        type=trfArgClasses.argFactory(trfArgClasses.argInt, runarg=True),
+                        help='Set to any digit !=0 if the output files should be cleaned up --saveList may help to save selected files')

Generators/EvgenJobTransforms/scripts/AfterBurner_tf.py

-#! /usr/bin/env python
+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
 
-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+#! /usr/bin/env python
 
 """
 Run event simulation and produce an EVNT file.

Generators/EvgenJobTransforms/scripts/EvtGenFromEVNT_tf.py

-#! /usr/bin/env python
+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
 
-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+#! /usr/bin/env python
 
 import sys
 import time

Generators/EvgenJobTransforms/scripts/Generate_tf.py

-#! /usr/bin/env python
+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
 
-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+#! /usr/bin/env python
 
 """
 Run event simulation and produce an EVNT file.
@@ -18,7 +18,7 @@ import PyJobTransforms.trfArgClasses as trfArgClasses
 
 ## Prodsys1 hack...
 # TODO: Remove!
-ListOfDefaultPositionalKeys=['--AMIConfig', '--AMITag', '--argJSON', '--asetup', '--athena', '--athenaMPMergeTargetSize', '--athenaopts', '--attempt', '--checkEventCount', '--command', '--dumpJSON', '--dumpPickle', '--ecmEnergy', '--env', '--eventAcceptanceEfficiency', '--evgenJobOpts', '--execOnly', '--fileValidation', '--firstEvent', '--ignoreErrors', '--ignoreFiles', '--ignorePatterns', '--imf', '--inputEVNT_PreFile', '--inputFileValidation', '--inputGenConfFile', '--inputGeneratorFile', '--jobConfig', '--jobid', '--maxEvents', '--orphanKiller', '--outputEVNTFile', '--outputEVNT_PreFile', '--outputFileValidation', '--outputNTUP_TRUTHFile', '--outputTXTFile', '--parallelFileValidation', '--postExec', '--postInclude', '--preExec', '--preInclude', '--printEvts', '--randomSeed', '--reportName', '--reportType', '--rivetAnas', '--runNumber', '--showGraph', '--showPath', '--showSteps', '--skipEvents', '--skipFileValidation', '--skipInputFileValidation', '--skipOutputFileValidation', '--steering', '--taskid', '--tcmalloc', '--valgrind', '--valgrindbasicopts', '--valgrindextraopts']
+ListOfDefaultPositionalKeys=['--AMIConfig', '--AMITag', '--argJSON', '--asetup', '--athena', '--athenaMPMergeTargetSize', '--athenaopts', '--attempt', '--checkEventCount', '--command', '--dumpJSON', '--dumpPickle', '--ecmEnergy', '--env', '--eventAcceptanceEfficiency', '--evgenJobOpts', '--execOnly', '--fileValidation', '--firstEvent', '--ignoreErrors', '--ignoreFiles', '--ignorePatterns', '--imf', '--inputEVNT_PreFile', '--inputFileValidation', '--inputGenConfFile', '--inputGeneratorFile', '--jobConfig', '--jobid', '--maxEvents', '--orphanKiller', '--outputEVNTFile', '--outputEVNT_PreFile', '--outputFileValidation', '--outputNTUP_TRUTHFile', '--outputTXTFile', '--parallelFileValidation', '--postExec', '--postInclude', '--preExec', '--preInclude', '--printEvts', '--randomSeed', '--reportName', '--reportType', '--rivetAnas', '--runNumber', '--showGraph', '--showPath', '--showSteps', '--skipEvents', '--skipFileValidation', '--skipInputFileValidation', '--skipOutputFileValidation', '--steering', '--taskid', '--tcmalloc', '--valgrind', '--valgrindbasicopts', '--valgrindextraopts', '--lheOnly', '--localPath', '--cleanOut', '--saveList']
 
 class EvgenExecutor(athenaExecutor):
     "Specialised trf executor class for event generation jobs"
@@ -132,9 +132,16 @@ class EvgenExecutor(athenaExecutor):
             else:    
                 msg.info('ATLAS_SW_BASE path: %s' %sw_base)
 
+            local_path = None
+            if ("localPath" in self._trf.argdict ):
+                local_path = self._trf.argdict["localPath"].value
+                print("local path",local_path)
             cvmfs_path = os.path.join(sw_base, "atlas.cern.ch")
             
-            if os.path.exists(cvmfs_path):
+            if ((local_path is not None) and (os.path.exists(local_path))) :
+              mk_jo_proxy(local_path, "MC15JobOptions","_joproxy15")
+              print("JO fragments taken from local path i.e. ",local_path)             
+            elif os.path.exists(cvmfs_path):
                 # TODO: Make the package name configurable
                 if "MC14" in str(joparam):
                     cvmfs_mc14 = os.path.join(cvmfs_path, "repo/sw/Generators/MC14JobOptions/latest/")
@@ -154,10 +161,29 @@ class EvgenExecutor(athenaExecutor):
 
         ## Expand tarball input event and generator conf files, if provided
         if "inputGeneratorFile" in self._trf.argdict:
-            expand_if_archive(self._trf.argdict["inputGeneratorFile"].value)
+#            expand_if_archive(self._trf.argdict["inputGeneratorFile"].value)
+            myinputfiles = self._trf.argdict["inputGeneratorFile"].value
+            genInputFiles = myinputfiles.split(',')
+            for file in genInputFiles:
+               expand_if_archive(file)
         if "inputGenConfFile" in self._trf.argdict:
             expand_if_archive(self._trf.argdict["inputGenConfFile"].value)
 
+def move_files(main_dir,tmp_dir,whitelist):
+    files = os.listdir(tmp_dir)
+    print("list of files ",files)
+    print("white list ",whitelist)    
+    files.sort()
+    for f in files:
+       for i in whitelist:
+            if i in f:
+                src = tmp_dir+"/"+f
+                dest = main_dir+"/"+f
+#                print("src ",src)
+#                print("dest ",dest)
+                os.rename(src,dest)
+                break
+
 
 def getTransform():
     exeSet = set()
@@ -175,11 +201,65 @@ def getTransform():
 @sigUsrStackTrace
 def main():
     msg.info("This is %s" % sys.argv[0])
+    main_dir = os.getcwd()
     trf = getTransform()
     trf.parseCmdLineArgs(sys.argv[1:])
+#    print("main ",main_dir)
+    if (("cleanOut" in trf.argdict) and (trf.argdict["cleanOut"].value != 0)):
+      name_tmpdir = "tmprun"
+      tmp_dir = os.path.join(main_dir, name_tmpdir)
+#    print("tmpdir ",tmp_dir)
+      if os.path.isdir(tmp_dir):
+         shutil.rmtree(tmp_dir, ignore_errors=True)
+      os.mkdir("tmprun")
+      os.chdir("tmprun")
+      tmp_dir = os.getcwd()
+      whitelist_in = ['MC','group','TXT']
+      move_files(tmp_dir,main_dir,whitelist_in)
+      
+#    trf = getTransform()
+#    trf.parseCmdLineArgs(sys.argv[1:])
     trf.execute()
     trf.generateReport()
     msg.info("%s stopped at %s, trf exit code %d" % (sys.argv[0], time.asctime(), trf.exitCode))
+
+
+    if "outputTXTFile" in trf.argdict:
+      whitelist_out.append('TXT')
+
+# read files/dirs that should be saved and if present in cwd - remove
+
+    if (("cleanOut" in trf.argdict) and (trf.argdict["cleanOut"].value!=0)):
+#       print("mamy cleanOut !!!",trf.argdict["cleanOut"].value) 
+       whitelist_out = ['log.generate','.root']
+       if "saveList" in trf.argdict:
+         saveList_dic= trf.argdict["saveList"].value
+         saveList_str= str(saveList_dic)
+         saveList_str=saveList_str[10:-3]
+         saveList= saveList_str.split(",")
+         for item in saveList:
+           test_ex = os.path.join(main_dir,str(item))
+           if os.path.isdir(test_ex):
+             shutil.rmtree(test_ex, ignore_errors=True)
+#           print("directory removed ", test_ex)
+           elif os.path.isfile(test_ex):
+             os.remove(test_ex)
+#           print("file removed ", test_ex)        
+#       new_list = list(list(int(a) for a in b) for b in T1 if a.isdigit())
+         if not saveList[0].isdigit():
+             whitelist_out=whitelist_out+saveList 
+
+       move_files(main_dir,tmp_dir,whitelist_out)
+       os.chdir(main_dir)
+       if "saveList" not in trf.argdict:
+         shutil.rmtree(tmp_dir, ignore_errors=True)
+#    elif firstSaveList!=1:
+       elif not saveList[0].isdigit():
+         shutil.rmtree(tmp_dir, ignore_errors=True)  
+
+    if (("lheOnly" in trf.argdict ) and (trf.argdict["lheOnly"].value == 1)):
+          outputName = ''.join(trf.argdict["outputEVNTFile"].value)
+          os.remove(outputName)
     sys.exit(trf.exitCode)
 
 

Generators/EvgenJobTransforms/scripts/Rivet_tf.py

-#! /usr/bin/env python
+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
 
-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+#! /usr/bin/env python
 
 """
 Take an input EVNT file and run Rivet on it

Generators/EvgenJobTransforms/share/Generate_TruthJets.py

+
+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
 ## Configure and schedule jet finding algorithms for the jet 
 ## containers defined in the user's JO fragment
 ##
@@ -6,12 +9,14 @@
 
 
 # Convert HepMC event record to xAODTruth for use by jet tools
+print 'bedzie konwersja !!'
 from xAODTruthCnv.xAODTruthCnvConf import xAODMaker__xAODTruthCnvAlg
 if not hasattr(prefiltSeq, 'xAODCnv'):
-    prefiltSeq += xAODMaker__xAODTruthCnvAlg('xAODCnv')
+    prefiltSeq += xAODMaker__xAODTruthCnvAlg('xAODCnv',WriteTruthMetaData=False)
 prefiltSeq.xAODCnv.AODContainerName = 'GEN_EVENT'
 
 # Turn off jet tools specific to detector-level jets
+print 'bedzie wylaczenie !!'
 from JetRec.JetRecFlags import jetFlags
 jetFlags.useTracks = False
 jetFlags.eventShapeTools = []
@@ -23,6 +28,7 @@ else: raise NameError('jetFlags has no attribute usePflow or usePFlow')
 jetFlags.useCaloQualityTool = False
 
 # Attach jet algorithms
+print 'podlacz algorytmy jetowe !!'
 from JetRec.JetAlgorithm import addJetRecoToAlgSequence
 if not hasattr(prefiltSeq, 'jetalg'):
     addJetRecoToAlgSequence(prefiltSeq)

Generators/EvgenJobTransforms/share/Generate_ecmenergies.py

+
+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
+
 ## Propagate energy argument to the generators
 # TODO: Standardise energy setting in the GenModule interface
 
@@ -30,8 +34,8 @@ elif 'Herwig' in evgenConfig.generators:
     genSeq.Herwig.HerwigCommand.append("beam2energy %s" % eBeam)
 elif 'Herwigpp' in evgenConfig.generators:
     genSeq.Herwigpp.Commands.append("set /Herwig/Generators/LHCGenerator:EventHandler:LuminosityFunction:Energy %s" % eCM)
-elif 'Herwig7' in evgenConfig.generators:
-    genSeq.Herwig7.Commands.append("set /Herwig/Generators/LHCGenerator:EventHandler:LuminosityFunction:Energy %s" % eCM)
+#elif 'Herwig7' in evgenConfig.generators:
+#    genSeq.Herwig7.Commands.append("set /Herwig/Generators/LHCGenerator:EventHandler:LuminosityFunction:Energy %s" % eCM)
 elif 'Sherpa' in evgenConfig.generators:
     genSeq.Sherpa_i.Parameters.append("BEAM_ENERGY_1=%s" % eBeam)
     genSeq.Sherpa_i.Parameters.append("BEAM_ENERGY_2=%s" % eBeam)

Generators/EvgenJobTransforms/share/Generate_randomseeds.py

+
+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
+
 ## Set random seeds depending on used generators
 
 genseeds = {

Generators/EvgenJobTransforms/share/skeleton.ABtoEVGEN.py

+
+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
 """Functionality core of the Generate_tf transform"""
 
 ##==============================================================
@@ -252,16 +255,21 @@ if joparts[0].startswith("MC"): #< if this is an "official" JO
     expectedgenpart = expectedgenpart.replace("HerwigJimmy", "Herwig")
     def _norm(s):
         # TODO: add EvtGen to this normalization for MC14?
-        return s.replace("Photospp", "").replace("Photos", "").replace("Tauola", "").replace("Tauolapp", "").replace("TauolaPP", "")
+        return s.replace("Photospp", "").replace("Photos", "").replace("TauolaPP", "").replace("Tauolapp", "").replace("Tauola", "")
     def _norm2(s):
         return s.replace("Py", "Pythia").replace("MG","MadGraph").replace("Ph","Powheg").replace("Hpp","Herwigpp").replace("H7","Herwig7").replace("Sh","Sherpa").replace("Ag","Alpgen").replace("EG","EvtGen").replace("PG","ParticleGun")
 
     def _short2(s):
          return s.replace("Pythia","Py").replace("MadGraph","MG").replace("Powheg","Ph").replace("Herwigpp","Hpp").replace("Herwig7","H7").replace("Sherpa","Sh").replace("Alpgen","Ag").replace("EvtGen","EG").replace("PG","ParticleGun")
+ 
+    if genpart != _norm(expectedgenpart)  and _norm2(genpart) != _norm(expectedgenpart):
+        evgenLog.error("Expected first part of JO name to be '%s' or '%s', but found '%s'" % (_norm(expectedgenpart), _norm(_short2(expectedgenpart)), genpart))
+        evgenLog.error("gennames '%s' " %(expectedgenpart))
+        sys.exit(1)
 
-    if genpart != expectedgenpart and _norm(genpart) != _norm(expectedgenpart) and _norm2(genpart) != expectedgenpart and _norm2(genpart) != _norm(expectedgenpart):
-        evgenLog.error("Expected first part of JO name to be '%s' or '%s' or '%s', but found '%s'" % (_norm(expectedgenpart), expectedgenpart, _short2(expectedgenpart), genpart))
-        sys.exit(1) 
+#    if genpart != expectedgenpart and _norm(genpart) != _norm(expectedgenpart) and _norm2(genpart) != expectedgenpart and _norm2(genpart) != _norm(expectedgenpart):
+#        evgenLog.error("Expected first part of JO name to be '%s' or '%s' or '%s', but found '%s'" % (_norm(expectedgenpart), expectedgenpart, _short2(expectedgenpart), genpart))
+#        sys.exit(1) 
 
     del _norm
     ## Check if the tune/PDF part is needed, and if so whether it's present
@@ -273,17 +281,38 @@ if joparts[0].startswith("MC"): #< if this is an "official" JO
 
 ## Check that the evgenConfig.minevents setting is acceptable
 ## minevents defines the production event sizes and must be sufficiently "round"
+if hasattr(runArgs,'inputGeneratorFile') and ',' in runArgs.inputGeneratorFile:   
+   multiInput = runArgs.inputGeneratorFile.count(',')+1
+else:
+   multiInput = 0
 if evgenConfig.minevents < 1:
     raise RunTimeError("evgenConfig.minevents must be at least 1")
 else:
-    allowed_minevents_lt1000 = [1, 2, 5, 10, 20, 25, 50, 100, 200, 500]
+    allowed_minevents_lt1000 = [1, 2, 5, 10, 20, 25, 50, 100, 200, 500, 1000]
     msg = "evgenConfig.minevents = %d: " % evgenConfig.minevents
-    if evgenConfig.minevents >= 1000 and evgenConfig.minevents % 1000 != 0:
-        msg += "minevents in range >= 1000 must be a multiple of 1000"
-        raise RuntimeError(msg)
+    if multiInput !=0 :
+        dummy_minevents = evgenConfig.minevents*(multiInput)
+        evgenLog.info('Replacing input minevents '+str(evgenConfig.minevents)+' with calculated '+str(dummy_minevents) + ' rounded to ' + str(int(round(dummy_minevents))))
+        evgenConfig.minevents = dummy_minevents
+        rest1000 = evgenConfig.minevents % 1000
+        if multiInput !=0 :
+            rounding=1
+            if rest1000 < 1000-rest1000:
+                evgenLog.info('Replacing input minevents '+str(evgenConfig.minevents)+' with calculated '+str(evgenConfig.minevents-rest1000))
+                evgenConfig.minevents = evgenConfig.minevents-rest1000
+            else:
+                evgenLog.info('Replacing input minevents '+str(evgenConfig.minevents)+' with calculated '+str(evgenConfig.minevents-rest1000+1000))
+                evgenConfig.minevents = evgenConfig.minevents-rest1000+1000
+        else:    
+           msg += "minevents in range >= 1000 must be a multiple of 1000"
+           raise RuntimeError(msg)
     elif evgenConfig.minevents < 1000 and evgenConfig.minevents not in allowed_minevents_lt1000:
-        msg += "minevents in range < 1000 must be one of %s" % allowed_minevents_lt1000
-        raise RuntimeError(msg)
+        if multiInput !=0:
+           rounding=1 
+           evgenConfig.minevents=min(allowed_minevents_lt1000,key=lambda x:abs(x-evgenConfig.minevents))
+        else:
+           msg += "minevents in range <= 1000 must be one of %s" % allowed_minevents_lt1000
+           raise RuntimeError(msg)
 
 ## Check that the keywords are in the list of allowed words (and exit if processing an official JO)
 if evgenConfig.keywords:

Generators/EvgenJobTransforms/share/skeleton.EVGENtoRivet.py

+
+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
 """Functionality core of the Rivet_tf transform"""
 
 ##==============================================================

Generators/EvgenJobTransforms/share/skeleton.EvtGenFromEVNT.py

+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
 #############################
 ## basic jobO configuration
 include("PATJobTransforms/CommonSkeletonJobOptions.py")

Generators/EvgenJobTransforms/share/skeleton.GENtoEVGEN.py

+
+#  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
 """Functionality core of the Generate_tf transform"""
 
 ##==============================================================
@@ -6,7 +9,7 @@
 
 ## Create sequences for generators, clean-up algs, filters and analyses
 ## and import standard framework objects with standard local scope names
-import os, re, string
+import os, re, string, subprocess
 import AthenaCommon.AlgSequence as acas
 import AthenaCommon.AppMgr as acam
 from AthenaCommon.AthenaCommonFlags import jobproperties
@@ -120,10 +123,10 @@ if not hasattr(postSeq, "CopyEventWeight"):
 # TODO: Rewrite in Python?
 from EvgenProdTools.EvgenProdToolsConf import CountHepMC
 svcMgr.EventSelector.FirstEvent = runArgs.firstEvent
-
+theApp.EvtMax = -1
 # This is necessary for athenaMP
-if hasattr(runArgs, "maxEvents"):
-  theApp.EvtMax = runArgs.maxEvents
+#if hasattr(runArgs, "maxEvents"):
+#  theApp.EvtMax = runArgs.maxEvents
 
 if not hasattr(postSeq, "CountHepMC"):
     postSeq += CountHepMC()
@@ -131,7 +134,7 @@ if not hasattr(postSeq, "CountHepMC"):
 
 postSeq.CountHepMC.FirstEvent = runArgs.firstEvent
 postSeq.CountHepMC.CorrectHepMC = False
-postSeq.CountHepMC.CorrectEventID = True
+postSeq.CountHepMC.CorrectEventID = False
 
 ## Print out the contents of the first 5 events (after filtering)
 # TODO: Allow configurability from command-line/exec/include args
@@ -260,17 +263,23 @@ if joparts[0].startswith("MC"): #< if this is an "official" JO
     expectedgenpart = expectedgenpart.replace("HerwigJimmy", "Herwig")
     def _norm(s):
         # TODO: add EvtGen to this normalization for MC14?
-        return s.replace("Photospp", "").replace("Photos", "").replace("Tauola", "").replace("Tauolapp", "").replace("TauolaPP", "")
+        return s.replace("Photospp", "").replace("Photos", "").replace("TauolaPP", "").replace("Tauolapp", "").replace("Tauola", "")
     def _norm2(s):
         return s.replace("Py", "Pythia").replace("MG","MadGraph").replace("Ph","Powheg").replace("Hpp","Herwigpp").replace("H7","Herwig7").replace("Sh","Sherpa").replace("Ag","Alpgen").replace("EG","EvtGen").replace("PG","ParticleGun")
         
     def _short2(s):
          return s.replace("Pythia","Py").replace("MadGraph","MG").replace("Powheg","Ph").replace("Herwigpp","Hpp").replace("Herwig7","H7").replace("Sherpa","Sh").replace("Alpgen","Ag").replace("EvtGen","EG").replace("PG","ParticleGun")
      
-    if genpart != expectedgenpart and _norm(genpart) != _norm(expectedgenpart) and _norm2(genpart) != expectedgenpart and _norm2(genpart) != _norm(expectedgenpart):
-        evgenLog.error("Expected first part of JO name to be '%s' or '%s' or '%s', but found '%s'" % (_norm(expectedgenpart), expectedgenpart, _short2(expectedgenpart), genpart))
+#    if genpart != expectedgenpart and _norm(genpart) != _norm(expectedgenpart) and _norm2(genpart) != expectedgenpart and _norm2(genpart) != _norm(expectedgenpart):
+#        evgenLog.error("Expected first part of JO name to be '%s' or '%s' or '%s', but found '%s'" % (_norm(expectedgenpart), expectedgenpart, _short2(expectedgenpart), genpart))
+#        evgenLog.error("gennames '%s' " %(expectedgenpart))
+#        sys.exit(1)
+
+    if genpart != _norm(expectedgenpart)  and _norm2(genpart) != _norm(expectedgenpart):
+        evgenLog.error("Expected first part of JO name to be '%s' or '%s', but found '%s'" % (_norm(expectedgenpart), _norm(_short2(expectedgenpart)), genpart))
         evgenLog.error("gennames '%s' " %(expectedgenpart))
         sys.exit(1)
+
     del _norm
     ## Check if the tune/PDF part is needed, and if so whether it's present
     if not gens_notune(gennames) and len(jo_physshortparts) < 3:
@@ -288,20 +297,50 @@ if gen_require_steering(gennames):
 
 ## Check that the evgenConfig.minevents setting is acceptable
 ## minevents defines the production event sizes and must be sufficiently "round"
+rounding = 0
+if hasattr(runArgs,'inputGeneratorFile') and ',' in runArgs.inputGeneratorFile:   multiInput = runArgs.inputGeneratorFile.count(',')+1
+else:
+   multiInput = 0
+   
 if evgenConfig.minevents < 1:
     raise RunTimeError("evgenConfig.minevents must be at least 1")
 else:
-    allowed_minevents_lt1000 = [1, 2, 5, 10, 20, 25, 50, 100, 200, 500]
+    allowed_minevents_lt1000 = [1, 2, 5, 10, 20, 25, 50, 100, 200, 500, 1000]
     msg = "evgenConfig.minevents = %d: " % evgenConfig.minevents
+    if multiInput !=0 :
+        dummy_minevents = evgenConfig.minevents*(multiInput)
+        evgenLog.info('Replacing input minevents '+str(evgenConfig.minevents)+' with calculated '+str(dummy_minevents))
+        evgenConfig.minevents = dummy_minevents
+
     if evgenConfig.minevents >= 1000 and evgenConfig.minevents % 1000 != 0:
-        msg += "minevents in range >= 1000 must be a multiple of 1000"
-        raise RuntimeError(msg)
+        rest1000 = evgenConfig.minevents % 1000
+        if multiInput !=0 :
+            rounding=1
+            if rest1000 < 1000-rest1000:
+                evgenLog.info('Replacing minevents '+str(evgenConfig.minevents)+' with roundeded '+str(evgenConfig.minevents-rest1000))
+                evgenConfig.minevents = evgenConfig.minevents-rest1000
+            else:
+                evgenLog.info('Replacing input minevents '+str(evgenConfig.minevents)+' with calculated '+str(evgenConfig.minevents-rest1000+1000))
+                evgenConfig.minevents = evgenConfig.minevents-rest1000+1000
+        else:    
+           msg += "minevents in range >= 1000 must be a multiple of 1000"
+           raise RuntimeError(msg)
     elif evgenConfig.minevents < 1000 and evgenConfig.minevents not in allowed_minevents_lt1000:
-        msg += "minevents in range < 1000 must be one of %s" % allowed_minevents_lt1000
-        raise RuntimeError(msg)
-    else:
-        postSeq.CountHepMC.RequestedOutput = evgenConfig.minevents if runArgs.maxEvents == -1 else runArgs.maxEvents
-
+        if multiInput !=0:
+           rounding=1
+#           minimum_list = [abs(variable - evgenConfig.minevents) for variable in allowed_minevents_lt1000]
+#           from operator import itemgetter
+#           evgenLog.info("Minimum list: %s" % minimum_list)
+#           evgenLog.info("index of the min. value: " + str(min(enumerate(minimum_list), key=itemgetter(1))[0]) + " and the value is: " + str(allowed_minevents_lt1000[min(enumerate(minimum_list), key=itemgetter(1))[0]]))
+           round_minevents=min(allowed_minevents_lt1000,key=lambda x:abs(x-evgenConfig.minevents))
+           evgenLog.info('Replacing minevents lt 1000 '+str(evgenConfig.minevents)+' with rounded '+str(round_minevents))
+           evgenConfig.minevents=round_minevents
+        else:
+           msg += "minevents in range <= 1000 must be one of %s" % allowed_minevents_lt1000
+           raise RuntimeError(msg)
+#    else:
+    postSeq.CountHepMC.RequestedOutput = evgenConfig.minevents if runArgs.maxEvents == -1 or rounding==1 else runArgs.maxEvents
+    evgenLog.info('Requested output events '+str(postSeq.CountHepMC.RequestedOutput))
 
 ## Check that the keywords are in the list of allowed words (and exit if processing an official JO)
 if evgenConfig.keywords:
@@ -474,6 +513,54 @@ def find_unique_file(pattern):
         raise RuntimeError("More than one '%s' file found" % pattern)
     return files[0]
 
+# This function merges a list of input LHE file to make one outputFile.  The header is taken from the first
+# file, but the number of events is updated to equal the total number of events in all the input files
+def merge_lhe_files(listOfFiles,outputFile):
+    if(os.path.exists(outputFile)):
+      print "outputFile ",outputFile," already exists.  Will rename to ",outputFile,".OLD"
+      os.rename(outputFile,outputFile+".OLD")
+    output = open(outputFile,'w')
+    holdHeader = ""
+    nevents=0
+    for file in listOfFiles:
+       cmd = "grep /event "+file+" | wc -l"
+       nevents+=int(subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True))
+
+    for file in listOfFiles:
+       inHeader = True
+       header = ""
+       print "*** Starting file ",file
+       for line in open(file,"r"):
+##        Reading first event signals that we are done with all the header information
+##        Using this approach means the script will properly handle any metadata stored
+##        at the beginning of the file.  Note:  aside from the number of events, no metadata
+##        is updated after the first header is read (eg the random number seed recorded will be
+##        that of the first file.
+          if("<event" in line and inHeader):
+             inHeader = False
+             if(len(holdHeader)<1):
+                holdHeader = header
+                output.write(header)
+##        each input file ends with "</LesHouchesEvents>".  We don't want to write this out until all
+##        the files have been read.  The elif below writes out all the events.
+          elif(not inHeader and not ("</LesHouchesEvents>" in line)):
+              output.write(line)
+          if(inHeader):
+##           Format for storing number of events different in MG and Powheg 
+             if("nevents" in line):
+##              MG5 format is "n = nevents"
+                tmp = line.split("=")
+                line = line.replace(tmp[0],str(nevents))
+             elif("numevts" in line):
+##              Powheg format is numevts n
+                tmp = line.split(" ")
+                nnn = str(nevents)
+                line = line.replace(tmp[1],nnn)
+             header+=line
+    output.write("</LesHouchesEvents>")
+    output.close()
+
+
 def mk_symlink(srcfile, dstfile):
     "Make a symlink safely"
     if dstfile:
@@ -493,13 +580,42 @@ if eventsFile or datFile:
     if evgenConfig.inputfilecheck and not re.search(evgenConfig.inputfilecheck, runArgs.inputGeneratorFile):
         raise RuntimeError("inputGeneratorFile=%s is incompatible with inputfilecheck '%s' in %s" %
                            (runArgs.inputGeneratorFile, evgenConfig.inputfilecheck, runArgs.jobConfig))
-    inputroot = os.path.basename(runArgs.inputGeneratorFile).split("._")[0]
+#    inputroot = os.path.basename(runArgs.inputGeneratorFile).split("._")[0]
     if datFile:
+#        inputroot = os.path.basename(runArgs.inputGeneratorFile).split("._")[0]
+        inputroot = os.path.basename(runArgs.inputGeneratorFile).split(".tar.")[0]
         realDatFile = find_unique_file('*%s*.dat' % inputroot)
         mk_symlink(realDatFile, datFile)
     if eventsFile:
-        realEventsFile = find_unique_file('*%s.*.ev*ts' % inputroot)
-        mk_symlink(realEventsFile, eventsFile)
+#        realEventsFile = find_unique_file('*%s.*.ev*ts' % inputroot)
+#        mk_symlink(realEventsFile, eventsFile)
+        myinputfiles = runArgs.inputGeneratorFile
+        genInputFiles = myinputfiles.split(',')
+        numberOfFiles = len(genInputFiles)
+        # if there is a single file, make a symlink.  If multiple files, merge them into one output eventsFile
+        if(numberOfFiles<2):
+#           inputroot = os.path.basename(runArgs.inputGeneratorFile).split("._")[0]
+           inputroot = os.path.basename(runArgs.inputGeneratorFile).split(".tar.")[0]
+           realEventsFile = find_unique_file('*%s.*ev*ts' % inputroot)
+           mk_symlink(realEventsFile, eventsFile)
+        else:
+           allFiles = []
+           for file in genInputFiles:
+#             Since we can have multiple files from the same task, inputroot must include more of the filename
+#             to make it unique
+#              input0 = os.path.basename(file).split("._")[0]
+#              input1 = (os.path.basename(file).split("._")[1]).split(".")[0]
+#              inputroot = input0+"._"+input1
+              inputroot = os.path.basename(file).split(".tar.")[0]
+#              print "inputroot ",inputroot
+              realEventsFile = find_unique_file('*%s.*ev*ts' % inputroot)
+#             The only input format where merging is permitted is LHE
+              with open(realEventsFile, 'r') as f:
+                 first_line = f.readline()
+                 if(not ("LesHouche" in first_line)):
+                    raise RuntimeError("%s is NOT a LesHouche file" % realEventsFile)
+                 allFiles.append(realEventsFile)
+           merge_lhe_files(allFiles,eventsFile)
 else:
     if hasattr(runArgs, "inputGeneratorFile") and runArgs.inputGeneratorFile != "NONE":
         raise RuntimeError("inputGeneratorFile arg specified for %s, but generators %s do not require an input file" %

Generators/EvgenJobTransforms/test/EvgenJobTransforms_TestConfiguration.xml

@@ -54,18 +54,30 @@
       <timelimit>15</timelimit>
       <prescript>source ${ATN_PACKAGE}/test/setup_testenv</prescript>
     </TEST>
+
+    <TEST name="Py8TauolaPPZtautau" type="script" suite="EvgenJobTransforms-Core">
+      <doc>Tests a Pythia8 Z->tautau job, with Photos++, Tauola++ and EG</doc>
+      <classification>
+        <displayClass>OfflineValidation</displayClass>
+        <displayProcess>EvGen</displayProcess>
+        <displayComponent>Athena-Core</displayComponent>
+      </classification>
+      <options_atn>${ATN_PACKAGE}/test/test_ph8TauolappEG_ztautau</options_atn>
+      <timelimit>15</timelimit>
+      <prescript>source ${ATN_PACKAGE}/test/setup_testenv</prescript>
+    </TEST>
    
 
     
-    <TEST name="Py8EvtGen" type="script" suite="EvgenJobTransforms-Core">
-      <doc>Test Pythia8+EvtGen</doc>
+    <TEST name="powhegpy8TauolaPPEG" type="script" suite="EvgenJobTransforms-Core">
+      <doc>Test Powheg+Pythia8+TauolaPP+EvtGen</doc>
       <classification>
         <displayClass>OfflineValidation</displayClass>
         <displayProcess>EvGen</displayProcess>
         <displayComponent>Athena-Core</displayComponent>
       </classification>
-      <options_atn>${ATN_PACKAGE}/test/test_pythia8_evtgen</options_atn>
-      <timelimit>15</timelimit>
+      <options_atn>${ATN_PACKAGE}/test/test_powhegpy8TauolaPPEG</options_atn>
+      <timelimit>25</timelimit>
       <prescript>source ${ATN_PACKAGE}/test/setup_testenv</prescript>
     </TEST>
     
@@ -155,17 +167,29 @@
     
 
     
-    <TEST name="Herwig++" type="script" suite="EvgenJobTransforms-Core">
-      <doc>Test minimum bias generation with Herwigpp</doc>
+    <TEST name="Herwig7" type="script" suite="EvgenJobTransforms-Core">
+      <doc>Test minimum bias generation with Herwig7</doc>
       <classification>
         <displayClass>OfflineValidation</displayClass>
         <displayProcess>EvGen</displayProcess>
         <displayComponent>Athena-Core</displayComponent>
       </classification>
-      <options_atn>${ATN_PACKAGE}/test/test_herwigpp</options_atn>
+      <options_atn>${ATN_PACKAGE}/test/test_herwig7</options_atn>
       <timelimit>15</timelimit>
       <prescript>source ${ATN_PACKAGE}/test/setup_testenv</prescript>
     </TEST>
+
+    <TEST name="Herwig7dijet" type="script" suite="EvgenJobTransforms-Core">
+      <doc>Test dijets generation with Herwig7</doc>
+      <classification>
+        <displayClass>OfflineValidation</displayClass>
+        <displayProcess>EvGen</displayProcess>
+        <displayComponent>Athena-Core</displayComponent>
+      </classification>
+      <options_atn>${ATN_PACKAGE}/test/test_herwig7dijet</options_atn>
+      <timelimit>35</timelimit>
+      <prescript>source ${ATN_PACKAGE}/test/setup_testenv</prescript>
+    </TEST>
     
     <TEST name="PowhegWenu" type="script" suite="EvgenJobTransforms-Core">
       <doc>Test W-enu generation with Powheg</doc>

Generators/EvgenJobTransforms/test/setup_testenv

 export TESTDATA=/afs/cern.ch/atlas/offline/ProdData/17.2.X
 export JODIR=/afs/cern.ch/atlas/groups/Generators/MC14JobOptions/latest
-export JODIR15=/afs/cern.ch/atlas/groups/Generators/MC15JobOptions/latest
+export JODIR15=/cvmfs/atlas.cern.ch/repo/sw/Generators/MC15JobOptions/latest