From d9b501161f4799a8aa7ee429656b1b3076ebace0 Mon Sep 17 00:00:00 2001
From: Dan Andrei Ciubotaru <dan.andrei.ciubotaru@cern.ch>
Date: Wed, 16 Oct 2019 12:19:59 +0000
Subject: [PATCH] Replacing AthFile with MetaReader in G4AtlasApps package
---
.../G4AtlasApps/python/G4Atlas_Metadata.py | 136 +++++++++---------
1 file changed, 66 insertions(+), 70 deletions(-)
diff --git a/Simulation/G4Atlas/G4AtlasApps/python/G4Atlas_Metadata.py b/Simulation/G4Atlas/G4AtlasApps/python/G4Atlas_Metadata.py
index 8d6096a43f0..94411343947 100644
--- a/Simulation/G4Atlas/G4AtlasApps/python/G4Atlas_Metadata.py
+++ b/Simulation/G4Atlas/G4AtlasApps/python/G4Atlas_Metadata.py
@@ -6,75 +6,74 @@
from AthenaCommon.Logging import *
simMDlog = logging.getLogger('Sim_Metadata')
-def getAthFile():
- ## Allow the input check to be skipped. This should only be done in production
- ## jobs, in order to avoid peeking and spoiling performance on some systems
- inputAthFileObject = None
+
+def get_metadata(mode='lite'):
+ # Allow the input check to be skipped. This should only be done in production
+ # jobs, in order to avoid peeking and spoiling performance on some systems
import os
if not ('G4ATLAS_SKIPFILEPEEK' in os.environ and os.environ['G4ATLAS_SKIPFILEPEEK']):
from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
if athenaCommonFlags.PoolEvgenInput.statusOn:
try:
- import PyUtils.AthFile as af
- inputAthFileObject = af.fopen(athenaCommonFlags.PoolEvgenInput()[0])
+ from PyUtils.MetaReader import read_metadata
+ input_file = athenaCommonFlags.PoolEvgenInput()[0]
+ metadata = read_metadata(input_file, mode = mode)
+ metadata = metadata[input_file] # promote all keys one level up
+ return metadata
except:
- simMDlog.warning("AthFile failed to open %s", athenaCommonFlags.PoolEvgenInput()[0])
+ simMDlog.warning("MetaReader failed to open %s", athenaCommonFlags.PoolEvgenInput()[0])
else:
simMDlog.info("G4ATLAS_SKIPFILEPEEK environment variable present, so skipping all input file peeking.")
- return inputAthFileObject
+ return None
+
-inputAthFileObject = getAthFile()
+metadata_lite = get_metadata(mode = 'lite')
+metadata_full = get_metadata(mode = 'full')
-### Check that we aren't trying to pass a data file as the input to the simulation!
+# Check that we aren't trying to pass a data file as the input to the simulation!
def inputFileValidityCheck():
- from G4AtlasApps.G4Atlas_Metadata import inputAthFileObject
- if inputAthFileObject is not None:
- ## Check that event type is SIMULATION (as it must be)
- if "evt_type" in inputAthFileObject.infos.keys():
- evttypes = inputAthFileObject.infos["evt_type"]
- evttype0 = str(evttypes[0])
- if not evttype0.startswith("IS_SIMULATION"):
- msg = "This input file has incorrect evt_type: %s\n" % str(evttypes)
+ if metadata_lite is not None:
+ # Check that event type is IS_SIMULATION (as it must be)
+ if "eventTypes" in metadata_lite:
+ if 'IS_SIMULATION' not in metadata_lite['eventTypes']:
+ msg = "This input file has incorrect eventTypes: %s\n" % str(metadata_lite['eventTypes'])
msg += "Please make sure you have set input file metadata correctly - "
msg += "consider using the job transforms for earlier steps if you aren't already doing so."
simMDlog.fatal(msg)
- raise SystemExit("Input file evt_type is incorrect: please check your evgen jobs.")
+ raise SystemExit("Input file eventTypes is incorrect: please check your evgen jobs.")
else:
- simMDlog.warning("Could not find 'evt_type' key in athfile.infos. Unable to that check evt_type is correct.")
+ simMDlog.warning("Could not find 'eventTypes' key in meta-reader Unable to that check eventTypes is correct.")
else:
- simMDlog.info("No input Evgen AthFile object available, so skipping check for input file validity.")
+ simMDlog.info("No input Evgen MetaData object available, so skipping check for input file validity.")
-### Check whether mc_channel_number is set in tag_info metadata and add if required.
+
+# Check whether mc_channel_number is set in tag_info metadata and add if required.
def patch_mc_channel_numberMetadata(addToFile=True):
- from G4AtlasApps.G4Atlas_Metadata import inputAthFileObject
- if inputAthFileObject is not None:
- mc_channel_number=0
- if 'mc_channel_number' in inputAthFileObject.infos and len(inputAthFileObject.infos['mc_channel_number'])>0:
- mc_channel_number=inputAthFileObject.infos['mc_channel_number'][0]
- elif 'mc_channel_number' in inputAthFileObject.infos['tag_info']:
- mc_channel_number=inputAthFileObject.infos['tag_info']['mc_channel_number']
+ if metadata_lite is not None:
+ mc_channel_number = 0
+ if 'mc_channel_number' in metadata_lite:
+ mc_channel_number = int(metadata_lite['mc_channel_number'])
else:
- simMDlog.warning("No mc_channel_number in input file metadata. Using run number.")
- mc_channel_number=inputAthFileObject.infos['run_number'][0]
+ simMDlog.warning("No mc_channel_number in input file metadata. Using run number.")
+ mc_channel_number = metadata_lite['run_number'][0]
if addToFile:
- simMDlog.info('Adding mc channel number to taginfo: %s',str(mc_channel_number))
+ simMDlog.info('Adding mc_channel_number to /TagInfo: %s', str(mc_channel_number))
# Initialize tag info management
import EventInfoMgt.EventInfoMgtInit
from AthenaCommon.AppMgr import ServiceMgr
- ServiceMgr.TagInfoMgr.ExtraTagValuePairs += ["mc_channel_number", str(mc_channel_number) ]
+ ServiceMgr.TagInfoMgr.ExtraTagValuePairs += ["mc_channel_number", str(mc_channel_number)]
return mc_channel_number
else:
- simMDlog.info("No input Evgen AthFile object available so skipping patch of mc channel number metadata.")
+ simMDlog.info("No input Evgen MetaData object available so skipping patch of mc_channel_number metadata.")
+
-### Read in special simulation job option fragments based on metadata passed by the evgen stage
def checkForSpecialConfigurationMetadata():
- from G4AtlasApps.G4Atlas_Metadata import inputAthFileObject
- if inputAthFileObject is not None:
- if "specialConfiguration" in inputAthFileObject.infos["tag_info"]:
+ if metadata_full is not None:
+ if 'specialConfiguration' in metadata_full['/TagInfo']:
from G4AtlasApps.SimFlags import simFlags
simFlags.specialConfiguration = dict()
- item = inputAthFileObject.infos["tag_info"]["specialConfiguration"]
+ item = metadata_full['/TagInfo']['specialConfiguration']
## Parse the specialConfiguration string
## Format is 'key1=value1;key2=value2;...'. or just '
spcitems = item.split(";")
@@ -100,7 +99,7 @@ def checkForSpecialConfigurationMetadata():
for inc in someIncludes:
include(inc)
else:
- simMDlog.info("No input Evgen AthFile object available so skipping check for specialConfiguration metadata.")
+ simMDlog.info("No input Evgen MetaData object available so skipping check for specialConfiguration metadata.")
def fillCommonMetadata(dbFiller):
pass
@@ -184,25 +183,24 @@ def fillISFMetadata(dbFiller):
def createSimulationParametersMetadata():
from IOVDbMetaDataTools import ParameterDbFiller
dbFiller = ParameterDbFiller.ParameterDbFiller()
- from G4AtlasApps.G4Atlas_Metadata import inputAthFileObject
## Set run numbers
minrunnum = 0
- maxrunnum = 2147483647 # MAX
+ maxrunnum = 2147483647 # MAX
from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
from G4AtlasApps.SimFlags import simFlags
if hasattr(simFlags, 'RunNumber') and simFlags.RunNumber.statusOn:
minrunnum = simFlags.RunNumber()
## FIXME need to use maxrunnum = 2147483647 for now to keep overlay working but in the future this should be set properly.
#maxrunnum = minrunnum + 1
- elif inputAthFileObject is not None:
- if len(inputAthFileObject.run_numbers) > 0:
- minrunnum = inputAthFileObject.run_numbers[0]
- maxrunnum = minrunnum + 1
- else:
- raise Exception('IllegalRunNumber')
+ elif metadata_lite is not None:
+ if len(metadata_lite['runNumbers']) > 0:
+ minrunnum = metadata_lite['runNumbers'][0]
+ maxrunnum = minrunnum + 1
+ else:
+ raise Exception('IllegalRunNumber')
else:
- simMDlog.info('Skipping run number setting - would need to set simFlags.RunNumber for this.')
- simMDlog.info("Using the following run number range for MetaData IOV: ("+str(minrunnum)+","+str(maxrunnum)+").")
+ simMDlog.info('Skipping runNumbers setting - would need to set simFlags.RunNumber for this.')
+ simMDlog.info("Using the following runNumbers range for MetaData IOV: ("+str(minrunnum)+","+str(maxrunnum)+").")
dbFiller.setBeginRun(minrunnum)
dbFiller.setEndRun(maxrunnum)
@@ -223,8 +221,7 @@ def createSimulationParametersMetadata():
def createTBSimulationParametersMetadata():
from IOVDbMetaDataTools import ParameterDbFiller
dbFiller = ParameterDbFiller.ParameterDbFiller()
- from G4AtlasApps.G4Atlas_Metadata import inputAthFileObject
- ## Set run numbers
+ # Set run numbers
minrunnum = 0
maxrunnum = 2147483647 # MAX
from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
@@ -233,15 +230,15 @@ def createTBSimulationParametersMetadata():
minrunnum = simFlags.RunNumber()
## FIXME need to use maxrunnum = 2147483647 for now to keep overlay working but in the future this should be set properly.
#maxrunnum = minrunnum + 1
- elif inputAthFileObject is not None:
- if len(inputAthFileObject.run_numbers) > 0:
- minrunnum = inputAthFileObject.run_numbers[0]
- maxrunnum = minrunnum + 1
- else:
- raise Exception('IllegalRunNumber')
+ elif metadata_lite is not None:
+ if len(metadata_lite['runNumbers']) > 0:
+ minrunnum = metadata_lite['runNumbers'][0]
+ maxrunnum = minrunnum + 1
+ else:
+ raise Exception('IllegalRunNumber')
else:
- simMDlog.info('Skipping run number setting - would need to set simFlags.RunNumber for this.')
- simMDlog.info("Using the following run number range for MetaData IOV: ("+str(minrunnum)+","+str(maxrunnum)+").")
+ simMDlog.info('Skipping runNumbers setting - would need to set simFlags.RunNumber for this.')
+ simMDlog.info("Using the following runNumbers range for MetaData IOV: ("+str(minrunnum)+","+str(maxrunnum)+").")
dbFiller.setBeginRun(minrunnum)
dbFiller.setEndRun(maxrunnum)
@@ -265,7 +262,6 @@ def configureRunNumberOverrides():
myRunNumber = 1
myFirstLB = 1
myInitialTimeStamp = 0
- from G4AtlasApps.G4Atlas_Metadata import inputAthFileObject
from G4AtlasApps.SimFlags import simFlags
if hasattr(simFlags, "RunNumber") and simFlags.RunNumber.statusOn:
myRunNumber = simFlags.RunNumber.get_Value()
@@ -289,7 +285,7 @@ def configureRunNumberOverrides():
## FIXME need to use maxRunNumber = 2147483647 for now to keep overlay working but in the future this should be set properly.
# Using event numbers to avoid "some very large number" setting
from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
- totalNumber = 1000000 # TODO possibly get this from AthFile??
+ totalNumber = 1000000 # TODO possibly get this from MetaReader??
if athenaCommonFlags.EvtMax() is not None and athenaCommonFlags.EvtMax()>0: totalNumber = athenaCommonFlags.EvtMax()+1
if athenaCommonFlags.SkipEvents() is not None and athenaCommonFlags.SkipEvents()>0: totalNumber += athenaCommonFlags.SkipEvents()
try:
@@ -300,15 +296,15 @@ def configureRunNumberOverrides():
myInitialTimeStamp = 1
ServiceMgr.EvtIdModifierSvc.add_modifier(run_nbr=myRunNumber, lbk_nbr=myFirstLB, time_stamp=myInitialTimeStamp, nevts=totalNumber)
if hasattr(ServiceMgr.EventSelector,'OverrideRunNumberFromInput'): ServiceMgr.EventSelector.OverrideRunNumberFromInput = True
- elif inputAthFileObject is not None:
- ## Get evgen run number and lumi block
- if len(inputAthFileObject.run_numbers) > 0:
- myRunNumber = inputAthFileObject.run_numbers[0]
- simMDlog.info('Found run number %d in hits file metadata.'% myRunNumber)
+ elif metadata_lite is not None:
+ # Get evgen run number and lumi block
+ if len(metadata_lite['runNumbers']) > 0:
+ myRunNumber = metadata_lite['runNumbers'][0]
+ simMDlog.info('Found runNumbers %d in hits file metadata.'% myRunNumber)
else:
- simMDlog.warning('Failed to find run number in hits file metadata.')
- if inputAthFileObject.lumi_block:
- myFirstLB = inputAthFileObject.lumi_block[0]
+ simMDlog.warning('Failed to find runNumbers in hits file metadata.')
+ if metadata_lite['lumiBlockNumbers']:
+ myFirstLB = metadata_lite['lumiBlockNumbers'][0]
else:
simMDlog.warning('Requires simFlags.RunNumber to be specified in this running mode.\
Using default value of 1 for RunNumber.')
--
GitLab