# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
### This module contains functions which may need to peek at the input file metadata
## Get the logger
from AthenaCommon.Logging import *
simMDlog = logging.getLogger('Sim_Metadata')
def get_metadata(mode='lite'):
# Allow the input check to be skipped. This should only be done in production
# jobs, in order to avoid peeking and spoiling performance on some systems
import os
if not ('G4ATLAS_SKIPFILEPEEK' in os.environ and os.environ['G4ATLAS_SKIPFILEPEEK']):
from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
if athenaCommonFlags.PoolEvgenInput.statusOn:
try:
from PyUtils.MetaReader import read_metadata
input_file = athenaCommonFlags.PoolEvgenInput()[0]
metadata = read_metadata(input_file, mode = mode)
metadata = metadata[input_file] # promote all keys one level up
return metadata
except:
simMDlog.warning("MetaReader failed to open %s", athenaCommonFlags.PoolEvgenInput()[0])
else:
simMDlog.info("G4ATLAS_SKIPFILEPEEK environment variable present, so skipping all input file peeking.")
return None
metadata_lite = get_metadata(mode = 'lite')
metadata_full = get_metadata(mode = 'full')
# Check that we aren't trying to pass a data file as the input to the simulation!
def inputFileValidityCheck():
if metadata_lite is not None:
# Check that event type is IS_SIMULATION (as it must be)
if "eventTypes" in metadata_lite:
if 'IS_SIMULATION' not in metadata_lite['eventTypes']:
msg = "This input file has incorrect eventTypes: %s\n" % str(metadata_lite['eventTypes'])
msg += "Please make sure you have set input file metadata correctly - "
msg += "consider using the job transforms for earlier steps if you aren't already doing so."
simMDlog.fatal(msg)
raise SystemExit("Input file eventTypes is incorrect: please check your evgen jobs.")
else:
simMDlog.warning("Could not find 'eventTypes' key in meta-reader Unable to that check eventTypes is correct.")
else:
simMDlog.info("No input Evgen MetaData object available, so skipping check for input file validity.")
# Check whether mc_channel_number is set in tag_info metadata and add if required.
def patch_mc_channel_numberMetadata(addToFile=True):
if metadata_lite is not None:
mc_channel_number = 0
if 'mc_channel_number' in metadata_lite:
mc_channel_number = int(metadata_lite['mc_channel_number'])
else:
simMDlog.warning("No mc_channel_number in input file metadata. Using run number.")
mc_channel_number = metadata_lite['run_number'][0]
if addToFile:
simMDlog.info('Adding mc_channel_number to /TagInfo: %s', str(mc_channel_number))
# Initialize tag info management
import EventInfoMgt.EventInfoMgtInit
from AthenaCommon.AppMgr import ServiceMgr
ServiceMgr.TagInfoMgr.ExtraTagValuePairs += ["mc_channel_number", str(mc_channel_number)]
return mc_channel_number
else:
simMDlog.info("No input Evgen MetaData object available so skipping patch of mc_channel_number metadata.")
def checkForSpecialConfigurationMetadata():
if metadata_full is not None:
if 'specialConfiguration' in metadata_full['/TagInfo']:
from G4AtlasApps.SimFlags import simFlags
simFlags.specialConfiguration = dict()
item = metadata_full['/TagInfo']['specialConfiguration']
## Parse the specialConfiguration string
## Format is 'key1=value1;key2=value2;...'. or just '
spcitems = item.split(";")
params = {}
someIncludes=[]
for spcitem in spcitems:
#print spcitem
## Ignore empty or "NONE" substrings, e.g. from consecutive or trailing semicolons
if not spcitem or spcitem.upper() == "NONE":
continue
## If not in key=value format, treat as v, with k="preInclude"
if "=" not in spcitem:
spcitem = "preInclude=" + spcitem
## Handle k=v directives
k, v = spcitem.split("=")
if k == "preInclude":
someIncludes+=[v]
else:
params[k] = v
simMDlog.info(params)
simFlags.specialConfiguration.get_Value().update(params)
from AthenaCommon.Include import include
for inc in someIncludes:
include(inc)
else:
simMDlog.info("No input Evgen MetaData object available so skipping check for specialConfiguration metadata.")
def fillCommonMetadata(dbFiller):
pass
def fillAtlasMetadata(dbFiller):
## Write sim parameters as metadata keys
from AthenaCommon.JobProperties import JobProperty
from G4AtlasApps.SimFlags import simFlags
simParams = [sf for sf in dir(simFlags) if isinstance(getattr(simFlags, sf), JobProperty)]
for sp in simParams:
## Don't write out random number seeds or RunDict as metadata
if sp in ("RandomSeedList", "RandomSeedOffset", "RunDict"):
continue
## Don't write out Tool and Service names
if sp in ("TruthService"):
continue
## Only store InitFunction names
if sp in ("InitFunctions"):
initfuncdict = dict()
for level in getattr(simFlags, sp).get_Value():
initfuncdict.setdefault(level, [])
for element in getattr(simFlags, sp).get_Value()[level]:
initfuncdict[level].append(element.__name__)
testValue = str( initfuncdict ) if getattr(simFlags, sp).statusOn else 'default'
dbFiller.addSimParam(sp, testValue)
continue
testValue = str( getattr(simFlags, sp).get_Value() ) if getattr(simFlags, sp).statusOn else 'default'
dbFiller.addSimParam(sp, testValue)
import os
dbFiller.addSimParam('G4Version', str(os.environ['G4VERS']))
dbFiller.addSimParam('RunType', 'atlas')
from AthenaCommon.BeamFlags import jobproperties
dbFiller.addSimParam('beamType', jobproperties.Beam.beamType.get_Value())
## Simulated detector flags: add each enabled detector to the simulatedDetectors list
from AthenaCommon.DetFlags import DetFlags
simDets = []
for det in ['pixel','SCT','TRT','BCM','DBM','Lucid','FwdRegion','ZDC','ALFA','AFP','LAr','HGTD','Tile','MDT','CSC','TGC','RPC','Micromegas','sTGC','Truth']:
attrname = det+"_on"
checkfn = getattr(DetFlags.geometry, attrname, None)
if checkfn is None:
continue
if checkfn():
simDets.append(det)
dbFiller.addSimParam('SimulatedDetectors', repr(simDets))
## Hard-coded simulation hit file magic number (for major changes)
dbFiller.addSimParam('hitFileMagicNumber', '0') ##FIXME Remove this?
def fillTestBeamMetadata(dbFiller):
dbFiller.addSimParam('RunType','testbeam')
SimParams = ['CalibrationRun','DoLArBirk','LArParameterization',
'MagneticField','PhysicsList','Seeds','SeedsG4','SimLayout',
'WorldZRange','NeutronTimeCut','NeutronEnergyCut','ApplyEMCuts','RunNumber']
from G4AtlasApps.SimFlags import simFlags
for o in [ o for o in SimParams if simFlags.__dict__.keys().__contains__(o) ]:
testValue = 'default'
if (simFlags.__dict__.__contains__(o) and simFlags.__dict__.get(o).statusOn) :
testValue = simFlags.__dict__.get(o).get_Value()
if not isinstance(testValue, str):
testValue = str(simFlags.__dict__.get(o).get_Value())
dbFiller.addSimParam(o, testValue)
if (simFlags.__dict__.__contains__('EventFilter') and simFlags.EventFilter.statusOn) :
dbFiller.addSimParam('EtaPhiStatus', str(simFlags.EventFilter.get_Value()['EtaPhiFilters']))
dbFiller.addSimParam('VRangeStatus', str(simFlags.EventFilter.get_Value()['VertexRangeChecker']))
else :
dbFiller.addSimParam('EtaPhiStatus', 'default')
dbFiller.addSimParam('VRangeStatus', 'default')
import os
dbFiller.addSimParam('G4Version', str(os.environ['G4VERS']))
from AthenaCommon.BeamFlags import jobproperties
dbFiller.addSimParam('beamType',jobproperties.Beam.beamType.get_Value())
####### Hard coded simulation hit file magic number (for major changes) ######
dbFiller.addSimParam('hitFileMagicNumber','0')
def fillISFMetadata(dbFiller):
from ISF_Config.ISF_jobProperties import ISF_Flags
dbFiller.addSimParam('Simulator', ISF_Flags.Simulator())
def createSimulationParametersMetadata():
from IOVDbMetaDataTools import ParameterDbFiller
dbFiller = ParameterDbFiller.ParameterDbFiller()
## Set run numbers
minrunnum = 0
maxrunnum = 2147483647 # MAX
from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
from G4AtlasApps.SimFlags import simFlags
if hasattr(simFlags, 'RunNumber') and simFlags.RunNumber.statusOn:
minrunnum = simFlags.RunNumber()
## FIXME need to use maxrunnum = 2147483647 for now to keep overlay working but in the future this should be set properly.
#maxrunnum = minrunnum + 1
elif metadata_lite is not None:
if len(metadata_lite['runNumbers']) > 0:
minrunnum = metadata_lite['runNumbers'][0]
maxrunnum = minrunnum + 1
else:
raise Exception('IllegalRunNumber')
else:
simMDlog.info('Skipping runNumbers setting - would need to set simFlags.RunNumber for this.')
simMDlog.info("Using the following runNumbers range for MetaData IOV: ("+str(minrunnum)+","+str(maxrunnum)+").")
dbFiller.setBeginRun(minrunnum)
dbFiller.setEndRun(maxrunnum)
fillAtlasMetadata(dbFiller)
if simFlags.ISFRun:
fillISFMetadata(dbFiller)
## Write the db info
dbFiller.genSimDb()
folder = "/Simulation/Parameters"
dbConnection = "sqlite://;schema=SimParams.db;dbname=SIMPARAM"
import IOVDbSvc.IOVDb
from AthenaCommon.AppMgr import ServiceMgr
ServiceMgr.IOVDbSvc.Folders += [ folder + "<dbConnection>" + dbConnection + "</dbConnection>" ]
ServiceMgr.IOVDbSvc.FoldersToMetaData += [folder]
ServiceMgr.IOVSvc.partialPreLoadData = True
def createTBSimulationParametersMetadata():
from IOVDbMetaDataTools import ParameterDbFiller
dbFiller = ParameterDbFiller.ParameterDbFiller()
# Set run numbers
minrunnum = 0
maxrunnum = 2147483647 # MAX
from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
from G4AtlasApps.SimFlags import simFlags
if hasattr(simFlags, 'RunNumber') and simFlags.RunNumber.statusOn:
minrunnum = simFlags.RunNumber()
## FIXME need to use maxrunnum = 2147483647 for now to keep overlay working but in the future this should be set properly.
#maxrunnum = minrunnum + 1
elif metadata_lite is not None:
if len(metadata_lite['runNumbers']) > 0:
minrunnum = metadata_lite['runNumbers'][0]
maxrunnum = minrunnum + 1
else:
raise Exception('IllegalRunNumber')
else:
simMDlog.info('Skipping runNumbers setting - would need to set simFlags.RunNumber for this.')
simMDlog.info("Using the following runNumbers range for MetaData IOV: ("+str(minrunnum)+","+str(maxrunnum)+").")
dbFiller.setBeginRun(minrunnum)
dbFiller.setEndRun(maxrunnum)
fillTestBeamMetadata(dbFiller)
if simFlags.ISFRun:
fillISFMetadata(dbFiller)
## Write the db info
dbFiller.genSimDb()
folder = "/Simulation/Parameters"
dbConnection = "sqlite://;schema=SimParams.db;dbname=SIMPARAM"
import IOVDbSvc.IOVDb
from AthenaCommon.AppMgr import ServiceMgr
ServiceMgr.IOVDbSvc.Folders += [ folder + "<dbConnection>" + dbConnection + "</dbConnection>" ]
ServiceMgr.IOVDbSvc.FoldersToMetaData += [folder]
ServiceMgr.IOVSvc.partialPreLoadData = True
# Check on run numbers and update them if necessary
# copied from do_run_number_modifications() in PyG4Atlas.py
def configureRunNumberOverrides():
myRunNumber = 1
myFirstLB = 1
myInitialTimeStamp = 0
from G4AtlasApps.SimFlags import simFlags
if hasattr(simFlags, "RunNumber") and simFlags.RunNumber.statusOn:
myRunNumber = simFlags.RunNumber.get_Value()
simMDlog.info('Found run number %d in sim flags.' % myRunNumber)
## Set event selector details based on evgen metadata
######update the run/event info for each event
from AthenaCommon.AppMgr import ServiceMgr
if not hasattr(ServiceMgr,'EvtIdModifierSvc'):
import AthenaServices.Configurables as asc
ServiceMgr +=asc.EvtIdModifierSvc(EvtStoreName="StoreGateSvc")
from AthenaCommon.AppMgr import theApp
theApp.CreateSvc += ["EvtIdModifierSvc"]
else:
simMDlog.warning('Will override the settings of the EvtIdModifierSvc that was previously set up!')
#fix iov metadata
if not hasattr(ServiceMgr.ToolSvc, 'IOVDbMetaDataTool'):
from AthenaCommon import CfgMgr
ServiceMgr.ToolSvc += CfgMgr.IOVDbMetaDataTool()
ServiceMgr.ToolSvc.IOVDbMetaDataTool.MinMaxRunNumbers = [myRunNumber, 2147483647]
## FIXME need to use maxRunNumber = 2147483647 for now to keep overlay working but in the future this should be set properly.
# Using event numbers to avoid "some very large number" setting
from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
totalNumber = 1000000 # TODO possibly get this from MetaReader??
if athenaCommonFlags.EvtMax() is not None and athenaCommonFlags.EvtMax()>0: totalNumber = athenaCommonFlags.EvtMax()+1
if athenaCommonFlags.SkipEvents() is not None and athenaCommonFlags.SkipEvents()>0: totalNumber += athenaCommonFlags.SkipEvents()
try:
from RunDependentSimComps.RunDMCFlags import runDMCFlags
myInitialTimeStamp = runDMCFlags.RunToTimestampDict.getTimestampForRun(myRunNumber)
#print "FOUND TIMESTAMP ", str(myInitialTimeStamp)
except:
myInitialTimeStamp = 1
ServiceMgr.EvtIdModifierSvc.add_modifier(run_nbr=myRunNumber, lbk_nbr=myFirstLB, time_stamp=myInitialTimeStamp, nevts=totalNumber)
if hasattr(ServiceMgr.EventSelector,'OverrideRunNumberFromInput'): ServiceMgr.EventSelector.OverrideRunNumberFromInput = True
elif metadata_lite is not None:
# Get evgen run number and lumi block
if len(metadata_lite['runNumbers']) > 0:
myRunNumber = metadata_lite['runNumbers'][0]
simMDlog.info('Found runNumbers %d in hits file metadata.'% myRunNumber)
else:
simMDlog.warning('Failed to find runNumbers in hits file metadata.')
if metadata_lite['lumiBlockNumbers']:
myFirstLB = metadata_lite['lumiBlockNumbers'][0]
else:
simMDlog.warning('Requires simFlags.RunNumber to be specified in this running mode.\
Using default value of 1 for RunNumber.')
from AthenaCommon.AppMgr import ServiceMgr
ServiceMgr.EventSelector.OverrideRunNumber = True
ServiceMgr.EventSelector.RunNumber = myRunNumber
ServiceMgr.EventSelector.FirstLB = myFirstLB
# Necessary to avoid a crash
ServiceMgr.EventSelector.InitialTimeStamp = myInitialTimeStamp