diff --git a/Event/EventOverlay/EventOverlayJobTransforms/CMakeLists.txt b/Event/EventOverlay/EventOverlayJobTransforms/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..7fcde34640cafff33c8d163f6a47ca31f9b2846d --- /dev/null +++ b/Event/EventOverlay/EventOverlayJobTransforms/CMakeLists.txt @@ -0,0 +1,18 @@ +################################################################################ +# Package: EventOverlayJobTransforms +################################################################################ + +# Declare the package name: +atlas_subdir( EventOverlayJobTransforms ) + +# Declare the package's dependencies: +atlas_depends_on_subdirs( PUBLIC + Tools/PyJobTransforms + Tools/PyJobTransformsCore ) + +# Install files from the package: +atlas_install_python_modules( python/*.py ) +atlas_install_joboptions( share/*.py ) +atlas_install_runtime( scripts/*.py ) +atlas_install_runtime( share/*.db ) + diff --git a/Event/EventOverlay/EventOverlayJobTransforms/python/__init__.py b/Event/EventOverlay/EventOverlayJobTransforms/python/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..74583d364ec2ca794156596c7254d9b234a940c6 --- /dev/null +++ b/Event/EventOverlay/EventOverlayJobTransforms/python/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration + diff --git a/Event/EventOverlay/EventOverlayJobTransforms/python/overlayTransformUtils.py b/Event/EventOverlay/EventOverlayJobTransforms/python/overlayTransformUtils.py index 5c61d5baa8fc9b82cdc9aa68905e439f599fea6e..9efd5e3efa67c0cda5967332842925f0de94ad96 100644 --- a/Event/EventOverlay/EventOverlayJobTransforms/python/overlayTransformUtils.py +++ b/Event/EventOverlay/EventOverlayJobTransforms/python/overlayTransformUtils.py @@ -2,7 +2,7 @@ ## @brief Specialist reconstruction and bytestream transforms # @author atlas-comp-jt-dev@cern.ch -# @version $Id: overlayTransformUtils.py 687841 2015-08-06 09:20:13Z tkharlam $ +# @version $Id: overlayTransformUtils.py 733403 2016-04-01 18:38:53Z ahaas $ import os import re @@ -21,47 +21,58 @@ import PyJobTransforms.trfValidation as trfValidation from PyJobTransforms.trfExitCodes import trfExit from PyJobTransforms.trfExe import athenaExecutor -## @brief write me -# numbers class BSJobSplitterExecutor(athenaExecutor): def preExecute(self, input = set(), output = set()): - msg.debug('Preparing for execution of {0} with inputs {1} and outputs {2}'.format(self.name, input, output)) + msg.info('Preparing for BSJobSplitterExecutor execution of {0} with inputs {1} and outputs {2}'.format(self.name, input, output)) + + #See if we need to unpack a TAR file + if 'hitarFile' in self.conf.argdict: + print "Untarring inputHITARFile", self.conf.argdict['hitarFile'].value + try: + f=tarfile.open(name=self.conf.argdict['hitarFile'].value[0]) + f.list() + f.extractall() + f.close() + except Exception, e: + raise trfExceptions.TransformSetupException(trfExit.nameToCode('TRF_EXEC_SETUP_FAIL'), 'Error while unpacking and extracting HI input files for transform: {0}'.format(e)) # There are two ways to configure this transform: # - Give an inputZeroBiasBSFile argument directly - # - Give a overlayConfigFile and jobNumber argument + # - Give a inputBSCONFIGFile and jobNumber argument # Check now that we have a configuration that works - if 'inputZeroBiasBSFile' in self.conf.argdict and 'overlayConfigFile' in self.conf.argdict: - raise trfExceptions.TransformSetupException(trfExit.nameToCode('TRF_EXEC_SETUP_FAIL'), 'Both inputZeroBiasBSFile and overlayConfigFile have been specified - please use only one.') + if 'inputZeroBiasBSFile' in self.conf.argdict and 'inputBSCONFIGFile' in self.conf.argdict: + #raise trfExceptions.TransformSetupException(trfExit.nameToCode('TRF_EXEC_SETUP_FAIL'), 'Both inputZeroBiasBSFile and inputBSCONFIGFile have been specified - please use only one.') + del self.conf.argdict['inputZeroBiasBSFile'] + print "WARNING - removed the inputZeroBiasBSFile argument, because inputZeroBiasBSFile and inputBSCONFIGFile were already specified" - if 'overlayConfigFile' in self.conf.argdict: + if 'inputBSCONFIGFile' in self.conf.argdict: if 'jobNumber' not in self.conf.argdict: - raise trfExceptions.TransformSetupException(trfExit.nameToCode('TRF_EXEC_SETUP_FAIL'), 'overlayConfigFile is specified, but no jobNumber was given.') + raise trfExceptions.TransformSetupException(trfExit.nameToCode('TRF_EXEC_SETUP_FAIL'), 'inputBSCONFIGFile is specified, but no jobNumber was given.') # Job number has to wrap around from 500, dropping back to 1 wrappedJobNumber = (self.conf.argdict['jobNumber'].value-1)%500 + 1 self._inputFilelist = 'filelist_{0}.txt'.format(wrappedJobNumber) - self._lblList = 'lbn_anal_map_{0}.txt'.format(wrappedJobNumber) + self._lbnList = 'lbn_anal_map_{0}.txt'.format(wrappedJobNumber) try: - f=tarfile.open(name=self.conf.argdict['overlayConfigFile'].value) + print self.conf.argdict['inputBSCONFIGFile'].value + f=tarfile.open(name=self.conf.argdict['inputBSCONFIGFile'].value[0]) f.extract('filelist_{0}.txt'.format(wrappedJobNumber)) f.extract('lbn_anal_map_{0}.txt'.format(wrappedJobNumber)) f.close() - bsInputs = open(self._inputFilelist).readline().rstrip().split(',') self.conf.addToArgdict('inputZeroBiasBSFile', trfArgClasses.argBSFile(bsInputs, io='input', type='BS', subtype='BS_ZeroBias')) - self.conf.addToDataDictionary('BS', self.conf.argdict['inputZeroBiasBSFile']) - input.add('BS') + self.conf.addToDataDictionary('ZeroBiasBS', self.conf.argdict['inputZeroBiasBSFile']) + input.add('ZeroBiasBS') msg.info('Validating resolved input bytestream files') - trfValidation.performStandardFileValidation({'BS': self.conf.argdict['inputZeroBiasBSFile']}, io='input') + trfValidation.performStandardFileValidation({'ZeroBiasBS': self.conf.argdict['inputZeroBiasBSFile']}, io='input') except Exception, e: raise trfExceptions.TransformSetupException(trfExit.nameToCode('TRF_EXEC_SETUP_FAIL'), 'Error while unpacking and extracting input files for transform: {0}'.format(e)) # Now setup correct input arguments - self.conf.argdict['InputLbnMapFile'] = trfArgClasses.argString(self._lblList) + self.conf.argdict['InputLbnMapFile'] = trfArgClasses.argString(self._lbnList) self.conf.argdict['InputFileMapFile'] = trfArgClasses.argString(self._inputFilelist) else: @@ -170,9 +181,15 @@ def addOverlay_BSArguments(parser): addCommonOverlayArguments(parser) ### Add Sub-step Methods +def addOverlayBSTrigFilterSubstep(executorSet): + executorSet.add(BSJobSplitterExecutor(name = 'BSTrigFilter', skeletonFile = 'EventOverlayJobTransforms/skeleton.BSOverlayFilter_tf.py', substep='overlayBSTrigFilt', + perfMonFile = 'ntuple.pmon.gz', inData = ['ZeroBiasBS','BS_SKIM'], outData = ['BS_TRIGSKIM'])) def addOverlayBSFilterSubstep(executorSet): executorSet.add(BSJobSplitterExecutor(name = 'BSFilter', skeletonFile = 'EventOverlayJobTransforms/skeleton.BSOverlayFilter_tf.py', substep='overlayBSFilt', - perfMonFile = 'ntuple.pmon.gz', inData = ['ZeroBiasBS'], outData = ['BS_SKIM','TXT_EVENTID'])) + perfMonFile = 'ntuple.pmon.gz', inData = ['ZeroBiasBS','BSCONFIG'], outData = ['BS_SKIM','TXT_EVENTID'])) +def addOverlayHITARMakerSubstep(executorSet): + executorSet.add(BSJobSplitterExecutor(name = 'HITARMaker', skeletonFile = 'EventOverlayJobTransforms/skeleton.HITARMaker_tf.py', substep='HITARMaker', + inData = ['TXT_EVENTID'], outData = ['TAR_CONFIG'])) def addOverlay_PoolSubstep(executorSet): executorSet.add(athenaExecutor(name = 'OverlayPool', skeletonFile = 'EventOverlayJobTransforms/skeleton.OverlayPool_tf.py', @@ -185,6 +202,10 @@ def addOverlay_BSSubstep(executorSet): inData = [('HITS', 'BS_SKIM')], outData = ['RDO', 'RDO_SGNL'])) ### Append Sub-step Methods +def appendOverlayBSTrigFilterSubstep(trf): + executor = set() + addOverlayBSTrigFilterSubstep(executor) + trf.appendToExecutorSet(executor) def appendOverlayBSFilterSubstep(trf): executor = set() addOverlayBSFilterSubstep(executor) diff --git a/Event/EventOverlay/EventOverlayJobTransforms/python/overlayTrfArgs.py b/Event/EventOverlay/EventOverlayJobTransforms/python/overlayTrfArgs.py index 4f2407f2dd21fea9899b34d2d63cb019596a7463..ca824c30d380a2f36ac9f23923d66fa641b0a896 100644 --- a/Event/EventOverlay/EventOverlayJobTransforms/python/overlayTrfArgs.py +++ b/Event/EventOverlay/EventOverlayJobTransforms/python/overlayTrfArgs.py @@ -18,27 +18,36 @@ def addOverlayChainOverrideArgs(parser): ## Add Overlay transform arguments to an argparse ArgumentParser def addOverlayBSFilterArgs(parser): parser.defineArgGroup('Overlay Filter', 'Overlay filter transform arguments') - parser.add_argument('--overlayConfigFile', - type=argFactory(argSubstep, defaultSubstep='overlayBSFilt'), #argFactory(argSubstep, defaultSubstep='overlayBSFilt'), + parser.add_argument('--inputBSCONFIGFile', + type=argFactory(argFile, io='input', type='TAR', subtype='TAR_CONFIG'), help='Input overlay configuration tarball file', group='Overlay Filter') parser.add_argument('--inputZeroBiasBSFile', nargs='+', type=argFactory(argBSFile, io='input', type='BS', subtype='BS_ZeroBias'), help='Input overlay BS file(s)', group='Overlay Filter') parser.add_argument('--inputFilterFile', type=argFactory(argString), - help='Input overlay BS filter file (for HI)', group='Overlay Filter') + help='Input overlay BS filter file (or trigger file)', group='Overlay Filter') parser.add_argument('--lumiBlockMapFile', - type=argFactory(argSubstep, defaultSubstep='overlayBSFilt'), + type=argFactory(argString), help='Lumi block information file', group='Overlay Filter') parser.add_argument('--outputBS_SKIMFile', '--outputBSFile', type=argFactory(argBSFile, io='output', type='BS', subtype='BS_SKIM'), help='Output skimmed BS file', group='Overlay Filter') - -# parser.add_argument('--eventIdFile', -# type=argFactory(argSubstep, defaultSubstep='overlayBSFilt'), help='The name of the file to write to for EventIdModifierSvc lines', group='Overlay Filter') - parser.add_argument('--outputTXT_EVENTIDFile', default = 'events.txt', - type=argFactory(argFile, io = 'output', type='TXT', subtype='TXT_EVENTID'), help='The name of the file to write the EventIdModifierSvc config for the EVNTtoHITS step', group='Overlay Filter') - + parser.add_argument('--inputBS_SKIMFile', '--inputBSFile', + type=argFactory(argBSFile, io='input', type='BS', subtype='BS_SKIM'), + help='Input skimmed BS file', group='Overlay Filter') + parser.add_argument('--outputBS_TRIGSKIMFile', '--outputBSTRIGFile', + type=argFactory(argBSFile, io='output', type='BS', subtype='BS_TRIGSKIM'), + help='Output skimmed BS file', group='Overlay Filter') + parser.add_argument('--HIfiltervtxdirectory', + type=argFactory(argString), + help='The directory containing the filter/vtx.data.txt files for the HITAR creation during HI MinBias skimming', group='Overlay Filter') + parser.add_argument('--outputTXT_EVENTIDFile', + type=argFactory(argFile, io = 'output', type='TXT', subtype='TXT_EVENTID'), + help='file for EventIdModifierSvc config for the EVNTtoHITS step', group='Overlay Filter') + parser.add_argument('--outputTAR_CONFIGFile', + type=argFactory(argFile, io = 'output', type='TAR', subtype='TAR_CONFIG'), + help='Out TAR file of filter/vtx.data.txt files for the skimmed events', group='overlay Filter') parser.add_argument('--jobNumber', type=argFactory(argInt), help='Job number', group='Overlay Filter') @@ -48,6 +57,9 @@ def addOverlayBSFilterArgs(parser): parser.add_argument('--maxFilesPerSubjob', type=argFactory(argSubstepInt, defaultSubstep='overlayBSFilt'), help='Number of bytestream input files for each athena subjob', group='Overlay Filter') + parser.add_argument('--WriteRDOFileMetaData', + type=argFactory(argBool), default = 'false', + help='Write the /Digitization/Parameters/ metadata into the RDO file', group='Overlay Filter') # jobNumber=102 # InputDataTarFile=/afs/cern.ch/work/e/efeld/overlay/prep/mytar.tar.gz @@ -92,4 +104,7 @@ def addOverlayInputSimArgs(parser): parser.add_argument('--inputTXT_EVENTIDFile', nargs='+', type=argFactory(argFile, io='input', type='TXT', subtype='TXT_EVENTID'), help='The name of the file to read to configure the EventIdModifierSvc', group='EventOverlayInputSim') + parser.add_argument('--hitarFile', + type=argFactory(argFile, io='input', type='TAR', subtype='TAR_CONFIG'), + help='Input Heavy Ion vertex and filter tarball file', group='EventOverlayInputSim') diff --git a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/atlas_error_ignore.db b/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/atlas_error_ignore.db deleted file mode 100755 index 1e5500232123bdacecedca5db939dd702c5f0da7..0000000000000000000000000000000000000000 --- a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/atlas_error_ignore.db +++ /dev/null @@ -1,90 +0,0 @@ -#Each line contains 3 fields, separated by comma's: -#atlas_release_regexp, who_prints_it, error_message_regexp -# For the regular expression syntax that can be used in <error_message_regexp>, see: -# http://docs.python.org/lib/re-syntax.html -# Note in particular the special regexp characters that need to be backslashed if meant litteral: ()[]{}^$.*+? -# In constructing the total regular expression used to match the lines: -# - whitespace is stripped of both ends of the fields <atlas_release_regexp> and <who_prints_it>, -# and from the right end of <error_message_regexp> -# - zero or more whitespace characters are allowed between <who_prints_it> and <error_message_regexp> -# - if the <who_prints_it> field is empty, the <error_message_regexp> is the total regexp. -# error detection can be tested by running on a relevant log file: -# checklog.py someLogFile - -## Errors to ignore for ALL releases -## ================================= -ALL ,.*?, INFO .+ -ALL ,ByteStreamInputSvc,ERROR Skipping bad event -ALL ,ToolSvc.CscSplitClusterFitter,ERROR Peak-to-Val dist is [-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? Val-to-Peak dist is [-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? Shouldnot be negative value :[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? [-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? [-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? -ALL ,AlgErrorAuditor,ERROR Illegal Return Code: Algorithm CscThresholdClusterBuilder reported an ERROR, but returned a StatusCode "SUCCESS" -ALL ,AlgErrorAuditor,ERROR Illegal Return Code: Algorithm InDetSCTRawDataProvider reported an ERROR, but returned a StatusCode "SUCCESS" -ALL ,(?:Py:)?Athena , ERROR inconsistent case used in property name ".*?" of ApplicationMgr -ALL ,(?:Py:)?Athena , ERROR Algorithm ".*?": not in TopAlg or other known list, no properties set -ALL ,(?:Py:)?Athena , ERROR Algorithm ".*?": type missing, no properties set -ALL ,(?:Py:)?Athena , ERROR attempt to add .* to non-existent property .*? -ALL ,(?:Py:)?Configurable, ERROR .* undeclared or uses a backdoor -ALL ,(?:Py:)?Configurable, ERROR children\(\) is deprecated -ALL ,(?:Py:)?Configurable, ERROR getChildren\(\) returns a copy -ALL ,(?:Py:)?Configurable, ERROR jobOptName\(\) is deprecated -# Reco -ALL ,(?:Py:)?Configurable, ERROR attempt to add a duplicate \(CellCalibrator.CellCalibrator.H1WeightCone7H1Tower\) -ALL ,(?:Py:)?ResourceLimits,ERROR failed to set max resource limits -ALL ,AlgErrorAuditor, ERROR Illegal Return Code: Algorithm StreamESD reported an ERROR, but returned a StatusCode "SUCCESS" -# Trigger BStoRDO -ALL ,AthenaRefIOHandler, ERROR Failed to set ElementLink -ALL ,ElementLink, ERROR toPersistent: the internal state of link -ALL ,StoreGateSvc, ERROR record: object not added to store -ALL ,StoreGateSvc, ERROR setupProxy:: error setting up proxy -ALL ,AlgErrorAuditor, ERROR Illegal Return Code: Algorithm MooHLTAlgo -ALL ,AlgErrorAuditor, ERROR Illegal Return Code: Algorithm TrigSteer_EF -ALL ,AlgErrorAuditor, ERROR Illegal Return Code: Algorithm muFast_(?:Muon|900GeV) - -# Trigger reco_ESD -ALL ,THistSvc, ERROR already registered an object with identifier "/EXPERT/ -ALL ,RpcRawDataNtuple , ERROR .* -ALL ,CBNT_L1CaloROD\S+ , ERROR .* -ALL ,CBNTAA_Tile\S+ , ERROR .* -ALL ,TileDigitsMaker , ERROR .* -ALL ,MdtDigitToMdtRDO , ERROR .* -ALL ,HelloWorld , ERROR .* -ALL ,HelloWorld , FATAL .* -ALL ,PythiaB , ERROR ERROR in PYTHIA PARAMETERS -ALL ,ToolSvc , ERROR Tool .* not found and creation not requested -ALL ,ToolSvc , ERROR Unable to finalize the following tools -ALL ,ToolSvc , ERROR Factory for Tool .* not found -ALL ,CBNT_Audit , ERROR Memory leak!.* -ALL ,ToolSvc.InDetSCTRodDecoder , ERROR Unknown offlineId for OnlineId* -ALL ,THistSvc.sysFinali, FATAL Standard std::exception is caught -ALL ,,.*Message limit reached for .* -ALL ,,\s+ERROR IN C-S .*=.* -ALL ,,.*ERROR\s+\|.* -ALL ,,^\s*FATAL ERROR\s*$ -ALL ,,ERROR \(poolDb\): -ALL ,,ERROR \(pool\): -ALL ,,ERROR - G4Navigator::ComputeStep\(\) -ALL ,,.*ERROR OCCURED DURING A SECONDARY SCATTER AND WAS -ALL ,THistSvc , ERROR already registered an object with identifier .* -ALL ,,ERROR MuonDetectorManager::getCscReadoutElement stNameindex out of range .* -ALL ,muFast_\S+ , ERROR CSM for Subsystem \d+, MrodId \d+, LinkId \d+ not found -ALL ,TRTDetectorManager , FATAL Unable to apply Inner Detector alignments -ALL ,TRTDetectorManager , ERROR AlignableTransformContainer for key \/TRT\/Align is empty -ALL ,,ERROR in Single_Process::CalculateTotalXSec -ALL ,,.*ERROR WITH DELM.* -#ALL ,ToolSvc.TrigTSerializer,ERROR Errors while decoding -ALL ,AlgErrorAuditor,ERROR Illegal Return Code: Algorithm - - -## Errors to ignore for specific releases -## ====================================== -# bug #47761 -15.[0-99].[0-99]|(?:rel|dev|bug)_[0-6],PoolSvc, ERROR Failed to get ContainerHandle (for: POOLContainer_DataHeader_p3|to set POOL property\.) -# Cosmic Reco -14.[0-99].[0-99]|(?:rel|dev|bug)_[0-6],(?:Py:)?Configurable, ERROR attempt to add a duplicate \(.*\) \.\.\. dupe ignored -14.[0-99].[0-99]|(?:rel|dev|bug)_[0-6],ToolSvc, ERROR Cannot create tool ICalorimeterNoiseTool .* -14.[0-99].[0-99]|(?:rel|dev|bug)_[0-6],ToolSvc.MdtPrepDataProviderTool, ERROR Error in MDT RDO decoder for subdetId/mrodId/csmId* -14.[0-99].[0-99]|(?:rel|dev|bug)_[0-6],AlgErrorAuditor, ERROR Illegal Return Code: Algorithm MdtRdoToMdtPrepData reported an ERROR* -14.[0-99].[0-99]|(?:rel|dev|bug)_[0-6],\S+::fillBranch, ERROR <index out of range, (pat|hit) not added to ntuple> 100 -# csc_recoESD, 14.2.21.1 -14.2.[0-99]|(?:rel|dev|bug)_[0-6],TrigSteer_EF, ERROR std exception thrown from algorithm \(basic error\)To many objects in trigger collection of one type, end: 65535 >= 65535 -14.2.[0-99]|(?:rel|dev|bug)_[0-6],AlgErrorAuditor, ERROR Illegal Return Code: Algorithm TrigCaloCellMaker_jet reported an ERROR, but returned a StatusCode "SUCCESS" - diff --git a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/data12_8TeV.periodAllYear_DetStatus-v49-pro13-03_CoolRunQuery-00-04-08_Atlas_Ready.xml b/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/data12_8TeV.periodAllYear_DetStatus-v49-pro13-03_CoolRunQuery-00-04-08_Atlas_Ready.xml deleted file mode 100644 index 1238728e2d807c226be8705b60f7b58b1a3c4675..0000000000000000000000000000000000000000 --- a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/data12_8TeV.periodAllYear_DetStatus-v49-pro13-03_CoolRunQuery-00-04-08_Atlas_Ready.xml +++ /dev/null @@ -1,475 +0,0 @@ -<?xml version="1.0" ?> -<!DOCTYPE LumiRangeCollection - SYSTEM 'http://atlas-runquery.cern.ch/LumiRangeCollection.dtd'> -<!-- Good-runs-list created by AtlRunQuery.py on 2012-07-20 02:04:53.491441 --> -<?xml-stylesheet type="text/xsl" href="/atlasdqm/grlview/grl.xsl" title="grlview" ?> -<LumiRangeCollection> - <NamedLumiRange> - <Name>Atlas_Ready</Name> - <Version>2.1</Version> - <Metadata Name="Query">find run data12_8TeV.periodAllYear and partition ATLAS and db DATA and ptag data12_8TeV and dq global_status DEFECTS#DetStatus-v49-pro13-03 g and dq lumi DEFECTS#DetStatus-v49-pro13-03 g </Metadata> - <Metadata Name="RunList">206614,206573,206564,206497,206409,206369,206368,206367,206299,205113,205112,205071,205055,205017,205016,205010,204976,204955,204954,204932,204910,204857,204853,204796,204772,204769,204763,204726,204707,204668,204633,204564,204474,204442,204416,204265,204240,204158,204153,204134,204073,204071,204026,204025,203934,203876,203875,203792,203779,203760,203745,203739,203719,203680,203636,203605,203602,203524,203523,203456,203454,203432,203353,203336,203335,203277,203258,203256,203228,203195,203191,203169,203027,202991,202987,202965,202798,202740,202712,202668,202660,201556,201555,201494,201489,201289,201280,201269,201257,201191,201190,201138,201120,201113,201052,201006,200987,200982,200967,200965,200926,200913,200863,200842,200841</Metadata> - <Metadata Name="RQTSVNVersion">CoolRunQuery-00-04-08</Metadata> - <Metadata Name="StreamListInfo"> - <Stream Name="physics_Background" TotalNumOfEvents="24753317" NumOfSelectedEvents="21341263"/> - <Stream Name="physics_Bphysics" TotalNumOfEvents="75013593" NumOfSelectedEvents="74950674"/> - <Stream Name="physics_CosmicCalo" TotalNumOfEvents="15697532" NumOfSelectedEvents="7295368"/> - <Stream Name="physics_CosmicMuons" TotalNumOfEvents="378039" NumOfSelectedEvents="376009"/> - <Stream Name="physics_Egamma" TotalNumOfEvents="234689196" NumOfSelectedEvents="233427111"/> - <Stream Name="physics_EnhancedBias" TotalNumOfEvents="3246211" NumOfSelectedEvents="3246211"/> - <Stream Name="physics_HadDelayed" TotalNumOfEvents="71148836" NumOfSelectedEvents="70609675"/> - <Stream Name="physics_IDCosmic" TotalNumOfEvents="157654" NumOfSelectedEvents="157188"/> - <Stream Name="physics_JetCalibDelayed" TotalNumOfEvents="1476687" NumOfSelectedEvents="1385121"/> - <Stream Name="physics_JetTauEtmiss" TotalNumOfEvents="259059470" NumOfSelectedEvents="257665735"/> - <Stream Name="physics_L1Calo" TotalNumOfEvents="454255" NumOfSelectedEvents="0"/> - <Stream Name="physics_L1Muon" TotalNumOfEvents="49709" NumOfSelectedEvents="0"/> - <Stream Name="physics_MinBias" TotalNumOfEvents="21497504" NumOfSelectedEvents="21116234"/> - <Stream Name="physics_Muons" TotalNumOfEvents="226312112" NumOfSelectedEvents="222999951"/> - <Stream Name="physics_Standby" TotalNumOfEvents="45123129" NumOfSelectedEvents="0"/> - <Stream Name="physics_ZeroBias" TotalNumOfEvents="2619584" NumOfSelectedEvents="2431086"/> - <Stream Name="physics_ZeroBiasOverlay" TotalNumOfEvents="24409616" NumOfSelectedEvents="24266437"/> - </Metadata> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">206614</Run> - <LBRange Start="206" End="275"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">206573</Run> - <LBRange Start="3" End="188"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">206564</Run> - <LBRange Start="102" End="392"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">206497</Run> - <LBRange Start="60" End="82"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">206409</Run> - <LBRange Start="212" End="881"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">206369</Run> - <LBRange Start="192" End="756"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">206368</Run> - <LBRange Start="3" End="521"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">206367</Run> - <LBRange Start="33" End="63"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">206299</Run> - <LBRange Start="42" End="85"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">205113</Run> - <LBRange Start="590" End="745"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">205112</Run> - <LBRange Start="264" End="480"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">205071</Run> - <LBRange Start="224" End="1313"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">205055</Run> - <LBRange Start="301" End="1246"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">205017</Run> - <LBRange Start="3" End="407"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">205016</Run> - <LBRange Start="3" End="365"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">205010</Run> - <LBRange Start="78" End="198"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204976</Run> - <LBRange Start="123" End="331"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204955</Run> - <LBRange Start="3" End="466"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204954</Run> - <LBRange Start="87" End="324"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204932</Run> - <LBRange Start="92" End="244"/> - <LBRange Start="246" End="649"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204910</Run> - <LBRange Start="302" End="749"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204857</Run> - <LBRange Start="126" End="201"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204853</Run> - <LBRange Start="46" End="214"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204796</Run> - <LBRange Start="93" End="121"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204772</Run> - <LBRange Start="3" End="499"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204769</Run> - <LBRange Start="113" End="517"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204763</Run> - <LBRange Start="56" End="1015"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204726</Run> - <LBRange Start="136" End="154"/> - <LBRange Start="156" End="237"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204707</Run> - <LBRange Start="196" End="201"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204668</Run> - <LBRange Start="29" End="695"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204633</Run> - <LBRange Start="158" End="327"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204564</Run> - <LBRange Start="585" End="1475"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204474</Run> - <LBRange Start="96" End="843"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204442</Run> - <LBRange Start="177" End="260"/> - <LBRange Start="271" End="271"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204416</Run> - <LBRange Start="15" End="323"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204265</Run> - <LBRange Start="134" End="1580"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204240</Run> - <LBRange Start="589" End="1447"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204158</Run> - <LBRange Start="340" End="958"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204153</Run> - <LBRange Start="71" End="293"/> - <LBRange Start="295" End="306"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204134</Run> - <LBRange Start="169" End="268"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204073</Run> - <LBRange Start="144" End="405"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204071</Run> - <LBRange Start="377" End="551"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204026</Run> - <LBRange Start="117" End="574"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">204025</Run> - <LBRange Start="123" End="202"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203934</Run> - <LBRange Start="653" End="1170"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203876</Run> - <LBRange Start="460" End="1154"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203875</Run> - <LBRange Start="74" End="151"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203792</Run> - <LBRange Start="138" End="211"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203779</Run> - <LBRange Start="319" End="689"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203760</Run> - <LBRange Start="197" End="224"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203745</Run> - <LBRange Start="3" End="695"/> - <LBRange Start="699" End="760"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203739</Run> - <LBRange Start="558" End="988"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203719</Run> - <LBRange Start="80" End="534"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203680</Run> - <LBRange Start="352" End="689"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203636</Run> - <LBRange Start="61" End="1052"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203605</Run> - <LBRange Start="314" End="336"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203602</Run> - <LBRange Start="25" End="938"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203524</Run> - <LBRange Start="61" End="367"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203523</Run> - <LBRange Start="241" End="327"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203456</Run> - <LBRange Start="3" End="166"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203454</Run> - <LBRange Start="89" End="156"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203432</Run> - <LBRange Start="117" End="469"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203353</Run> - <LBRange Start="260" End="743"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203336</Run> - <LBRange Start="178" End="789"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203335</Run> - <LBRange Start="36" End="351"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203277</Run> - <LBRange Start="150" End="479"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203258</Run> - <LBRange Start="51" End="843"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203256</Run> - <LBRange Start="242" End="265"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203228</Run> - <LBRange Start="111" End="220"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203195</Run> - <LBRange Start="78" End="357"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203191</Run> - <LBRange Start="81" End="111"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203169</Run> - <LBRange Start="647" End="724"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">203027</Run> - <LBRange Start="425" End="1070"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">202991</Run> - <LBRange Start="273" End="558"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">202987</Run> - <LBRange Start="122" End="198"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">202965</Run> - <LBRange Start="122" End="337"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">202798</Run> - <LBRange Start="446" End="920"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">202740</Run> - <LBRange Start="73" End="123"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">202712</Run> - <LBRange Start="297" End="546"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">202668</Run> - <LBRange Start="33" End="479"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">202660</Run> - <LBRange Start="322" End="459"/> - <LBRange Start="660" End="745"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201556</Run> - <LBRange Start="421" End="1041"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201555</Run> - <LBRange Start="53" End="198"/> - <LBRange Start="202" End="205"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201494</Run> - <LBRange Start="6" End="220"/> - <LBRange Start="241" End="264"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201489</Run> - <LBRange Start="6" End="374"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201289</Run> - <LBRange Start="96" End="566"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201280</Run> - <LBRange Start="143" End="200"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201269</Run> - <LBRange Start="95" End="671"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201257</Run> - <LBRange Start="362" End="1014"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201191</Run> - <LBRange Start="57" End="411"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201190</Run> - <LBRange Start="84" End="498"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201138</Run> - <LBRange Start="279" End="769"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201120</Run> - <LBRange Start="292" End="458"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201113</Run> - <LBRange Start="208" End="561"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201052</Run> - <LBRange Start="142" End="315"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">201006</Run> - <LBRange Start="80" End="639"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">200987</Run> - <LBRange Start="42" End="686"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">200982</Run> - <LBRange Start="81" End="153"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">200967</Run> - <LBRange Start="197" End="197"/> - <LBRange Start="206" End="261"/> - <LBRange Start="265" End="330"/> - <LBRange Start="334" End="395"/> - <LBRange Start="398" End="456"/> - <LBRange Start="604" End="604"/> - <LBRange Start="609" End="668"/> - <LBRange Start="670" End="729"/> - <LBRange Start="734" End="796"/> - <LBRange Start="798" End="858"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">200965</Run> - <LBRange Start="26" End="116"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">200926</Run> - <LBRange Start="324" End="546"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">200913</Run> - <LBRange Start="104" End="280"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">200863</Run> - <LBRange Start="54" End="332"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">200842</Run> - <LBRange Start="3" End="24"/> - <LBRange Start="27" End="85"/> - </LumiBlockCollection> - <LumiBlockCollection> - <Run PrescaleRD0="8" PrescaleRD1="8">200841</Run> - <LBRange Start="134" End="135"/> - <LBRange Start="171" End="172"/> - <LBRange Start="174" End="174"/> - <LBRange Start="176" End="177"/> - <LBRange Start="179" End="181"/> - </LumiBlockCollection> - </NamedLumiRange> -</LumiRangeCollection> diff --git a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/lbn_anal_map_splitter.py b/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/lbn_anal_map_splitter.py index 550c087808b4c887c409e0ab53f2a7152bdea11a..578503cafc4151d6d3fdb35229376856416b8098 100644 --- a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/lbn_anal_map_splitter.py +++ b/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/lbn_anal_map_splitter.py @@ -4,22 +4,23 @@ import sys #get the list of special run,lbn with more than 1 file special_run_lbn = {} -with open("runs_lbn_files_lb.txt") as f: +with open("runs_lbn_files.txt") as f: for line in f: #print line - linfo=line.split("\t") - #print linfo[1] - finfo=linfo[1].split(".") - run=finfo[1] - lbn=finfo[5].replace("_lb","") - #print run+"_"+lbn + if not line.startswith("root"): continue + filenm=line.rstrip() + print filenm + finfo=filenm.split(".") + run=finfo[3] + lbn=finfo[7].replace("_lb","") + print run+"_"+lbn if run+"_"+lbn in special_run_lbn: - if linfo[1] in special_run_lbn[run+"_"+lbn]: - print linfo[1], "already in map for ",run,lbn + if filenm in special_run_lbn[run+"_"+lbn]: + print filenm, "already in map for ",run,lbn else: - special_run_lbn[run+"_"+lbn].append(linfo[1]) + special_run_lbn[run+"_"+lbn].append(filenm) else: - special_run_lbn[run+"_"+lbn]=[linfo[1]] + special_run_lbn[run+"_"+lbn]=[filenm] for key in special_run_lbn.keys(): #if len(special_run_lbn[key])>1: print special_run_lbn[key] pass @@ -29,11 +30,11 @@ with open("runs_lbn_files_lb.txt") as f: maxstream = int(sys.argv[1]) print "maxstream=",maxstream for s in range(0,maxstream+1): #50 - with open("/home/ahaas/nfs3/zerobias_skim/lbn_anal_map_"+str(s)+"wanted.txt") as f: + with open("lbn_anal_map_stream"+str(s)+".txt") as f: tot=0 nfile=1 - of=open("/home/ahaas/nfs3/zerobias_skim/lbn_anal_map_"+str(s)+"wanted_"+str(nfile)+".txt","w") - of2=open("/home/ahaas/nfs3/zerobias_skim/filelist_"+str(s)+"wanted_"+str(nfile)+".txt","w") + of=open("output_stream"+str(s)+"/lbn_anal_map_"+str(nfile)+".txt","w") + of2=open("output_stream"+str(s)+"/filelist_"+str(nfile)+".txt","w") for line in f: linfo=line.split(" ") #print line @@ -58,8 +59,8 @@ for s in range(0,maxstream+1): #50 #print run, lbn of.close(); of2.close(); nfile+=1 - of=open("/home/ahaas/nfs3/zerobias_skim/lbn_anal_map_"+str(s)+"wanted_"+str(nfile)+".txt","w") - of2=open("/home/ahaas/nfs3/zerobias_skim/filelist_"+str(s)+"wanted_"+str(nfile)+".txt","w") + of=open("output_stream"+str(s)+"/lbn_anal_map_"+str(nfile)+".txt","w") + of2=open("output_stream"+str(s)+"/filelist_"+str(nfile)+".txt","w") tot=extra #the number left over from the last set rline=line.replace(linfo[16]+" wanted",str(extra)+" wanted") @@ -81,8 +82,8 @@ for s in range(0,maxstream+1): #50 print run,lbn," not in files map!" of.close(); of2.close(); nfile+=1 - of=open("/home/ahaas/nfs3/zerobias_skim/lbn_anal_map_"+str(s)+"wanted_"+str(nfile)+".txt","w") - of2=open("/home/ahaas/nfs3/zerobias_skim/filelist_"+str(s)+"wanted_"+str(nfile)+".txt","w") + of=open("output_stream"+str(s)+"/lbn_anal_map_"+str(nfile)+".txt","w") + of2=open("output_stream"+str(s)+"/filelist_"+str(nfile)+".txt","w") tot=0 else: of.write(line) diff --git a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/query.sh b/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/query.sh index 73af4092fa0c18c75ba4ad5c5f72f1a479936387..4be39343aa4c3eee251880f2f1de8f5263bf1f09 100755 --- a/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/query.sh +++ b/Event/EventOverlay/EventOverlayJobTransforms/scripts/0biasselection/query.sh @@ -1,86 +1,80 @@ #!/bin/bash -#localSetupDQ2Client -#asetup 17.2.12.7,here +#setupATLAS +#lsetup rucio #voms.sh #use GRL, from: http://atlasdqm.web.cern.ch/atlasdqm/grlgen/All_Good/?C=M;O=D -#dq="data12_8TeV.periodAllYear_DetStatus-v61-pro14-02_DQDefects-00-01-00_PHYS_StandardGRL_All_Good.xml" -dq="data15_13TeV.periodAllYear_DetStatus-v64-pro19_DQDefects-00-01-02_PHYS_StandardGRL_All_Good.xml" -#dq="data15_13TeV.periodAllYear_DetStatus-v65-pro19-01_DQDefects-00-01-02_PHYS_StandardGRL_All_Good.xml" -wget -nc http://atlasdqm.web.cern.ch/atlasdqm/grlgen/All_Good/$dq +#dq="data15_13TeV.periodAllYear_DetStatus-v73-pro19-08_DQDefects-00-01-02_PHYS_StandardGRL_All_Good.xml" +#wget -nc http://atlasdqm.web.cern.ch/atlasdqm/grlgen/All_Good/$dq -#Can just go here: https://atlas-lumicalc.cern.ch/ , then copy into file "lbn" +#Can just go here: https://atlas-lumicalc.cern.ch/ ... note use --online lumi tag for HI, unless a better one is available +#then wget "Raw iLimuiCalc.exe output", and "mv output.txt lbn_2015; ln -s lbn_2015 lbn" #rm -f lbn # if you want to redo this if [ ! -f "lbn" ]; then -#iLumiCalc.exe -t L1_ZB -V -x $dq --lumitag=OflLumi-8TeV-003 --livetrigger=L1_EM30 > lbn -iLumiCalc.exe -t L1_ZB -V -x $dq --livetrigger=L1_EM30 > lbn +iLumiCalc.exe -t L1_ZB -V -x $dq --livetrigger=L1_EM12 > lbn fi -grep ": Total" lbn +grep ": Total" lbn ; echo #get info on number of events and files per lumi block in each run #rm -f lbnevents* # if you want to redo this -#rm -f runs_lbn_files.txt runs_lbn_files_lb.txt # if you want to redo this +#rm -f runs_lbn_files.txt # if you want to redo this +#sstream="physics_ZeroBias" #pp +sstream="physics_MinBiasOverlay" #HI +rm -f runs_temp.txt +for runn in `grep "subchild node value" lbn|cut -f 3 -d : |sed 's/[\x01-\x1F\x7F]//g'|sed 's%\[0m%%g'|sed 's%\[34m%%g'|sed ':a;N;$!ba;s/\n/ /g' `; do + if [ ! -f lbnevents_${runn}.txt ]; then + echo "Getting Nevent info for run $runn "; + python GetNEventsLB.py -r $runn -s $sstream |grep "... Run" > lbnevents_${runn}.txt;# Only works on lxplus now! + fi + if [ ! -f runs_lbn_files.txt ]; then + echo "Getting file info for run $runn " + #rucio list-file-replicas data15_13TeV.00${runn}.${sstream}.merge.RAW | grep "CERN-PROD_DATADISK" | cut -d '|' -f 6|sed "s% CERN-PROD_DATADISK: gsiftp://eosatlassftp.cern.ch:2811/%root://eosatlas.cern.ch//%" | sed -r 's/\s+//g' | grep "_lb" >> runs_temp.txt + rucio list-file-replicas data15_hi.00${runn}.${sstream}.daq.RAW | grep "CERN-PROD_DATADISK" | cut -d '|' -f 6|sed "s% CERN-PROD_DATADISK: gsiftp://eosatlassftp.cern.ch:2811/%root://eosatlas.cern.ch//%" | sed -r 's/\s+//g' | grep "_lb" >> runs_temp.txt + fi +done #loop over all runs +if [ ! -f runs_lbn_files.txt ]; then mv runs_temp.txt runs_lbn_files.txt ; fi -#sstream="physics_ZeroBiasOverlay" -sstream="physics_ZeroBias" -#for runn in `echo "200842,200863,200913,200926,200965,200967,200982,200987,201006,201052,201113,201120,201138,201190,201191,201257,201269,201280,201289,201489,201494,201555,201556,202660,202668,202712,202740,202798,202965,202991,203027,203169,203191,203195,203228,203256,203258,203277,203335,203336,203353,203432,203454,203456,203523,203524,203602,203605,203636,203680,203719,203739,203745,203760,203779,203792,203875,203876,203934,204025,204026,204071,204073,204134,204153,204158,204240,204265,204416,204442,204474,204564,204633,204668,204726,204763,204769,204772,204796,204853,204857,204910,204932,204954,204955,204976,205010,205016,205017,205055,205071,205112,205113,206368,206369,206409,206497,206573,206614,206955,206962,206971,207044,207046,207221,207262,207304,207306,207332,207397,207447,207490,207528,207531,207532,207582,207589,207620,207664,207696,207749,207772,207800,207809,207845,207864,207865,207931,207934,207975,207982,208123,208126,208179,208184,208189,208258,208261,208354,208485,208631,208642,208662,208705,208717,208720,208780,208781,208811,208870,208930,208931,208970,208982,209024,209025,209074,209084,209109,209161,209183,209214,209254,209265,209269,209353,209381,209550,209580,209608,209628,209629,209736,209776,209787,209812,209864,209866,209899,209980,209995,210302,210308,211620,211670,211697,211772,211787,211867,211902,211937,212034,212103,212142,212144,212172,212199,212272,212619,212663,212687,212721,212742,212809,212815,212858,212967,212993,213039,213079,213092,213130,213155,213157,213204,213250,213359,213431,213479,213486,213539,213627,213640,213684,213695,213702,213754,213796,213816,213819,213900,213951,213964,213968,214021,214086,214160,214176,214216,214388,214390,214494,214523,214544,214553,214618,214651,214680,214714,214721,214758,214777,215027,215061,215063,215091,215414,215433,215456,215464,215473,215541,215571,215589,215643"|sed "s%,% %g"` ; do -for runn in `echo "276262,276329,276336,276416,276511,276689,276731,276778,276790,276952,276954" |sed "s%,% %g"` ; do - if [ ! -f lbnevents_${runn}.txt ]; then python GetNEventsLB.py -r $runn -s $sstream |grep "... Run" > lbnevents_${runn}.txt; fi ## Only works on lxplus now!! - if [ ! -f runs_lbn_files_lb.txt ]; then - echo "Getting event info for run $runn " - #dq2-ls -f -H data12_8TeV.00${runn}.physics_ZeroBiasOverlay.merge.RAW >> runs_lbn_files.txt - dq2-ls -f -H data15_13TeV.00${runn}.physics_ZeroBias.merge.RAW >> runs_lbn_files.txt -fi -done -grep "_lb" runs_lbn_files.txt > runs_lbn_files_lb.txt - -#make lbn_anal_map.txt #rm -f lbn_anal_map.txt # if you want to redo this -if [ ! -f lbn_anal_map.txt ]; then root -l -b -q run_lbn_analyze.C > ~/nfs3/zerobias_skim/log_lbn_analyze.txt ; fi +if [ ! -f lbn_anal_map.txt ]; then root -l -b -q run_lbn_analyze.C > log_lbn_analyze.txt ; fi echo -n "Total events in dataset before GRL: " cat lbnevents_*.txt |tr -s ' ' | cut -d ' ' -f 8 |awk '{total = total + $1}END{print total}' echo -n "Total events in dataset from log_lbn_analyze.txt: " -grep "stream 0" ~/nfs3/zerobias_skim/log_lbn_analyze.txt|cut -d ' ' -f 6 |awk '{total = total + $1}END{print total}' +grep "stream 0" log_lbn_analyze.txt|cut -d ' ' -f 6 |awk '{total = total + $1}END{print total}' echo -n "Total events in dataset from lbn_anal_map: " grep "stream 1," lbn_anal_map.txt |cut -d ' ' -f 8 |awk '{total = total + $1}END{print total}' echo -n "Selected events per stream: " grep "stream 8," lbn_anal_map.txt |cut -d ' ' -f 17 |awk '{total = total + $1}END{print total}' -#rm ~/nfs3/zerobias_skim/filelist_* -#rm ~/nfs3/zerobias_skim/lbn_anal_map_* - -maxstream=1 -#do this for all streams -for s in $(seq 0 $maxstream); do grep "stream ${s}," lbn_anal_map.txt | grep -v "0 want" > ~/nfs3/zerobias_skim/lbn_anal_map_${s}wanted.txt ; done -#rm lbn_anal_map.txt #could remove this now +maxstream=1 #up to 49 +#split into all the desired streams +for s in $(seq 0 $maxstream); do + grep "stream ${s}," lbn_anal_map.txt | grep -v "and 0 wanted" > lbn_anal_map_stream${s}.txt + mkdir -p output_stream${s} +done +#rm lbn_anal_map.txt #remove this now -#split into jobs for each stream -python lbn_anal_map_splitter.py $maxstream #to redo this -rm ~/nfs3/zerobias_skim/*501.txt +#split into jobs for each stream and make filelist and lbn_anal_map files +rm -f output_stream*/filelist_*.txt output_stream*/lbn_anal_map_*.txt +python lbn_anal_map_splitter.py $maxstream > log_lbn_anal_map_splitter.txt +rm output_stream*/*501.txt -for s in $(seq 0 $maxstream); do +for s in $(seq 1 $maxstream); do echo "renaming stream $s to 0" - for f in {1..500}; do sed -i -e "s%stream $s,%stream 0,%g" ~/nfs3/zerobias_skim/lbn_anal_map_${s}wanted_${f}.txt; done -done - -cd ~/nfs3/zerobias_skim/ -it=2015_1 -for s in $(seq 0 $maxstream); do - tar cfz ${s}wanted${it}.tar.gz *_${s}wanted_*.txt + for f in {1..500}; do sed -i -e "s%stream $s,%stream 0,%g" output_stream${s}/lbn_anal_map_${f}.txt; done done -cd - -#send to afs -scp ~/nfs3/zerobias_skim/*wanted${it}.tar.gz ahaas@lxplus.cern.ch:public/overlay/lists/ +it=2015_HI_1 #just a name to tag this set of files +for s in $(seq 0 $maxstream); do cd output_stream${s}; tar cfz stream${s}_${it}.tar.gz *.txt; cd ..; done +mv -v output_stream*/stream*_${it}.tar.gz ~/public/overlay/lists/ #to check the events in each map: for s in $(seq 0 $maxstream); do echo -n "files with 100 events in stream $s :" - for f in {1..500}; do cat ~/nfs3/zerobias_skim/lbn_anal_map_${s}wanted_$f.txt | cut -d ' ' -f 17 |awk '{total = total + $1}END{print total}'; done |grep -c 100 + for f in {1..500}; do cat output_stream${s}/lbn_anal_map_$f.txt | cut -d ' ' -f 17 |awk '{total = total + $1}END{print total}'; done |grep -c 100 done #to show number of files in each filelist: -for f in ~/nfs3/zerobias_skim/filelist_0wanted_*.txt; do echo -n `grep -o "data" $f | wc -l`; echo " : $f"; done |sort -n +echo ; echo "files in each filelist (in stream 0)..." ; for f in output_stream0/filelist_*.txt; do echo -n `grep -o $sstream $f | wc -l`; echo " : $f"; done |sort -n diff --git a/Event/EventOverlay/EventOverlayJobTransforms/scripts/BSOverlayFilter_tf.py b/Event/EventOverlay/EventOverlayJobTransforms/scripts/BSOverlayFilter_tf.py index e2f09df8fdd77db4fe8dab4e64cf234cc8690271..94f0653a76713f8869e73bfb49ddae4781777f00 100755 --- a/Event/EventOverlay/EventOverlayJobTransforms/scripts/BSOverlayFilter_tf.py +++ b/Event/EventOverlay/EventOverlayJobTransforms/scripts/BSOverlayFilter_tf.py @@ -3,7 +3,7 @@ # Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration ## Transform for preparation of BS overlay -# @version $Id: BSOverlayFilter_tf.py 695685 2015-09-18 16:46:57Z ahaas $ +# @version $Id: BSOverlayFilter_tf.py 733403 2016-04-01 18:38:53Z ahaas $ import os.path import sys @@ -39,8 +39,10 @@ def main(): def getTransform(): executorSet = set() - from EventOverlayJobTransforms.overlayTransformUtils import addOverlayBSFilterSubstep, addOverlayBSFilterArguments + from EventOverlayJobTransforms.overlayTransformUtils import addOverlayBSFilterSubstep, addOverlayBSFilterArguments, addOverlayBSTrigFilterSubstep, addOverlayHITARMakerSubstep + addOverlayBSTrigFilterSubstep(executorSet) addOverlayBSFilterSubstep(executorSet) + addOverlayHITARMakerSubstep(executorSet) trf = transform(executor = executorSet, description = 'Filter BS data based on trigger bit') addAthenaArguments(trf.parser) addOverlayBSFilterArguments(trf.parser) diff --git a/Event/EventOverlay/EventOverlayJobTransforms/scripts/OverlayChain_tf.py b/Event/EventOverlay/EventOverlayJobTransforms/scripts/OverlayChain_tf.py index 4ef103e5f5a7e7351316cc5b2d539859f4a45f9c..c9c3fd67d6fd5a1138480603b8c782de48c020a2 100755 --- a/Event/EventOverlay/EventOverlayJobTransforms/scripts/OverlayChain_tf.py +++ b/Event/EventOverlay/EventOverlayJobTransforms/scripts/OverlayChain_tf.py @@ -22,7 +22,7 @@ from PyJobTransforms.trfDecorators import stdTrfExceptionHandler, sigUsrStackTra import PyJobTransforms.trfArgClasses as trfArgClasses # Prodsys hack... -ListOfDefaultPositionalKeys=['--AFPOn', '--ALFAOn', '--AMIConfig', '--AMITag', '--AddCaloDigi', '--CosmicFilterVolume', '--CosmicFilterVolume2', '--CosmicPtSlice', '--DBRelease', '--DataRunNumber', '--FwdRegionOn', '--LucidOn', '--ReadByteStream', '--ZDCOn', '--argJSON', '--asetup', '--athena', '--athenaMPMergeTargetSize', '--athenaopts', '--attempt', '--beamType', '--checkEventCount', '--command', '--conditionsTag', '--digiRndmSvc', '--digiSeedOffset1', '--digiSeedOffset2', '--digiSteeringConf', '--doAllNoise', '--dumpJSON', '--dumpPickle', '--enableLooperKiller', '--env', '--eventAcceptanceEfficiency', '--outputTXT_EVENTIDFile','--inputTXT_EVENTIDFile', '--execOnly', '--fSampltag', '--fileValidation', '--firstEvent', '--geometryVersion', '--ignoreErrors', '--ignoreFiles', '--ignorePatterns', '--imf', '--inputBS_SKIMFile', '--inputEVNTFile', '--inputEVNT_CAVERNFile', '--inputEVNT_COSMICSFile', '--inputFileValidation', '--inputHITSFile', '--inputZeroBiasBSFile', '--jobNumber', '--jobid', '--lumiBlockMapFile', '--maxEvents', '--maxFilesPerSubjob', '--orphanKiller', '--outputBS_SKIMFile', '--outputEVNT_CAVERNTRFile', '--outputEVNT_COSMICSTRFile', '--outputFileValidation', '--outputHITSFile', '--outputRDOFile', '--outputRDO_FILTFile', '--outputRDO_SGNLFile', '--overlayConfigFile', '--parallelFileValidation', '--physicsList', '--postExec', '--postInclude', '--preExec', '--preInclude', '--randomSeed', '--reportName', '--reportType', '--runNumber', '--samplingFractionDbTag', '--showGraph', '--showPath', '--showSteps', '--simulator', '--skipEvents', '--skipFileValidation', '--skipInputFileValidation', '--skipOutputFileValidation', '--steering', '--taskid', '--tcmalloc', '--tmpRDO', '--tmpRDO_FILT', '--triggerBit', '--truthStrategy', '--useISF', '--valgrind', '--valgrindbasicopts', '--valgrindextraopts'] +ListOfDefaultPositionalKeys=['--AFPOn', '--ALFAOn', '--AMIConfig', '--AMITag', '--AddCaloDigi', '--CosmicFilterVolume', '--CosmicFilterVolume2', '--CosmicPtSlice', '--DBRelease', '--DataRunNumber', '--FwdRegionOn', '--LucidOn', '--ReadByteStream', '--ZDCOn', '--argJSON', '--asetup', '--athena', '--athenaMPMergeTargetSize', '--athenaopts', '--attempt', '--beamType', '--checkEventCount', '--command', '--conditionsTag', '--digiRndmSvc', '--digiSeedOffset1', '--digiSeedOffset2', '--digiSteeringConf', '--doAllNoise', '--dumpJSON', '--dumpPickle', '--enableLooperKiller', '--env', '--eventAcceptanceEfficiency', '--outputTXT_EVENTIDFile','--inputTXT_EVENTIDFile', '--execOnly', '--fSampltag', '--fileValidation', '--firstEvent', '--geometryVersion', '--ignoreErrors', '--ignoreFiles', '--ignorePatterns', '--imf', '--inputBS_SKIMFile', '--inputEVNTFile', '--inputEVNT_CAVERNFile', '--inputEVNT_COSMICSFile', '--inputFileValidation', '--inputHITSFile', '--inputZeroBiasBSFile', '--jobNumber', '--jobid', '--lumiBlockMapFile', '--maxEvents', '--maxFilesPerSubjob', '--orphanKiller', '--outputBS_SKIMFile', '--outputEVNT_CAVERNTRFile', '--outputEVNT_COSMICSTRFile', '--outputFileValidation', '--outputHITSFile', '--outputRDOFile', '--outputRDO_FILTFile', '--outputRDO_SGNLFile', '--overlayConfigFile', '--parallelFileValidation', '--physicsList', '--postExec', '--postInclude', '--preExec', '--preInclude', '--randomSeed', '--reportName', '--reportType', '--runNumber', '--samplingFractionDbTag', '--showGraph', '--showPath', '--showSteps', '--simulator', '--skipEvents', '--skipFileValidation', '--skipInputFileValidation', '--skipOutputFileValidation', '--steering', '--taskid', '--tcmalloc', '--tmpRDO', '--tmpRDO_FILT', '--triggerBit', '--truthStrategy', '--useISF', '--valgrind', '--valgrindbasicopts', '--valgrindextraopts', '--WriteRDOFileMetaData'] @stdTrfExceptionHandler @sigUsrStackTrace diff --git a/Event/EventOverlay/EventOverlayJobTransforms/share/InnerDetectorOverlay_jobOptions.py b/Event/EventOverlay/EventOverlayJobTransforms/share/InnerDetectorOverlay_jobOptions.py index 2029389b337fecd63a7be3f89d4715760a1111a8..b2e4f5c3a1a4fa5feeaf6031c7046744160eb5f9 100644 --- a/Event/EventOverlay/EventOverlayJobTransforms/share/InnerDetectorOverlay_jobOptions.py +++ b/Event/EventOverlay/EventOverlayJobTransforms/share/InnerDetectorOverlay_jobOptions.py @@ -30,7 +30,7 @@ if DetFlags.overlay.pixel_on() or DetFlags.overlay.SCT_on() or DetFlags.overlay. indetovl.do_Pixel = True if readBS and isRealData: job.InDetPixelRawDataProvider.EvtStore = "OriginalEvent_SG" - ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [ "PixelRDO_Container/PixelRDOs" ] + #ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [ "PixelRDO_Container/PixelRDOs" ] #ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [ "Trk::PixelClusterContainer/PixelOnlineClusters" ] else: indetovl.do_Pixel = False @@ -61,7 +61,7 @@ if DetFlags.overlay.pixel_on() or DetFlags.overlay.SCT_on() or DetFlags.overlay. indetovl.do_SCT = True if readBS and isRealData: job.InDetSCTRawDataProvider.EvtStore = "OriginalEvent_SG" - ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [ "SCT_RDO_Container/SCT_RDOs" ] + #ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [ "SCT_RDO_Container/SCT_RDOs" ] #ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [ "Trk::SCT_ClusterContainer/SCT_OnlineClusters" ] else: indetovl.do_SCT = False @@ -77,7 +77,7 @@ if DetFlags.overlay.pixel_on() or DetFlags.overlay.SCT_on() or DetFlags.overlay. indetovl.do_TRT = True if readBS and isRealData: job.InDetTRTRawDataProvider.EvtStore = "OriginalEvent_SG" - ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [ "TRT_RDO_Container/TRT_RDOs" ] + #ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [ "TRT_RDO_Container/TRT_RDOs" ] from TRT_ConditionsServices.TRT_ConditionsServicesConf import TRT_CalDbSvc InDetTRTCalDbSvc = TRT_CalDbSvc() diff --git a/Event/EventOverlay/EventOverlayJobTransforms/share/Level1Overlay_jobOptions.py b/Event/EventOverlay/EventOverlayJobTransforms/share/Level1Overlay_jobOptions.py index cb1b2ef7fa5eff57c5a8f7e38d2f542b10730d75..5697d8f93a73aa2fae90d8b386543ac5c51dd886 100644 --- a/Event/EventOverlay/EventOverlayJobTransforms/share/Level1Overlay_jobOptions.py +++ b/Event/EventOverlay/EventOverlayJobTransforms/share/Level1Overlay_jobOptions.py @@ -114,7 +114,7 @@ if DetFlags.overlay.LVL1_on(): #------------------------------------------------------- if DetFlags.simulateLVL1.Calo_on(): if DetFlags.simulateLVL1.LAr_on() and DetFlags.simulateLVL1.Tile_on(): - protectedInclude( "TrigT1CaloSim/TrigT1CaloSimJobOptions_TTL1.py" ) + protectedInclude( "TrigT1CaloSim/TrigT1CaloSimJobOptions_TTL1_NoCalib.py" ) #switched to NoCalib job.TriggerTowerMaker.DoOverlay = True job.TriggerTowerMaker.OverlayPedestal = 40. else: diff --git a/Event/EventOverlay/EventOverlayJobTransforms/share/MuonOverlay_jobOptions.py b/Event/EventOverlay/EventOverlayJobTransforms/share/MuonOverlay_jobOptions.py index 3f9b222c70b53dc1b7f7f3a99ff1db664c198b7b..9af52da34bf4bd19383aa133b4c1e56122eece8d 100644 --- a/Event/EventOverlay/EventOverlayJobTransforms/share/MuonOverlay_jobOptions.py +++ b/Event/EventOverlay/EventOverlayJobTransforms/share/MuonOverlay_jobOptions.py @@ -35,7 +35,7 @@ if DetFlags.overlay.MDT_on() or DetFlags.overlay.CSC_on() or DetFlags.overlay.RP digitizationFlags.doMuonNoise=False #FIXME should not be setting jobproperties at this point in the configuration. if readBS: - include("MuonRecExample/MuonReadBS_jobOptions.py") + include("MuonCnvExample/MuonReadBS_jobOptions.py") if DetFlags.overlay.CSC_on(): if readBS: diff --git a/Event/EventOverlay/EventOverlayJobTransforms/share/OverlayOutputItemList_jobOptions.py b/Event/EventOverlay/EventOverlayJobTransforms/share/OverlayOutputItemList_jobOptions.py index 707b3c89d6be2e4bbe895c19930cfc07fcf9646c..cc48257919af8210e888fa498dfa9cde35502cf7 100644 --- a/Event/EventOverlay/EventOverlayJobTransforms/share/OverlayOutputItemList_jobOptions.py +++ b/Event/EventOverlay/EventOverlayJobTransforms/share/OverlayOutputItemList_jobOptions.py @@ -39,14 +39,19 @@ if DetFlags.overlay.Truth_on(): if DetFlags.overlay.pixel_on(): outStream.ItemList += ["PixelRDO_Container#*"] + outStream.ItemList += ["InDetBSErrContainer#*"] if DetFlags.overlay.SCT_on(): outStream.ItemList += ["SCT_RDO_Container#*"] + outStream.ItemList += ["SCT_RDO_Container#*"] if DetFlags.overlay.TRT_on(): outStream.ItemList += ["TRT_RDO_Container#*"] + outStream.ItemList += ["TRT_BSIdErrContainer#*"] + outStream.ItemList += ["TRT_BSErrContainer#*"] if DetFlags.overlay.LAr_on(): outStream.ItemList+=["LArRawChannelContainer#*"] outStream.ItemList+=["LArDigitContainer#LArDigitContainer_MC_Thinned"] + outStream.ItemList+=["LArFebErrorSummary#*"] if DetFlags.overlay.Tile_on(): if isRealData: outStream.ItemList += [ "TileDigitsContainer#*" ] diff --git a/Event/EventOverlay/EventOverlayJobTransforms/share/Rt_override_CONDBR2-BLKPA-2015-12.py b/Event/EventOverlay/EventOverlayJobTransforms/share/Rt_override_CONDBR2-BLKPA-2015-12.py index 48280e14521635ef257c968db904aef66e2180ff..a0f685cf728f33ed0be9131f45dcca8f84f76413 100644 --- a/Event/EventOverlay/EventOverlayJobTransforms/share/Rt_override_CONDBR2-BLKPA-2015-12.py +++ b/Event/EventOverlay/EventOverlayJobTransforms/share/Rt_override_CONDBR2-BLKPA-2015-12.py @@ -44,8 +44,8 @@ print "RT OVERRIDE, for CONDBR2-BLKPA-2015-12" conddb.blockFolder("/MDT/T0BLOB") conddb.addFolderWithTag("MDT_OFL","/MDT/T0BLOB","MDTT0-RUN2-UPD4-15",force=True,forceData=True) -alignData=True -if alignData: +if not "EOJT_alignMC" in globals(): + print "EOJT_alignMC not found in globals(), so aligning ID to data conditions" conddb.blockFolder("/TRT/Align") conddb.addFolderWithTag("TRT_OFL","/TRT/Align","TRTAlign-RUN2-BLK-UPD4-10",force=True,forceData=True) conddb.blockFolder("/LAR/Align") @@ -53,6 +53,7 @@ if alignData: conddb.blockFolder("/Indet/Align") conddb.addFolderWithTag("INDET_OFL","/Indet/Align","InDetAlign-RUN2-BLK-UPD4-13",force=True,forceData=True) else: + print "EOJT_alignMC found in globals(), so aligning ID to MC conditions" conddb.blockFolder("/TRT/Align") conddb.addFolderWithTag("TRT_OFL","/TRT/Align","TRTAlign_Nominal2",force=True,forceMC=True) conddb.blockFolder("/LAR/Align") diff --git a/Event/EventOverlay/EventOverlayJobTransforms/share/UseOracle.py b/Event/EventOverlay/EventOverlayJobTransforms/share/UseOracle.py deleted file mode 100644 index e681e324c8d79bc40321dd3819dcf825fc0d5172..0000000000000000000000000000000000000000 --- a/Event/EventOverlay/EventOverlayJobTransforms/share/UseOracle.py +++ /dev/null @@ -1,7 +0,0 @@ -# setup DBReplicaSvc to choose closest Oracle replica, configurables style -from AthenaCommon.AppMgr import ServiceMgr -from PoolSvc.PoolSvcConf import PoolSvc -ServiceMgr+=PoolSvc(SortReplicas=True) -from DBReplicaSvc.DBReplicaSvcConf import DBReplicaSvc -ServiceMgr+=DBReplicaSvc(UseCOOLSQLite=False) - diff --git a/Event/EventOverlay/EventOverlayJobTransforms/share/g4runnumber2.py b/Event/EventOverlay/EventOverlayJobTransforms/share/g4runnumber2.py index de8116f968b6ec62f00e11cdffff6e63de5ed61a..6aef66f38c9f1de82626d8e776efd46efff91215 100644 --- a/Event/EventOverlay/EventOverlayJobTransforms/share/g4runnumber2.py +++ b/Event/EventOverlay/EventOverlayJobTransforms/share/g4runnumber2.py @@ -1,6 +1,7 @@ #set the max number of events -fname = "events.txt" +from OverlayCommonAlgs.OverlayFlags import overlayFlags +fname = overlayFlags.EventIDTextFile() num_lines = 0 with open(fname, 'r') as f: for line in f: @@ -16,5 +17,5 @@ athenaCommonFlags.EvtMax.set_Value_and_Lock(num_lines) #new MC12 way, should be a preInclude from G4AtlasApps.SimFlags import SimFlags SimFlags.load_atlas_flags() -SimFlags.VertexOverrideEventFile.set_Value_and_Lock("events.txt") +SimFlags.VertexOverrideEventFile.set_Value_and_Lock(fname) diff --git a/Event/EventOverlay/EventOverlayJobTransforms/share/muAlign_reco.py b/Event/EventOverlay/EventOverlayJobTransforms/share/muAlign_reco.py index 8955e213423931959c1de7773b8413d2b7b1ed90..dd46e0532134636ab6dfb9793a118985549e5a26 100644 --- a/Event/EventOverlay/EventOverlayJobTransforms/share/muAlign_reco.py +++ b/Event/EventOverlay/EventOverlayJobTransforms/share/muAlign_reco.py @@ -61,8 +61,12 @@ ServiceMgr += TRTMCCalibDBSvc ToolSvc.InDetTRT_DriftFunctionTool.IsOverlay=True ToolSvc.InDetTRT_DriftFunctionTool.TRTCalDbTool2=TRTMCCalibDBSvc -#to apply Lorentz Angle correction -svcMgr.PixelSiliconConditionsSvc.ForceUseGeoModel=True -svcMgr.PixelSiliconConditionsSvc.CheckGeoModel=True -svcMgr.PixelSiliconConditionsSvc.UseDBForHV=False -svcMgr.PixelLorentzAngleSvc.usePixelDefaults=True +if "EOJT_noLorentz" in globals(): + print "EOJT_noLorentz found in globals(), so not doing Lorentz corrections" + svcMgr.PixelSiliconConditionsSvc.ForceUseGeoModel=True + svcMgr.PixelSiliconConditionsSvc.CheckGeoModel=True + svcMgr.PixelSiliconConditionsSvc.UseDBForHV=False + svcMgr.PixelLorentzAngleSvc.usePixelDefaults=True +else: + print "EOJT_noLorentz not found in globals(), so doing standard Lorentz corrections" + diff --git a/Event/EventOverlay/EventOverlayJobTransforms/share/skeleton.BSOverlayFilter_tf.py b/Event/EventOverlay/EventOverlayJobTransforms/share/skeleton.BSOverlayFilter_tf.py index 964ac4d6ec902aab145f9f197c61004b5297ddfa..7e24f12d65adec7721e5ee07ea6c767bbaa8611d 100644 --- a/Event/EventOverlay/EventOverlayJobTransforms/share/skeleton.BSOverlayFilter_tf.py +++ b/Event/EventOverlay/EventOverlayJobTransforms/share/skeleton.BSOverlayFilter_tf.py @@ -5,15 +5,25 @@ BSFilterLog.info( '****************** STARTING BSFilter *****************' ) BSFilterLog.info( '**** Transformation run arguments' ) BSFilterLog.info( str(runArgs) ) +if hasattr( runArgs, 'trfSubstepName'): + if runArgs.trfSubstepName=="BSFilter" and hasattr(runArgs, 'InputLbnMapFile') and hasattr( runArgs, "triggerBit"): + delattr(runArgs, 'InputLbnMapFile') + BSFilterLog.info( '**** Removed InputLbnMapFile argument: transformation run arguments are now...' ) + BSFilterLog.info( str(runArgs) ) + #--------------------------- theApp.EvtMax = runArgs.maxEvents +svcMgr.MessageSvc.infoLimit=100000 +svcMgr.MessageSvc.defaultLimit=100000 from AthenaCommon.GlobalFlags import globalflags globalflags.InputFormat = "bytestream" globalflags.DataSource = "data" from AthenaCommon.AthenaCommonFlags import athenaCommonFlags -athenaCommonFlags.FilesInput=runArgs.inputZeroBiasBSFile - +if hasattr( runArgs, 'inputZeroBiasBSFile'): + athenaCommonFlags.FilesInput=runArgs.inputZeroBiasBSFile +else: + athenaCommonFlags.FilesInput=runArgs.inputBS_SKIMFile #--------------------------- ## Run performance monitoring (memory logging) @@ -28,8 +38,10 @@ BSFilterLog.info( '**** ByteStreamInputSvc configuration' ) include( "ByteStreamCnvSvc/BSEventStorageEventSelector_jobOptions.py" ) ByteStreamInputSvc = svcMgr.ByteStreamInputSvc # ByteStreamInputSvc.FullFileName = open(runArgs.InputFileMapFile).readline().rstrip().split(',') -ByteStreamInputSvc.FullFileName = runArgs.inputZeroBiasBSFile - +if hasattr( runArgs, 'inputZeroBiasBSFile'): + ByteStreamInputSvc.FullFileName=runArgs.inputZeroBiasBSFile +else: + ByteStreamInputSvc.FullFileName=runArgs.inputBS_SKIMFile print ByteStreamInputSvc # --------------------------- @@ -37,9 +49,11 @@ print ByteStreamInputSvc BSFilterLog.info( '**** ByteStreamOutputSvc configuration' ) from ByteStreamCnvSvc.ByteStreamCnvSvcConf import ByteStreamEventStorageOutputSvc -bsOutputSvc=ByteStreamEventStorageOutputSvc("BSESOutputSvc",OutputDirectory='./',SimpleFileName=runArgs.outputBS_SKIMFile) +if hasattr( runArgs, 'outputBS_SKIMFile'): + bsOutputSvc=ByteStreamEventStorageOutputSvc("BSESOutputSvc",OutputDirectory='./',SimpleFileName=runArgs.outputBS_SKIMFile) +else: + bsOutputSvc=ByteStreamEventStorageOutputSvc("BSESOutputSvc",OutputDirectory='./',SimpleFileName=runArgs.outputBS_TRIGSKIMFile) svcMgr += bsOutputSvc - print bsOutputSvc # --------------------------- @@ -54,42 +68,29 @@ if not hasattr( svcMgr, "ByteStreamAddressProviderSvc" ): from ByteStreamCnvSvcBase.ByteStreamCnvSvcBaseConf import ByteStreamAddressProviderSvc svcMgr += ByteStreamAddressProviderSvc() #svcMgr.ByteStreamAddressProviderSvc.TypeNames += ["ROIB::RoIBResult/RoIBResult", "MuCTPI_RDO/MUCTPI_RDO", "CTP_RDO/CTP_RDO", "MuCTPI_RIO/MUCTPI_RIO", "CTP_RIO/CTP_RIO"] -svcMgr.ByteStreamAddressProviderSvc.TypeNames += ["ROIB::RoIBResult/RoIBResult", "HLT::HLTResult/HLTResult_EF","HLT::HLTResult/HLTResult_L2","HLT::HLTResult/HLTResult_HLT", "CTP_RDO/CTP_RDO", "CTP_RIO/CTP_RIO"] - -# https://twiki.cern.ch/twiki/bin/viewauth/Atlas/TrigDecisionTool -from TriggerJobOpts.TriggerFlags import TriggerFlags -TriggerFlags.configurationSourceList=['ds'] -from TriggerJobOpts.TriggerConfigGetter import TriggerConfigGetter -cfg = TriggerConfigGetter() -# -#from TrigDecisionTool.TrigDecisionToolConf import Trig__TrigDecisionTool -#tdt = Trig__TrigDecisionTool("TrigDecisionTool") -#tdt.UseAODDecision=True -#tdt.TrigDecisionKey = "TrigDecision" -#ToolSvc += tdt -# -#from TrigDecisionMaker.TrigDecisionMakerConfig import WriteTrigDecision -#trigDecWriter = WriteTrigDecision() +svcMgr.ByteStreamAddressProviderSvc.TypeNames += ["ROIB::RoIBResult/RoIBResult", "HLT::HLTResult/HLTResult_EF","HLT::HLTResult/HLTResult_L2","HLT::HLTResult/HLTResult_HLT", + "CTP_RDO/CTP_RDO", "CTP_RIO/CTP_RIO"] # main alg -from OverlayCommonAlgs.OverlayCommonAlgsConf import BSFilter +from OverlayCommonAlgs.OverlayCommonAlgsConf import BSFilter filAlg = BSFilter("BSFilter") topSequence += filAlg if hasattr( runArgs, "triggerBit"): filAlg.TriggerBit = runArgs.triggerBit + #to get HLT info decoded + from TriggerJobOpts.TriggerFlags import TriggerFlags + TriggerFlags.configurationSourceList=['ds'] + from TriggerJobOpts.TriggerConfigGetter import TriggerConfigGetter + cfg = TriggerConfigGetter() else: filAlg.TriggerBit = -1 -#if hasattr(runArgs, 'eventIdFile'): -# filAlg.EventIdFile=runArgs.eventIdFile # The name of the file to write to for EventIdModifierSvc lines -#else: -# filAlg.EventIdFile="" if hasattr(runArgs, 'outputTXT_EVENTIDFile'): filAlg.EventIdFile=runArgs.outputTXT_EVENTIDFile # The name of the file to write to for EventIdModifierSvc lines else: filAlg.EventIdFile="" -if hasattr( runArgs, "inputFilterFile"): +if hasattr( runArgs, "inputFilterFile") and not hasattr( runArgs, 'InputLbnMapFile'): #don't filter if you're skimming using the map file filAlg.filterfile=runArgs.inputFilterFile # The thing made from the TAG files via "root -l -b -q HITAGprinter_run.C" else: filAlg.filterfile = "" @@ -101,6 +102,8 @@ if hasattr( runArgs, 'InputLbnMapFile'): from OverlayCommonAlgs.OverlayCommonAlgsConf import ByteStreamMultipleOutputStreamCopyTool bsCopyTool = ByteStreamMultipleOutputStreamCopyTool("MultipleOutputStreamBSCopyTool") bsCopyTool.lbn_map_file = runArgs.InputLbnMapFile + if hasattr( runArgs, "inputFilterFile"): bsCopyTool.trigfile = runArgs.inputFilterFile + else: bsCopyTool.trigfile = "" bsCopyTool.NoutputSvc = 1 bsCopyTool.ByteStreamOutputSvc0=bsOutputSvc else: @@ -125,8 +128,11 @@ OutputStreamBSCopy.AcceptAlgs =["BSFilter"] print topSequence # --------------------------- -# Post-include - +# Post-include/exec if hasattr(runArgs,"postInclude"): for fragment in runArgs.postInclude: include(fragment) +if hasattr(runArgs, "postExec") and runArgs.postExec != 'NONE': + for cmd in runArgs.postExec: + exec(cmd) + diff --git a/Event/EventOverlay/EventOverlayJobTransforms/share/skeleton.HITARMaker_tf.py b/Event/EventOverlay/EventOverlayJobTransforms/share/skeleton.HITARMaker_tf.py new file mode 100644 index 0000000000000000000000000000000000000000..a197f98ea79e9505dd6ef1fb95c1008abe7976c7 --- /dev/null +++ b/Event/EventOverlay/EventOverlayJobTransforms/share/skeleton.HITARMaker_tf.py @@ -0,0 +1,71 @@ +import os +from AthenaCommon.Logging import logging +logg = logging.getLogger('HITARMaker') +logg.info( '****************** STARTING HITARMaker *****************' ) + +logg.info( '**** Transformation run arguments' ) +logg.info( str(runArgs) ) + + +if hasattr( runArgs, 'outputTAR_CONFIGFile'): + logg.info("making outputTAR_CONFIGFile " + runArgs.outputTAR_CONFIGFile) +else: + logg.error("no outputTAR_CONFIGFile ??") + +if hasattr( runArgs, 'InputLbnMapFile'): + logg.info("using InputLbnMapFile " + runArgs.InputLbnMapFile) +else: + logg.error("no InputLbnMapFile ??") + +if hasattr( runArgs, 'HIfiltervtxdirectory'): + logg.info("using HIfiltervtxdirectory " + runArgs.HIfiltervtxdirectory) +else: + logg.error("no HIfiltervtxdirectory ??") + +#Don't want to run over any events... +theApp.EvtMax = 5 + +#Make the filter/vtx.data.txt files +with open(runArgs.InputLbnMapFile) as f: + for line in f: + if line=='': break + filenm=line.rstrip() + print filenm + finfo=filenm.split(" ") + run=finfo[3].replace(",","") + lbn=finfo[5].replace(",","").zfill(4) #pad 44->0044 + print run+"_"+lbn + + #copy the corresponding filter/vtx_run_lbn.data.txt files from EOS to here + #runArgs.HIfiltervtxdirectory=/eos/atlas/atlascerngroupdisk/phys-hi/DataOverlay2015 + retval=os.system("xrdcp root://eosatlas.cern.ch/"+runArgs.HIfiltervtxdirectory+"/Run"+run+"/HI_"+run+"_"+lbn+"_filter_v1.txt .") + if retval!=0: logg.error("failed to xrdcp root://eosatlas.cern.ch/"+runArgs.HIfiltervtxdirectory+"/Run"+run+"/HI_"+run+"_"+lbn+"_filter_v1.txt .") + else: logg.info("succeeded xrdcp root://eosatlas.cern.ch/"+runArgs.HIfiltervtxdirectory+"/Run"+run+"/HI_"+run+"_"+lbn+"_filter_v1.txt .") + retval=os.system("xrdcp root://eosatlas.cern.ch/"+runArgs.HIfiltervtxdirectory+"/Run"+run+"/HI_"+run+"_"+lbn+"_vtx_v1.txt .") + if retval!=0: logg.error("failed to xrdcp root://eosatlas.cern.ch/"+runArgs.HIfiltervtxdirectory+"/Run"+run+"/HI_"+run+"_"+lbn+"_vtx_v1.txt .") + else: logg.info("succeeded xrdcp root://eosatlas.cern.ch/"+runArgs.HIfiltervtxdirectory+"/Run"+run+"/HI_"+run+"_"+lbn+"_vtx_v1.txt .") + + #then append them to filter/vtx.data.txt + retval=os.system("cat HI_"+run+"_"+lbn+"_filter_v1.txt >> filter.data.txt") + if retval!=0: logg.error("failed to cat HI_"+run+"_"+lbn+"_filter_v1.txt >> filter.data.txt") + else: logg.info("added to filter.data.txt") + retval=os.system("cat HI_"+run+"_"+lbn+"_vtx_v1.txt >> vtx.data.txt") + if retval!=0: logg.error("failed to cat HI_"+run+"_"+lbn+"_vtx_v1.txt >> vtx.data.txt") + else: logg.info("added to vtx.data.txt") + +#Make the TAR file +import tarfile +f=tarfile.open(name=runArgs.outputTAR_CONFIGFile, mode='w:gz') +f.add("filter.data.txt") +f.add("vtx.data.txt") +f.close() + +# --------------------------- +# Post-include/exec +if hasattr(runArgs,"postInclude"): + for fragment in runArgs.postInclude: + include(fragment) +if hasattr(runArgs, "postExec") and runArgs.postExec != 'NONE': + for cmd in runArgs.postExec: + exec(cmd) + diff --git a/Event/EventOverlay/EventOverlayJobTransforms/share/skeleton.OverlayBS_tf.py b/Event/EventOverlay/EventOverlayJobTransforms/share/skeleton.OverlayBS_tf.py index 5960e9d7790cbde0bed9e0dd268ebfd98f479d96..9459384c05a51ada46ac212a1fe23901c66fb86d 100644 --- a/Event/EventOverlay/EventOverlayJobTransforms/share/skeleton.OverlayBS_tf.py +++ b/Event/EventOverlay/EventOverlayJobTransforms/share/skeleton.OverlayBS_tf.py @@ -13,7 +13,7 @@ if hasattr(runArgs, "preExec") and runArgs.preExec != 'NONE': for cmd in runArgs.preExec: exec(cmd) -TileFrameLength=7 +#TileFrameLength=7 from AthenaCommon.AppMgr import ServiceMgr from AthenaCommon.GlobalFlags import globalflags @@ -205,9 +205,11 @@ ServiceMgr += getConfigurable(digitizationFlags.rndmSvc.get_Value())() digitizationFlags.rndmSeedList.addtoService() digitizationFlags.rndmSeedList.printSeeds() -# Write Digitization MetaData -from Digitization.DigitizationWriteMetaData import writeDigitizationMetadata -writeDigitizationMetadata() +# Write Digitization MetaData if WriteRDOFileMetaData is set to true in jO (false by default) +if hasattr(runArgs,"WriteRDOFileMetaData"): + if runArgs.WriteRDOFileMetaData: + from Digitization.DigitizationWriteMetaData import writeDigitizationMetadata + writeDigitizationMetadata() #================================================================ print "overlay_trf: final outStream = ", outStream @@ -230,7 +232,7 @@ else: # conddb.addFolder("","/LAR/ElecCalibOfl/AutoCorrs/AutoCorr"+"<dbConnection>"+InputDBConnection+"</dbConnection>") # conddb.addOverride("/LAR/ElecCalibOfl/AutoCorrs/AutoCorr","") -ServiceMgr.TileInfoLoader.filenameDeltaPhysicsSuffix="of2_Delta_Phys_7Samples" +#ServiceMgr.TileInfoLoader.filenameDeltaPhysicsSuffix="of2_Delta_Phys_7Samples" # Post-include if hasattr(runArgs,"postInclude"):