Move SCTLorentzMonAlg configuration to python area; create separate test...

Move SCTLorentzMonAlg configuration to python area; create separate test script; add dependence on ID geometry

Control/AthenaMonitoring/CMakeLists.txt

@@ -47,6 +47,7 @@ atlas_add_component( AthenaMonitoring
 # Install files from the package:
 atlas_install_python_modules( python/*.py )
 atlas_install_joboptions( share/*.py )
+atlas_install_scripts( share/Run3DQTestingDriver.py )
 
 # Units tests C++:
 file( GLOB CXX_TEST_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/test/*.cxx )

Control/AthenaMonitoring/python/AthMonitorCfgHelper.py

@@ -93,9 +93,13 @@ class AthMonitorCfgHelper(object):
         '''
         from AthenaMonitoring.GenericMonitoringTool import GenericMonitoringTool
         tool = GenericMonitoringTool(name)
-        acc, histsvc = getDQTHistSvc(self.inputFlags)
-        self.resobj.merge(acc)
-        tool.THistSvc = histsvc
+        if self.inputFlags.DQ.isReallyOldStyle:
+            from AthenaCommon.AppMgr import ServiceMgr
+            tool.THistSvc = ServiceMgr.THistSvc
+        else:
+            acc = getDQTHistSvc(self.inputFlags)
+            self.resobj.merge(acc)
+
         tool.HistPath = self.inputFlags.DQ.FileKey + ('/%s' % topPath if topPath else '')
         alg.GMTools += [tool]
         return tool
@@ -204,11 +208,17 @@ def getDQTHistSvc(inputFlags):
     from GaudiSvc.GaudiSvcConf import THistSvc
 
     result = ComponentAccumulator()
+
+    if inputFlags.DQ.isReallyOldStyle:
+        from AthenaCommon.AppMgr import ServiceMgr
+        result.addService(ServiceMgr.THistSvc)
+        return result
+
     histsvc = THistSvc()
     histsvc.Output += ["%s DATAFILE='%s' OPT='RECREATE'" % (inputFlags.DQ.FileKey, 
                                                             inputFlags.Output.HISTFileName)]
     result.addService(histsvc)
-    return result, histsvc
+    return result
 
 def getTriggerTranslatorToolSimple(inputFlags):
     ''' Set up the Trigger Translator Tool; no reason for this to be called

Control/AthenaMonitoring/python/AthenaMonitoringCfg.py

+# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+
+from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator
+
+def AthenaMonitoringCfg(flags):
+    import logging
+    local_logger = logging.getLogger('AthenaMonitoringCfg')
+    result = ComponentAccumulator()
+
+    if flags.DQ.Steering.doSCTMon:
+        local_logger.info('Set up SCT monitoring')
+        from SCT_Monitoring.SCTLorentzMonAlg import SCTLorentzMonAlgConfig
+        result.merge(SCTLorentzMonAlgConfig(flags))
+
+    if flags.DQ.Steering.doTileMon:
+        local_logger.info('Set up Tile monitoring')
+        from TileMonitoring.TileJetMonitorAlgorithm import TileJetMonitoringConfig
+        result.merge(TileJetMonitoringConfig(flags))
+
+    if flags.DQ.Steering.doHLTMon:
+        local_logger.info('Set up HLT monitoring')
+        from TrigHLTMonitoring.TrigHLTMonitoringConfig import TrigHLTMonitoringConfig
+        result.merge(TrigHLTMonitoringConfig(flags))
+
+    return result

Control/AthenaMonitoring/python/DQConfigFlags.py

@@ -4,14 +4,39 @@
 
 from AthenaConfiguration.AthConfigFlags import AthConfigFlags
 
+_steeringFlags = [ 'doGlobalMon', 'LVL1CaloMon', 'doCTPMon', 'doHLTMon',
+                   'doPixelMon', 'doSCTMon', 'doTRTMon', 'doInDetMon',
+                   'doLArMon', 'doTileMon',
+                   'doCaloGlobalMon', 'doMuonMon',
+                   'doLucidMon', 'doAFPMon',
+                   'doHIMon', 'doEgammaMon', 'doJetMon', 'doMissingEtMon',
+                   'doTauMon', 'doJetTagMon' ]
+
+_lowLevelSteeringFlags = [ 'InDet.doGlobalMon', 'InDet.doAlignMon',
+                           'InDet.doPerfMon',  'Muon.doRawMon',
+                           'Muon.doTrackMon', 'Muon.doAlignMon',
+                           'Muon.doSegmentMon',
+                           'Muon.doPhysicsMon', 'Muon.doTrkPhysMon',
+                           'Muon.doCombinedMon'
+                           ]
+
 def createDQConfigFlags():
     acf=AthConfigFlags()
     acf.addFlag('DQ.doMonitoring', True)
-    acf.addFlag('DQ.doGlobalMon', True)
     acf.addFlag('DQ.doStreamAwareMon', True)
     acf.addFlag('DQ.disableAtlasReadyFilter', False)
     acf.addFlag('DQ.FileKey', 'CombinedMonitoring')
     acf.addFlag('DQ.useTrigger', True)
+
+    # temp thing for steering from inside old-style ...
+    acf.addFlag('DQ.isReallyOldStyle', False)
+
+    # steering ...
+    for flag in _steeringFlags + _lowLevelSteeringFlags:
+        acf.addFlag('DQ.Steering.' + flag, True)
+    # HLT steering ...
+    from TrigHLTMonitoring.TrigHLTMonitoringConfig import createHLTDQConfigFlags
+    acf.join(createHLTDQConfigFlags())
     return acf
 
 def createComplexDQConfigFlags():
@@ -44,3 +69,10 @@ def getEnvironment(flags):
     else:
         # this could use being rethought to properly encode input and output types perhaps ...
         return 'tier0'
+
+def allSteeringFlagsOff():
+    from AthenaConfiguration.AllConfigFlags import ConfigFlags
+    for flag in _steeringFlags:
+        setattr(getattr(ConfigFlags, 'DQ.Steering'), flag, False)
+        #print flag
+        #getattr(ConfigFlags, 'DQ.Steering.' + flag).set(False)

Control/AthenaMonitoring/python/TriggerInterface.py

@@ -65,6 +65,13 @@ def getTrigDecisionTool(flags):
     from TrigDecisionTool.TrigDecisionToolConf import Trig__TrigDecisionTool
  
     rv = ComponentAccumulator()
+
+    if flags.DQ.isReallyOldStyle:
+        from AthenaCommon.AppMgr import ToolSvc
+        rv.addPublicTool(ToolSvc.TrigDecisionTool)
+        getTrigDecisionTool.rv = rv
+        return getTrigDecisionTool.rv
+
     rv.merge(getTrigConfigSvc(flags))
     
     tdt = Trig__TrigDecisionTool('TrigDecisionTool', TrigConfigSvc=rv.getService('TrigConfigSvc'))

Control/AthenaMonitoring/share/DataQualitySteering_jobOptions.py

@@ -282,4 +282,22 @@ if DQMonFlags.doMonitoring():
             postprocfunc(tool)
             del postprocfunc
 
+   # # set up new-style monitoring with new-style configuration
+   # # only enable this when we understand details better...
+   # local_logger.info('Setting up new-style DQ monitoring')
+   # from AthenaMonitoring.AthenaMonitoringCfg import AthenaMonitoringCfg
+   # from AthenaCommon.Configurable import Configurable
+
+   # _ = Configurable.configurableRun3Behavior
+   # Configurable.configurableRun3Behavior = 1
+   # from AthenaConfiguration.AllConfigFlags import ConfigFlags
+   # ConfigFlags.Input.Files = jobproperties.AthenaCommonFlags.FilesInput()
+   # ConfigFlags.Output.HISTFileName = DQMonFlags.histogramFile()
+   # ConfigFlags.DQ.isReallyOldStyle = True
+   # _2 = AthenaMonitoringCfg(ConfigFlags)
+   # Configurable.configurableRun3Behavior = _
+   # _2.printConfig()
+   # _2.appendToGlobals()
+   # del _, _2
+
 del local_logger

Control/AthenaMonitoring/share/Run3DQTestingDriver.py

+#!/usr/bin/env python
+#
+#  Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+#
+
+'''@file DQTestingDriver.py
+@author C. D. Burton
+@author P. Onyisi
+@date 2019-06-20
+@brief Driver script to run DQ with new-style configuration on an ESD/AOD
+'''
+
+if __name__=='__main__':
+    from argparse import ArgumentParser
+    parser = ArgumentParser()
+    parser.add_argument('--preExec', help='Code to execute before locking configs')
+    parser.add_argument('--dqOffByDefault', action='store_true',
+                        help='Set all DQ steering flags to False, user must then switch them on again explicitly')
+    parser.add_argument('flags', nargs='*', help='Config flag overrides')
+    args = parser.parse_args()
+
+    # Setup the Run III behavior
+    from AthenaCommon.Configurable import Configurable
+    Configurable.configurableRun3Behavior = 1
+
+    # Setup logs
+    from AthenaCommon.Logging import log
+    from AthenaCommon.Constants import INFO
+    log.setLevel(INFO)
+
+    # Set the Athena configuration flags
+    from AthenaConfiguration.AllConfigFlags import ConfigFlags
+
+    ConfigFlags.Input.Files = ['/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/Tier0ChainTests/q431/21.0/myESD.pool.root']
+    ConfigFlags.Output.HISTFileName = 'ExampleMonitorOutput.root'
+    if args.dqOffByDefault:
+        from AthenaMonitoring.DQConfigFlags import allSteeringFlagsOff
+        allSteeringFlagsOff()
+    ConfigFlags.fillFromArgs(args.flags)
+
+    if args.preExec:
+        # bring things into scope
+        from AthenaMonitoring.DQConfigFlags import allSteeringFlagsOff
+        log.info('Executing preExec: %s', args.preExec)
+        exec(args.preExec)
+
+    log.info('FINAL CONFIG FLAGS SETTINGS FOLLOW')
+    ConfigFlags.dump()
+        
+    ConfigFlags.lock()
+
+    # Initialize configuration object, add accumulator, merge, and run.
+    from AthenaConfiguration.MainServicesConfig import MainServicesSerialCfg 
+    from AthenaPoolCnvSvc.PoolReadConfig import PoolReadCfg
+    cfg = MainServicesSerialCfg()
+    cfg.merge(PoolReadCfg(ConfigFlags))
+
+    # load DQ
+    from AthenaMonitoring.AthenaMonitoringCfg import AthenaMonitoringCfg
+    dq = AthenaMonitoringCfg(ConfigFlags)
+    cfg.merge(dq)
+
+    # If you want to turn on more detailed messages ...
+    # exampleMonitorAcc.getEventAlgo('ExampleMonAlg').OutputLevel = 2 # DEBUG
+    cfg.printConfig(withDetails=False) # set True for exhaustive info
+
+    cfg.run() #use cfg.run(20) to only run on first 20 events

InnerDetector/InDetMonitoring/SCT_Monitoring/CMakeLists.txt

@@ -53,4 +53,5 @@ atlas_add_component( SCT_Monitoring
 # Install files from the package:
 atlas_install_headers( SCT_Monitoring )
 atlas_install_joboptions( share/*.py )
+atlas_install_python_modules( python/*.py )
 

InnerDetector/InDetMonitoring/SCT_Monitoring/share/SCTLorentzMonAlg.py → InnerDetector/InDetMonitoring/SCT_Monitoring/python/SCTLorentzMonAlg.py

@@ -55,6 +55,10 @@ def SCTLorentzMonAlgConfig(inputFlags):
     # result.merge(caloNoiseAcc)
     # myMonAlg.CaloNoiseTool = caloNoiseTool
 
+    # set up geometry / conditions
+    from AtlasGeoModel.InDetGMConfig import InDetGeometryCfg
+    result.merge(InDetGeometryCfg(inputFlags))
+
     # # Then, add a tool that doesn't have its own configuration function. In
     # # this example, no accumulator is returned, so no merge is necessary.
     # from MyDomainPackage.MyDomainPackageConf import MyDomainTool
@@ -101,45 +105,8 @@ def SCTLorentzMonAlgConfig(inputFlags):
     # and the sequence containing the created algorithms. If we haven't called
     # any configuration other than the AthMonitorCfgHelper here, then we can 
     # just return directly (and not create "result" above)
-    return helper.result()
+    #return helper.result()
     
     # # Otherwise, merge with result object and return
-    # acc, seq = helper.result()
-    # result.merge(acc)
-    # return result
-
-
-if __name__=='__main__':
-    # Setup the Run III behavior
-    from AthenaCommon.Configurable import Configurable
-    Configurable.configurableRun3Behavior = 1
-
-    # Setup logs
-    from AthenaCommon.Logging import log
-    from AthenaCommon.Constants import DEBUG,INFO
-    log.setLevel(INFO)
-
-    # Set the Athena configuration flags
-    from AthenaConfiguration.AllConfigFlags import ConfigFlags
-    ConfigFlags.Input.Files = ["/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/RecExRecoTest/mc16_13TeV.361022.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ2W.recon.ESD.e3668_s3170_r10572_homeMade.pool.root"]
-    ConfigFlags.Input.isMC = True
-    ConfigFlags.Output.HISTFileName = 'SCTLorentzMonOutput.root'
-    ConfigFlags.GeoModel.Align.Dynamic = False
-    ConfigFlags.Detector.GeometryID = True
-    ConfigFlags.Detector.GeometryMuon = True
-    ConfigFlags.lock()
-
-    # Initialize configuration object, add accumulator, merge, and run.
-    from AthenaConfiguration.MainServicesConfig import MainServicesSerialCfg 
-    from AthenaPoolCnvSvc.PoolReadConfig import PoolReadCfg
-    cfg = MainServicesSerialCfg()
-    cfg.merge(PoolReadCfg(ConfigFlags))
-
-    from AtlasGeoModel.AtlasGeoModelConfig import AtlasGeometryCfg
-    geoCfg=AtlasGeometryCfg(ConfigFlags)
-    cfg.merge(geoCfg)
-
-    sctLorentzMonAcc = SCTLorentzMonAlgConfig(ConfigFlags)
-    
-    cfg.merge(sctLorentzMonAcc)
-    cfg.run()
+    result.merge(helper.result())
+    return result

InnerDetector/InDetMonitoring/SCT_Monitoring/share/SCTLorentzMonAlg_test.py

+#
+#  Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+#
+
+'''@file SCTLorentzMonAlg.py
+@author Susumu Oda
+@date 2019-04-02
+@brief Based on AthenaMonitoring/ExampleMonitorAlgorithm.py
+'''
+
+from SCTLorentzMonAlg import SCTLorentzMonAlgConfig
+
+if __name__=='__main__':
+    # Setup the Run III behavior
+    from AthenaCommon.Configurable import Configurable
+    Configurable.configurableRun3Behavior = 1
+
+    # Setup logs
+    from AthenaCommon.Logging import log
+    from AthenaCommon.Constants import DEBUG,INFO
+    log.setLevel(INFO)
+
+    # Set the Athena configuration flags
+    from AthenaConfiguration.AllConfigFlags import ConfigFlags
+    ConfigFlags.Input.Files = ["/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/RecExRecoTest/mc16_13TeV.361022.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ2W.recon.ESD.e3668_s3170_r10572_homeMade.pool.root"]
+    ConfigFlags.Input.isMC = True
+    ConfigFlags.Output.HISTFileName = 'SCTLorentzMonOutput.root'
+    ConfigFlags.GeoModel.Align.Dynamic = False
+    ConfigFlags.Detector.GeometryID = True
+    ConfigFlags.Detector.GeometryMuon = True
+    ConfigFlags.lock()
+
+    # Initialize configuration object, add accumulator, merge, and run.
+    from AthenaConfiguration.MainServicesConfig import MainServicesSerialCfg 
+    from AthenaPoolCnvSvc.PoolReadConfig import PoolReadCfg
+    cfg = MainServicesSerialCfg()
+    cfg.merge(PoolReadCfg(ConfigFlags))
+
+    from AtlasGeoModel.AtlasGeoModelConfig import AtlasGeometryCfg
+    geoCfg=AtlasGeometryCfg(ConfigFlags)
+    cfg.merge(geoCfg)
+
+    sctLorentzMonAcc = SCTLorentzMonAlgConfig(ConfigFlags)
+    
+    cfg.merge(sctLorentzMonAcc)
+    cfg.run()

Trigger/TrigMonitoring/TrigHLTMonitoring/python/TrigHLTMonitoringConfig.py

-# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+
+def createHLTDQConfigFlags():
+    from AthenaConfiguration.AthConfigFlags import AthConfigFlags
+    acf=AthConfigFlags()
+    acf.addFlag('DQ.Steering.HLT.doMET', True)
+    return acf
+
+def TrigHLTMonitoringConfig(flags):
+    from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator
+    result = ComponentAccumulator()
+
+    if flags.DQ.Steering.HLT.doMET:
+        from TrigMETMonitoring.TrigMETMonitorAlgorithm import TrigMETMonConfig
+        result.merge(TrigMETMonConfig(flags))
+
+    return result
 
 def HLTGeneralTool():
     #from AthenaCommon.AppMgr import ToolSvc