Merge branch 'torrence-noisystrips' into 'master'

NoisyStrips Update

See merge request !271

Control/CalypsoExample/Reconstruction/scripts/faser_reco.py

@@ -58,9 +58,9 @@ else:
     # Try to pick correct geometry from run number
     # This won't work for testbeam data, 
     # so lets call this a hack for now
-    runname = filepath.parts[-2]
+    runname = filepath.parts[-1]
     try:
-        runnumber = int(runname.split('-')[1])
+        runnumber = int(runname.split('-')[2])
     except Exception as e:
         print(f"Failed to find run number in {filepath}")
         print(f"Couldn't parse {runname}")

Tracker/TrackerConditions/FaserSCT_ConditionsAlgorithms/src/FaserSCT_ConfigurationCondAlg.h

@@ -35,7 +35,7 @@ class FaserSCT_ConfigurationCondAlg : public AthReentrantAlgorithm {
   SG::WriteCondHandleKey<FaserSCT_ConfigurationCondData> m_writeKey{this, "WriteKey", "FaserSCT_ConfigurationCondData", "Key of output (derived) conditions data"};
   ServiceHandle<ICondSvc> m_condSvc{this, "CondSvc", "CondSvc"};
   const FaserSCT_ID* m_idHelper{nullptr};
-  Gaudi::Property<double> m_occupancyThreshold {this, "OccupancyThreshold", 0.015, "Mask strips with an occupancy larger than the OccupancyCut"};
+  Gaudi::Property<double> m_occupancyThreshold {this, "OccupancyThreshold", 0.01, "Mask strips with an occupancy larger than the OccupancyCut"};
 };
 
 

Tracker/TrackerConditions/FaserSCT_ConditionsTools/python/FaserSCT_ConditionsSummaryToolConfig.py

@@ -11,19 +11,15 @@ def FaserSCT_ConfigurationCondAlgCfg(flags, name="FaserSCT_ConfigurationCondAlg"
     acc = ComponentAccumulator()
     dbInstance = kwargs.get("dbInstance", "INDET_OFL")
     dbFolder = kwargs.get("dbFolder", "/SCT/DAQ/NoisyStrips")
-    acc.merge(addFolders(flags, folderstrings=dbFolder, detDb=dbInstance, className="CondAttrListCollection"))
+    #dbFolder = kwargs.get("dbFolder", "/INDET/NoisyStrips")
+    acc.merge(addFolders(flags, dbFolder, dbInstance, className="CondAttrListCollection"))
     acc.addCondAlgo(FaserSCT_ConfigurationCondAlg(name, **kwargs))
     return acc
 
 
 def FaserSCT_ConfigurationConditionsToolCfg(flags, name="FaserSCT_ConfigurationCondAlg", **kwargs):
     acc = ComponentAccumulator()
-    # tool = kwargs.get("ConditionsTools", FaserSCT_ConfigurationConditionsToolCfg(flags))
-    # folder arguments
-    dbInstance = kwargs.get("dbInstance", "INDET_OFL")
-    dbFolder = kwargs.get("dbFolder", "/SCT/DAQ/NoisyStrips")
     acc.merge(FaserSCT_ConfigurationCondAlgCfg(flags, name="FaserSCT_ConfigurationCondAlg", **kwargs))
-    # acc.setPrivateTools(tool)
     acc.setPrivateTools(FaserSCT_ConfigurationConditionsTool(name, **kwargs))
     return acc
 

Tracker/TrackerConditions/FaserSCT_ConditionsTools/python/FaserSCT_NoisyStripsConfig.py

@@ -16,7 +16,7 @@ def FaserSCT_NoisyStripsCfg(flags, **kwargs):
     NoisyStripsTool may be provided in kwargs
     """
     acc = ComponentAccumulator()
-    dbInstance = kwargs.get("dbInstance", "TDAQ_OFL")
+    dbInstance = kwargs.get("dbInstance", "INDET_OFL")
     dbFolder = kwargs.get("dbFolder", "/SCT/DAQ/NoisyStrips")
     acc.merge(addFolders(flags, dbFolder, dbInstance, className="CondAttrListCollection"))
     # acc.addPublicTool(tool)

Tracker/TrackerRecAlgs/NoisyStripFinder/CMakeLists.txt

@@ -17,5 +17,5 @@ atlas_add_component( NoisyStripFinder
 
 atlas_install_python_modules( python/*.py )
 
-#atlas_install_scripts( test/*.py )
+atlas_install_scripts( share/*.py test/*.py )
 

Tracker/TrackerRecAlgs/NoisyStripFinder/share/makeEmptyNoisyStripDB.py

+#!/usr/bin/env python
+
+import ROOT
+
+from PyCool import cool
+
+dbSvc = cool.DatabaseSvcFactory.databaseService()
+# Data DB name
+connectString = 'sqlite://;schema=noisy_strips.db;dbname=CONDBR3'
+
+print('Creating empty database')
+
+dbSvc.dropDatabase(connectString)
+
+db = dbSvc.createDatabase(connectString)
+
+noisyStripsSpec = cool.RecordSpecification()
+noisyStripsSpec.extend('sensor', cool.StorageType.Int32)
+noisyStripsSpec.extend('strip', cool.StorageType.Int32)
+noisyStripsSpec.extend('occupancy', cool.StorageType.Float)
+
+description = '<timeStamp>run-lumi</timeStamp><addrHeader><address_header clid="1238547719" service_type="71" /></addrHeader><typeName>CondAttrListCollection</typeName>'
+noisyStripsFolderSpec = cool.FolderSpecification(cool.FolderVersioning.SINGLE_VERSION, noisyStripsSpec)
+noisyStripsFolder = db.createFolder('/SCT/DAQ/NoisyStrips', noisyStripsFolderSpec, description, True)
+
+firstValid = cool.ValidityKeyMin
+lastValid = cool.ValidityKeyMax
+
+numNoisyStrips = 0  # This is used as a channel...
+for dictkey in range(0, 192):
+    for bini in range(1): # Could go to 768, but no need to write too many empty strips
+        noisyStripsRecord = cool.Record(noisyStripsSpec)
+        noisyStripsRecord['sensor'] = int(dictkey)
+        noisyStripsRecord['strip'] = int(bini)
+        noisyStripsRecord['occupancy'] = 0.
+        numNoisyStrips += 1
+        noisyStripsFolder.storeObject(firstValid, lastValid, noisyStripsRecord, numNoisyStrips)
+
+db.closeDatabase()
+
+# Do this again for MC
+connectString = 'sqlite://;schema=noisy_strips.db;dbname=OFLP200'
+
+print('Creating empty MC database')
+
+# Don't drop the DB we just made
+#dbSvc.dropDatabase(connectString)
+
+# Create new DB (different name in same file for MC
+db = dbSvc.createDatabase(connectString)
+
+noisyStripsFolder = db.createFolder('/SCT/DAQ/NoisyStrips', noisyStripsFolderSpec, description, True)
+
+firstValid = cool.ValidityKeyMin
+lastValid = cool.ValidityKeyMax
+
+numNoisyStrips = 0  # This is used as a channel...
+for dictkey in range(0, 192):
+    for bini in range(1): # Could go to 768, but no need to write too many empty strips
+        noisyStripsRecord = cool.Record(noisyStripsSpec)
+        noisyStripsRecord['sensor'] = int(dictkey)
+        noisyStripsRecord['strip'] = int(bini)
+        noisyStripsRecord['occupancy'] = 0.
+        numNoisyStrips += 1
+        noisyStripsFolder.storeObject(firstValid, lastValid, noisyStripsRecord, numNoisyStrips)
+
+db.closeDatabase()
+
+print('Database completed')

Tracker/TrackerRecAlgs/NoisyStripFinder/share/makeNoisyStripDB.py

 #!/usr/bin/env python
 
+import os
+import sys
 import ROOT
 import argparse
+from PyCool import cool
+from CoolConvUtilities.AtlCoolLib import indirectOpen
 
 parser = argparse.ArgumentParser()
 parser.add_argument("file", nargs="+", help="full path to input file")
 parser.add_argument("-t", "--threshold", type=float, default=0.01, help="add strips with an occupancy larger this threshold to the database")
+parser.add_argument("--force", "-f", action="store_true", help="Overwrite existing DB")
+parser.add_argument("--output", "-o", default="noisy_strips.db", help="Specify output DB")
+parser.add_argument("--isMC", action="store_true", help="Write MC DB (default: real data")
 args = parser.parse_args()
 
 def GetKeyNames(self):
@@ -19,34 +26,69 @@ HistDict = {}
 
 ROOT.TH1.AddDirectory(0) # This is necessary in order to have the histogram data after closing the file
 
+iovlo = cool.ValidityKeyMax
+iovhi = cool.ValidityKeyMin
+
+# Keys to skip
+skipList = ["numEvents", "trigger", "IOVLoRun", "IOVLoLB", "IOVHiRun", "IOVHiLB"]
+
 for inputfile in args.file:
     f = ROOT.TFile.Open(inputfile, "r")
     numEvents += f.Get("numEvents").GetVal()
+    lorun = f.Get("IOVLoRun").GetVal()
+    hirun = f.Get("IOVHiRun").GetVal()
+    lo = (lorun << 32)
+    hi = ((hirun+1) << 32) - 1
+    if lo < iovlo: iovlo = lo
+    if hi > iovhi: iovhi = hi
+
     if nfiles == 0:
         trigger = f.Get("trigger").GetVal()
+
     for rootkey in f.GetKeyNames():
-        if rootkey == 'numEvents' or rootkey == 'trigger':
-            continue # skip over the root objects TParameters that store the trigger and number of events data
+
+        # skip over the root objects TParameters that store the trigger and number of events data
+        if rootkey in skipList: continue 
+
         if rootkey in HistDict: # if sensor histogram has already been stored, then add to it
             HistDict[rootkey].Add(f.Get(rootkey),1.0)
         else: # if sensor histogram has not already been stored, then store this histogram
             HistDict[rootkey] = f.Get(rootkey).Clone()
+
     nfiles += 1
     f.Close()
 
 print("Total # of root files analyzed = ", nfiles)
-print("Trigger mask = ", trigger)
+print(f"Trigger mask = 0x{trigger:02x}")
 print("Total number of events = ", numEvents)
+print(f"IOV from {(iovlo >> 32)}/{(iovlo & 0xFFFFFFFF)} to {(iovhi >> 32)}/{(iovhi & 0xFFFFFFFF)}")
 
-from PyCool import cool
+# Write DB
 
 dbSvc = cool.DatabaseSvcFactory.databaseService()
-connectString = 'sqlite://;schema=noisy_strips.db;dbname=OFLP200'
-
-print('Recreating database')
-
-dbSvc.dropDatabase(connectString)
-db = dbSvc.createDatabase(connectString)
+dbname = "CONDBR3"   # Real data
+if args.isMC:
+    dbname="OFLP200" # MC
+
+connectString = f'sqlite://;schema={args.output};dbname={dbname}'
+
+print(f"Using connection string {connectString}")
+
+if os.path.exists(args.output):
+    if args.force:
+        print(f"Deleting {args.output} due to --force")
+        os.remove(args.output)
+    else:
+        print(f"File {args.output} exists, use --force to overwrite")
+        sys.exit(1)
+
+try:
+    print('Creating database')
+    dbSvc.dropDatabase(connectString)
+    db = dbSvc.createDatabase(connectString)
+except Exception as e:
+    print(e)
+    sys.exit(1)
 
 noisyStripsSpec = cool.RecordSpecification()
 noisyStripsSpec.extend('sensor', cool.StorageType.Int32)
@@ -55,10 +97,12 @@ noisyStripsSpec.extend('occupancy', cool.StorageType.Float)
 
 description = '<timeStamp>run-lumi</timeStamp><addrHeader><address_header clid="1238547719" service_type="71" /></addrHeader><typeName>CondAttrListCollection</typeName>'
 noisyStripsFolderSpec = cool.FolderSpecification(cool.FolderVersioning.SINGLE_VERSION, noisyStripsSpec)
+
+print("Creating new folder")
 noisyStripsFolder = db.createFolder('/SCT/DAQ/NoisyStrips', noisyStripsFolderSpec, description, True)
 
-firstValid = 429 << 32
-lastValid = cool.ValidityKeyMax
+firstValid = iovlo
+lastValid  = iovhi
 
 numNoisyStrips = 0
 for dictkey in HistDict:

Tracker/TrackerRecAlgs/NoisyStripFinder/src/NoisyStripFinder.cxx

@@ -9,6 +9,7 @@
 #include "TrackerRawData/FaserSCT_RDO_Container.h"
 #include "TrackerRawData/FaserSCT_RDORawData.h"
 #include "StoreGate/WriteHandle.h"
+#include "xAODEventInfo/EventInfo.h"
 
 #include <sstream>
 #include <string.h>
@@ -24,6 +25,7 @@ NoisyStripFinder::NoisyStripFinder(const std::string& name, ISvcLocator* pSvcLoc
   AthReentrantAlgorithm(name, pSvcLocator),
   m_idHelper{nullptr}
 {
+  m_iovrange = IOVRange(IOVTime(), IOVTime());  // Make sure this starts undefined
 }
 
 // Initialize method:
@@ -32,6 +34,7 @@ StatusCode NoisyStripFinder::initialize() {
 
   ATH_CHECK(m_rdoContainerKey.initialize());
   ATH_CHECK(m_FaserTriggerData.initialize());
+  ATH_CHECK(m_eventInfo.initialize());
 
   // Get the SCT ID helper
   ATH_CHECK(detStore()->retrieve(m_idHelper, "FaserSCT_ID"));
@@ -43,7 +46,7 @@ StatusCode NoisyStripFinder::initialize() {
 StatusCode NoisyStripFinder::execute(const EventContext& ctx) const {
 
   SG::ReadHandle<xAOD::FaserTriggerData> xaod(m_FaserTriggerData, ctx);
-
+ 
   int trig_int = xaod->tap();
   int trigmask_int = m_triggerMask.value();
 
@@ -54,7 +57,21 @@ StatusCode NoisyStripFinder::execute(const EventContext& ctx) const {
   if (!(xaod->tap() & m_triggerMask.value())) return StatusCode::SUCCESS; // only process events that pass the trigger mask
 
   ATH_MSG_INFO("trigger passed mask");
-  ++m_numberOfEvents;                                                                
+  ++m_numberOfEvents;
+
+  // Keep track of run
+  SG::ReadHandle<xAOD::EventInfo> xevt(m_eventInfo, ctx);
+  ATH_MSG_DEBUG("Found run number: " << xevt->runNumber());
+  IOVTime iov(xevt->runNumber(), xevt->lumiBlock());
+  ATH_MSG_DEBUG("IOV: " << iov);
+
+  if (!m_iovrange.start().isValid()) 
+    m_iovrange = IOVRange(iov, iov);
+   
+  if (iov > m_iovrange.stop())
+    m_iovrange = IOVRange(m_iovrange.start(), iov);
+
+  ATH_MSG_DEBUG("Range: " << m_iovrange);
 
   // First, we have to retrieve and access the container, not because we want to 
   // use it, but in order to generate the proxies for the collections, if they 
@@ -110,6 +127,8 @@ StatusCode NoisyStripFinder::finalize()
     }
   }
 
+  ATH_MSG_INFO("IOV range found = " << m_iovrange);
+
   const char *outputname = m_OutputRootName.value().c_str();
 
   TFile* outputfile = new TFile(outputname,"RECREATE");
@@ -119,6 +138,27 @@ StatusCode NoisyStripFinder::finalize()
   TParameter("numEvents", m_numberOfEvents).Write();
   TParameter("trigger", trigmask_int).Write();
 
+  // Write IOV range so we can save this to the DB 
+  if (m_iovrange.start().isValid()) {
+    long run = m_iovrange.start().run();
+    long lb = m_iovrange.start().event();
+    TParameter("IOVLoRun", run).Write();  // re_time()
+    TParameter("IOVLoLB",  lb).Write();
+    ATH_MSG_INFO("IOV Lo: " << run << "," << lb );
+  }
+  else
+    ATH_MSG_WARNING("Starting IOV time invalid");
+
+  if (m_iovrange.stop().isValid()) {
+    long run = m_iovrange.stop().run();
+    long lb = m_iovrange.stop().event();
+    TParameter("IOVHiRun", run).Write();
+    TParameter("IOVHiLB", lb).Write();
+    ATH_MSG_INFO("IOV Hi: " << run << "," << lb );
+  }
+  else
+    ATH_MSG_WARNING("Ending IOV time invalid");
+
   std::map<int,TH1D*>::iterator it = NoisyStrip_histmap.begin();
   // Iterate over the map using Iterator till end.
   while (it != NoisyStrip_histmap.end()){

Tracker/TrackerRecAlgs/NoisyStripFinder/src/NoisyStripFinder.h

@@ -17,7 +17,8 @@
 #include "GaudiKernel/ToolHandle.h"
 
 #include "StoreGate/ReadHandleKey.h"
-
+#include "AthenaKernel/IOVRange.h"
+#include "xAODEventInfo/EventInfo.h"
 
 //STL
 #include <map>
@@ -72,6 +73,7 @@ class NoisyStripFinder : public AthReentrantAlgorithm {
 
     SG::ReadHandleKey<FaserSCT_RDO_Container> m_rdoContainerKey{this, "DataObjectName", "FaserSCT_RDOs", "FaserSCT RDOs"};
     SG::ReadHandleKey<xAOD::FaserTriggerData> m_FaserTriggerData{ this, "FaserTriggerDataKey", "FaserTriggerData", "ReadHandleKey for xAOD::FaserTriggerData"};
+    SG::ReadHandleKey<xAOD::EventInfo> m_eventInfo{ this, "EventInfoKey", "EventInfo", "ReadHandleKey for xAOD::EventInfo"};
 
     mutable int m_numberOfEvents{0};
     mutable std::atomic<int> m_numberOfRDOCollection{0};
@@ -79,6 +81,9 @@ class NoisyStripFinder : public AthReentrantAlgorithm {
 
     mutable std::map<int,TH1D*> NoisyStrip_histmap;
 
+    // Keep track of first/last IOV seen
+    // Stored as (run << 32) + lumi block
+    mutable IOVRange m_iovrange;
 };
 }
 #endif // NoisyStripFinder_H

Tracker/TrackerRecAlgs/NoisyStripFinder/test/NoisyStripFinderJob.py

+#!/usr/bin/env python
+"""
+    Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+"""
+
+import sys
+from AthenaCommon.Logging import log, logging
+from AthenaCommon.Constants import DEBUG, VERBOSE, INFO
+from AthenaCommon.Configurable import Configurable
+from CalypsoConfiguration.AllConfigFlags import ConfigFlags
+from CalypsoConfiguration.MainServicesConfig import MainServicesCfg
+from AthenaPoolCnvSvc.PoolWriteConfig import PoolWriteCfg
+from FaserByteStreamCnvSvc.FaserByteStreamCnvSvcConfig import FaserByteStreamCnvSvcCfg
+from TrackerPrepRawDataFormation.TrackerPrepRawDataFormationConfig import FaserSCT_ClusterizationCfg
+from NoisyStripFinder.NoisyStripFinderConfig import NoisyStripFinderCfg
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("file", nargs="+", help="full path to input file")
+parser.add_argument("--nevents", "-n", default=-1, type=int, help="Number of events to process")
+parser.add_argument("--outfile", "-o", default="NoisyStripFinderHist.root", help="Specify output file")
+
+parser.add_argument("--verbose", "-v", action="store_true", help="Debug output")
+args = parser.parse_args()
+
+if args.verbose:
+    log.setLevel(DEBUG)
+else:
+    log.setLevel(INFO)
+
+Configurable.configurableRun3Behavior = True
+
+#
+filelist = []
+for filename in args.file:
+    if filename[:22] == "/eos/experiment/faser/":
+        filelist.append(f"root://eospublic.cern.ch/{filename}")
+    else:
+        filelist.append(filename)
+
+ConfigFlags.Input.Files = args.file
+ConfigFlags.IOVDb.GlobalTag = "OFLCOND-FASER-02"
+#ConfigFlags.IOVDb.DatabaseInstance = "OFLP200"
+ConfigFlags.IOVDb.DatabaseInstance = "CONDBR3"
+ConfigFlags.Input.ProjectName = "data22"
+ConfigFlags.Input.isMC = False
+ConfigFlags.GeoModel.FaserVersion = "FASERNU-03"
+ConfigFlags.Common.isOnline = False
+ConfigFlags.GeoModel.Align.Dynamic = False
+ConfigFlags.Beam.NumberOfCollisions = 0.
+ConfigFlags.Detector.GeometryFaserSCT = True
+ConfigFlags.lock()
+
+acc = MainServicesCfg(ConfigFlags)
+acc.merge(PoolWriteCfg(ConfigFlags))
+acc.merge(FaserByteStreamCnvSvcCfg(ConfigFlags, OccupancyCut=-1))
+acc.merge(NoisyStripFinderCfg(ConfigFlags, OutputHistRootName=args.outfile))
+
+# Hack to avoid problem with our use of MC databases when isMC = False
+replicaSvc = acc.getService("DBReplicaSvc")
+replicaSvc.COOLSQLiteVetoPattern = ""
+replicaSvc.UseCOOLSQLite = True
+replicaSvc.UseCOOLFrontier = False
+replicaSvc.UseGeomSQLite = True
+
+if args.verbose:
+    acc.foreach_component("*").OutputLevel = VERBOSE
+else:
+    acc.foreach_component("*").OutputLevel = INFO
+
+sc = acc.run(maxEvents=args.nevents)
+sys.exit(not sc.isSuccess())

Tracking/Acts/FaserActsKalmanFilter/test/TI12CKF2.py

@@ -16,7 +16,7 @@ from TrackerSpacePointFormation.TrackerSpacePointFormationConfig import TrackerS
 from TrackerSegmentFit.TrackerSegmentFitConfig import SegmentFitAlgCfg
 from FaserActsKalmanFilter.GhostBustersConfig import GhostBustersCfg
 from FaserActsKalmanFilter.TI12CKF2Config import TI12CKF2Cfg
-from FaserGeoModel.FaserGeoModelConfig import FaserGeometryCfg
+
 import argparse
 
 parser = argparse.ArgumentParser()
@@ -40,7 +40,6 @@ ConfigFlags.GeoModel.Align.Dynamic = False
 ConfigFlags.Beam.NumberOfCollisions = 0.
 ConfigFlags.Detector.GeometryFaserSCT = True
 ConfigFlags.TrackingGeometry.MaterialSource = "geometry-maps.json"
-# ConfigFlags.TrackingGeometry.MaterialSource = "geometry-maps.json"
 ConfigFlags.lock()
 
 acc = MainServicesCfg(ConfigFlags)