Merge branch 'bookkeeper_redaction' into 'master'

Back out new BookkeeperTool

See merge request atlas/athena!21860

Event/EventBookkeeperTools/EventBookkeeperTools/BookkeeperTool.h

 /*
-  Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
 */
 
 #ifndef BOOKKEEPERTOOL_H
@@ -12,7 +12,15 @@
  *  $Id: $
  **/
 
-#include "AthenaKernel/GenericMetadataTool.h"
+//#include "GaudiKernel/AlgTool.h"
+#include "AthenaBaseComps/AthAlgTool.h"
+#include "AsgTools/AsgMetadataTool.h"
+#ifdef ASGTOOL_ATHENA
+#include "AthenaKernel/IMetaDataTool.h"
+#endif // ASGTOOL_ATHENA
+#include "GaudiKernel/IIncidentListener.h"
+#include "GaudiKernel/ServiceHandle.h"
+#include "AthenaKernel/ICutFlowSvc.h"
 
 #include "xAODCutFlow/CutBookkeeper.h"
 #include "xAODCutFlow/CutBookkeeperContainer.h"
@@ -21,24 +29,55 @@
 #include <string>
 
 
-class BookkeeperTool : public GenericMetadataTool <xAOD::CutBookkeeperContainer, xAOD::CutBookkeeperAuxContainer> 
+class BookkeeperTool : public asg::AsgMetadataTool
+#ifdef ASGTOOL_ATHENA
+                     , public virtual ::IMetaDataTool
+#endif // ASGTOOL_ATHENA
 {
-public: 
-   // Constructor and Destructor
+   ASG_TOOL_CLASS0(BookkeeperTool)
+public: // Constructor and Destructor
    /// Standard Service Constructor
-   BookkeeperTool(const std::string& type, 
-                  const std::string& name,
-                  const IInterface*  parent);
+   BookkeeperTool(const std::string& name = "BookkeeperTool");
    /// Destructor
    virtual ~BookkeeperTool();
 
+public:
+   //void handle(const Incident& incident);
+   virtual StatusCode metaDataStop();
+   virtual StatusCode beginInputFile() override {return StatusCode::SUCCESS;}
+   virtual StatusCode endInputFile() override {return StatusCode::SUCCESS;}
+   virtual StatusCode beginInputFile(const SG::SourceID&) override;
+   virtual StatusCode endInputFile(const SG::SourceID&) override;
+   virtual StatusCode initialize();
+   virtual StatusCode finalize();
+
 private:
   
-  /// Virtual method to update a container with information from another one
-  virtual StatusCode updateContainer(xAOD::CutBookkeeperContainer* contToUpdate,
-                               const xAOD::CutBookkeeperContainer* otherCont );
+  /// Helper class to update a container with information from another one
+  StatusCode updateContainer( xAOD::CutBookkeeperContainer* contToUpdate,
+                              const xAOD::CutBookkeeperContainer* otherCont );
+
+  StatusCode copyContainerToOutput(const std::string& outname);
+
+  /// Fill Cutflow information
+  StatusCode addCutFlow();
+ 
+  /// Pointer to cut flow svc 
+  //ServiceHandle<ICutFlowSvc> m_cutflowsvc;
+
+  /// The name of the output CutBookkeeperContainer
+  std::string m_outputCollName;
+  
+  /// The name of the input CutBookkeeperContainer
+  std::string  m_inputCollName;
+
+  /// The name of the CutFlowSvc CutBookkeeperContainer
+  std::string m_cutflowCollName;
+
+  bool m_cutflowTaken;
 
 };
 
 #endif
 
+

Event/EventBookkeeperTools/Root/BookkeeperTool.cxx

 ///////////////////////// -*- C++ -*- /////////////////////////////
 
 /*
-  Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+  Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
 */
 
 // Implementation file for class BookkeeperTool
@@ -13,28 +13,241 @@
 // STL include
 #include <algorithm>
 
-#include "AthenaKernel/MetaCont.h"
-#include "AthenaKernel/ClassID_traits.h"
+// #include "FillEBCFromFlat.h"
+
+#include "GaudiKernel/Incident.h"
+#include "GaudiKernel/FileIncident.h"
+#include "GaudiKernel/IIncidentSvc.h"
 #include "AthenaKernel/errorcheck.h"
-#include "AthenaKernel/GenericMetadataTool.h"
-#include "AthenaKernel/ICutFlowSvc.h"
 #include "AthenaBaseComps/AthCheckMacros.h"
-#include "AthContainersInterfaces/IConstAuxStoreMeta.h"
 
-BookkeeperTool::BookkeeperTool(const std::string& type, 
-                               const std::string& name,
-                               const IInterface*  parent)
-  : GenericMetadataTool <xAOD::CutBookkeeperContainer, xAOD::CutBookkeeperAuxContainer>(type,name,parent)
+
+BookkeeperTool::BookkeeperTool(const std::string& name)
+  : asg::AsgMetadataTool(name),
+    m_cutflowTaken(false)
 {
+  declareProperty("OutputCollName", m_outputCollName="CutBookkeepers",  
+    "The default name of the xAOD::CutBookkeeperContainer for output files");
+  declareProperty("InputCollName", m_inputCollName = "CutBookkeepers",
+    "The default name of the xAOD::CutBookkeeperContainer for input files");
+  declareProperty("CutFlowCollName", m_cutflowCollName = "CutBookkeepersFile",
+    "The default name of the xAOD::CutBookkeeperContainer for CutFlowSvc");
+#ifdef ASGTOOL_ATHENA
+  declareInterface< ::IMetaDataTool >( this );
+#endif // ASGTOOL_ATHENA
 }
 
+
+
 BookkeeperTool::~BookkeeperTool()
 {
 }
 
 
+
+StatusCode
+BookkeeperTool::initialize()
+{
+  ATH_MSG_DEBUG( "Initializing " << name() << " - package version " << PACKAGE_VERSION );
+
+  ATH_MSG_DEBUG("InputCollName = " << m_inputCollName);
+  ATH_MSG_DEBUG("OutputCollName = " << m_outputCollName);
+  ATH_MSG_DEBUG("CutFlowCollName = " << m_cutflowCollName);
+
+  return StatusCode::SUCCESS;
+}
+
+
+//__________________________________________________________________________
+StatusCode BookkeeperTool::beginInputFile(const SG::SourceID&)
+{
+  //OPENING NEW INPUT FILE
+  //Things to do:
+  // 1) note that a file is currently opened
+  // 2) Load CutBookkeepers from input file
+  //    2a) if incomplete from input, directly propagate to output
+  //    2b) if complete from input, wait for EndInputFile to decide what to do in output
+
+  // reset cutflow taken marker
+  m_cutflowTaken = false;
+
+  // Get the incomplete bookkeeper collection of the input metadata store
+  const xAOD::CutBookkeeperContainer* input_inc = 0;
+  // Construct input and output incomplete names
+  std::string inCollName = "Incomplete" + m_inputCollName;
+  std::string outCollName = "Incomplete" + m_outputCollName;
+  if (inputMetaStore()->contains<xAOD::CutBookkeeperContainer>(inCollName) ) {
+    StatusCode ssc = inputMetaStore()->retrieve( input_inc, inCollName );
+    if (ssc.isSuccess()) {
+      // First make sure there is an incomplete container in the output store
+      if( !(outputMetaStore()->contains<xAOD::CutBookkeeperContainer>(outCollName)) ) {
+        xAOD::CutBookkeeperContainer* inc = new xAOD::CutBookkeeperContainer();
+        xAOD::CutBookkeeperAuxContainer* auxinc = new xAOD::CutBookkeeperAuxContainer();
+        inc->setStore(auxinc);
+        ATH_CHECK(outputMetaStore()->record(inc,outCollName));
+        ATH_CHECK(outputMetaStore()->record(auxinc,outCollName+"Aux."));
+      }
+      // retrieve the incomplete output container
+      xAOD::CutBookkeeperContainer* incompleteBook(NULL);
+      ATH_CHECK(outputMetaStore()->retrieve( incompleteBook, outCollName));
+      // update incomplete output with any incomplete input
+      ATH_CHECK(this->updateContainer(incompleteBook,input_inc));
+      ATH_MSG_DEBUG("Successfully merged input incomplete bookkeepers with output");
+    }
+  }
+  else {
+    ATH_MSG_INFO("No incomplete bookkeepers in this file " << inCollName);
+  }
+
+  // Get the complete bookkeeper collection of the input metadata store
+  const xAOD::CutBookkeeperContainer* input_com = 0;
+  inCollName = m_inputCollName;
+  outCollName = m_outputCollName;
+  if (inputMetaStore()->contains<xAOD::CutBookkeeperContainer>(inCollName) ) {
+    if ( (inputMetaStore()->retrieve( input_com, inCollName )).isSuccess() ) {
+      // Check if a tmp is there. IT SHOULD NOT BE
+      //xAOD::CutBookkeeperContainer* incompleteBook(NULL);
+      if( !(outputMetaStore()->contains<xAOD::CutBookkeeperContainer>(outCollName+"tmp")) ) {
+        // Now create the tmp container
+        xAOD::CutBookkeeperContainer* tmp = new xAOD::CutBookkeeperContainer();
+        xAOD::CutBookkeeperAuxContainer* auxtmp = new xAOD::CutBookkeeperAuxContainer();
+        tmp->setStore(auxtmp);
+        if (updateContainer(tmp,input_com).isSuccess()) {
+          ATH_CHECK(outputMetaStore()->record(tmp,outCollName+"tmp"));
+          ATH_CHECK(outputMetaStore()->record(auxtmp,outCollName+"tmpAux."));
+        }
+        else {
+          ATH_MSG_WARNING("Could not update tmp container from input complete conatiner");
+        }
+      }
+    }
+    else {
+      ATH_MSG_WARNING("tmp collection already exists");
+      return StatusCode::SUCCESS;
+    }
+    ATH_MSG_DEBUG("Successfully copied complete bookkeepers to temp container");
+  }
+
+  //  Now make sure the output containers are in the output store
+  //
+  //  Make sure complete container exists in output
+  if( !(outputMetaStore()->contains<xAOD::CutBookkeeperContainer>(m_outputCollName)) ) {
+      // Now create the complete container
+    xAOD::CutBookkeeperContainer* inc = new xAOD::CutBookkeeperContainer();
+    xAOD::CutBookkeeperAuxContainer* auxinc = new xAOD::CutBookkeeperAuxContainer();
+    inc->setStore(auxinc);
+    ATH_CHECK(outputMetaStore()->record(inc,m_outputCollName));
+    ATH_CHECK(outputMetaStore()->record(auxinc,m_outputCollName+"Aux."));
+  }
+  else {
+    ATH_MSG_WARNING("complete collection already exists");
+    //return StatusCode::SUCCESS;
+  }
+  //  Make sure incomplete container exists in output
+  std::string inc_name = "Incomplete"+m_outputCollName;
+  if( !(outputMetaStore()->contains<xAOD::CutBookkeeperContainer>(inc_name)) ) {
+      // Now create the complete container
+    xAOD::CutBookkeeperContainer* coll = new xAOD::CutBookkeeperContainer();
+    xAOD::CutBookkeeperAuxContainer* auxcoll = new xAOD::CutBookkeeperAuxContainer();
+    coll->setStore(auxcoll);
+    ATH_CHECK(outputMetaStore()->record(coll,inc_name));
+    ATH_CHECK(outputMetaStore()->record(auxcoll,inc_name+"Aux."));
+  }
+  else {
+    ATH_MSG_WARNING("incomplete collection already exists");
+  }
+  
+  return StatusCode::SUCCESS;
+}
+
+
+StatusCode BookkeeperTool::endInputFile(const SG::SourceID&)
+{
+
+  if (copyContainerToOutput(m_outputCollName).isFailure()) return StatusCode::FAILURE;
+
+  if (!m_cutflowTaken) {
+    if (addCutFlow().isFailure()) {
+      ATH_MSG_ERROR("Could not add CutFlow information");
+    }
+    m_cutflowTaken = true;
+  }
+  else {
+    ATH_MSG_DEBUG("Cutflow information written into container before endInputFile");
+  }
+    
+  return StatusCode::SUCCESS;
+}
+
+StatusCode BookkeeperTool::metaDataStop()
+{
+  //TERMINATING THE JOB (EndRun)
+  //Things to do:
+  // 1) Create new incomplete CutBookkeepers if relevant
+  // 2) Print cut flow summary
+  // 3) Write root file if requested
+  //  Make sure incomplete container exists in output
+  std::string inc_name = "Incomplete"+m_outputCollName;
+  if (copyContainerToOutput(inc_name).isFailure()) return StatusCode::FAILURE;
+
+
+  if (!m_cutflowTaken) {
+    if (addCutFlow().isFailure()) {
+      ATH_MSG_ERROR("Could not add CutFlow information");
+    }
+  }
+  else {
+    ATH_MSG_DEBUG("Cutflow information written into container before metaDataStop");
+  }
+
+  // Reset after metadata stop
+  m_cutflowTaken = false;
+  
+  return StatusCode::SUCCESS;
+}
+
+
+StatusCode
+BookkeeperTool::finalize()
+{
+  ATH_MSG_DEBUG( "Finalizing " << name() << " - package version " << PACKAGE_VERSION );
+  return StatusCode::SUCCESS;
+}
+
+
+StatusCode BookkeeperTool::addCutFlow()
+{
+  // Add the information from the current processing to the complete output
+  // --> same paradigm as original CutFlowSvc
+  // Get the complete bookkeeper collection of the output meta-data store
+  xAOD::CutBookkeeperContainer* completeBook(NULL); 
+  if( !(outputMetaStore()->retrieve( completeBook, m_outputCollName) ).isSuccess() ) {
+    ATH_MSG_ERROR( "Could not get complete CutBookkeepers from output MetaDataStore" );
+    return StatusCode::FAILURE;
+  }
+
+  // Get the bookkeeper from the current processing
+  const xAOD::CutBookkeeperContainer* fileCompleteBook(NULL);
+  if( outputMetaStore()->contains<xAOD::CutBookkeeperContainer>(m_cutflowCollName) ) {
+    if( !(outputMetaStore()->retrieve( fileCompleteBook, m_cutflowCollName) ).isSuccess() ) {
+      ATH_MSG_WARNING( "Could not get CutFlowSvc CutBookkeepers from output MetaDataStore" );
+    }
+    else {
+      // update the complete output with the complete input
+      ATH_CHECK(this->updateContainer(completeBook,fileCompleteBook));
+    }
+  }
+  else {
+    ATH_MSG_INFO("No cutflow container " << m_cutflowCollName);
+  }
+
+  return StatusCode::SUCCESS;
+}
+
+
 namespace {
 
+
 xAOD::CutBookkeeper*
 resolveLink (const xAOD::CutBookkeeper* old,
              xAOD::CutBookkeeperContainer& contToUpdate,
@@ -111,9 +324,6 @@ BookkeeperTool::updateContainer( xAOD::CutBookkeeperContainer* contToUpdate,
         foundEBKToUpdate = true;
         break;
       }
-      else {
-        ATH_MSG_INFO("otherEBK("<<otherEBK->name()<<") != ebkToUpdate("<<ebkToUpdate->name());
-      }
     } // End: Inner loop over contToUpdate
     if (!foundEBKToUpdate) {
       xAOD::CutBookkeeper* newEBK = new xAOD::CutBookkeeper();
@@ -175,3 +385,34 @@ BookkeeperTool::updateContainer( xAOD::CutBookkeeperContainer* contToUpdate,
   return StatusCode::SUCCESS;
 }
 
+
+StatusCode BookkeeperTool::copyContainerToOutput(const std::string& outname)
+{
+
+  // Get the complete bookkeeper collection of the output meta-data store
+  xAOD::CutBookkeeperContainer* contBook(nullptr); 
+  if( !(outputMetaStore()->retrieve( contBook, outname) ).isSuccess() ) {
+    ATH_MSG_ERROR( "Could not get " << outname << " CutBookkeepers from output MetaDataStore" );
+    return StatusCode::FAILURE;
+  }
+
+  // Get the tmp bookkeeper from the input
+  const xAOD::CutBookkeeperContainer* tmpBook(NULL);
+  if ( outputMetaStore()->contains<xAOD::CutBookkeeperContainer>(outname+"tmp") ) {
+    if( !(outputMetaStore()->retrieve( tmpBook, outname+"tmp") ).isSuccess() ) {
+      ATH_MSG_WARNING( "Could not get tmp CutBookkeepers from output MetaDataStore" );
+    }
+    else {
+      // update the complete output with the complete input
+      ATH_CHECK(this->updateContainer(contBook,tmpBook));
+      // remove the tmp container
+      const SG::IConstAuxStore* tmpBookAux = tmpBook->getConstStore();
+      ATH_CHECK(outputMetaStore()->removeDataAndProxy(tmpBook));
+      ATH_CHECK(outputMetaStore()->removeDataAndProxy(tmpBookAux));
+    }
+  }
+    
+  return StatusCode::SUCCESS;
+}
+
+

Event/EventBookkeeperTools/python/CutFlowHelpers.py

@@ -44,7 +44,6 @@ def GetCurrentStreamName( msg, athFile=None ):
 
 
 def CreateCutFlowSvc( svcName="CutFlowSvc", athFile=None, seq=None, addAlgInPlace=False, addMetaDataToAllOutputFiles=True, SGkey="CutBookkeepers" ):
-    print "CreateCutFlowSvc"
     """
     Helper to create the CutFlowSvc, extract the needed information from
     the input file, and also schedule all the needed stuff.
@@ -64,7 +63,8 @@ def CreateCutFlowSvc( svcName="CutFlowSvc", athFile=None, seq=None, addAlgInPlac
     import AthenaCommon.CfgMgr as CfgMgr
     if not hasattr(svcMgr,"CutFlowSvc"): svcMgr += CfgMgr.CutFlowSvc()
     svcMgr.CutFlowSvc.InputStream   = inputStreamName
-
+    #if not hasattr(svcMgr,"FileCutFlowSvc"): svcMgr += CfgMgr.FileCutFlowSvc()
+    #svcMgr.FileCutFlowSvc.InputStream   = inputStreamName
 
     # Make sure MetaDataSvc is ready
     if not hasattr(svcMgr,'MetaDataSvc'):
@@ -79,19 +79,18 @@ def CreateCutFlowSvc( svcName="CutFlowSvc", athFile=None, seq=None, addAlgInPlac
     outname = "CutBookkeepers"
     cutflowtool = BookkeeperTool(outname+"Tool",
                                  InputCollName = inname,
-                                 OutputCollName= outname,
-                                 ProcessMetadataCollName=inname+"File") 
+                                 OutputCollName= outname) 
     svcMgr.ToolSvc += cutflowtool
 
-
     # Add tool to MetaDataSvc
     svcMgr.MetaDataSvc.MetaDataTools += [cutflowtool]
-    for x in svcMgr.MetaDataSvc.MetaDataTools:
-        print x
 
     # Add pdf sum of weights counts if appropriate
     from AthenaCommon.GlobalFlags  import globalflags
     if globalflags.DataSource() == 'geant4':
+        #from PyUtils import AthFile
+        #afc = AthFile.fopen( svcMgr.EventSelector.InputCollections[0] )
+
         # PDF
         name = "PDFSumOfWeights"
         pdfweighttool = BookkeeperTool(name,
@@ -143,8 +142,7 @@ def CreateCutFlowSvc( svcName="CutFlowSvc", athFile=None, seq=None, addAlgInPlac
         MSMgr.AddMetaDataItemToAllStreams( "xAOD::CutBookkeeperAuxContainer#"+SGkey+"Aux.*" )
         MSMgr.AddMetaDataItemToAllStreams( "xAOD::CutBookkeeperContainer#Incomplete"+SGkey )
         MSMgr.AddMetaDataItemToAllStreams( "xAOD::CutBookkeeperAuxContainer#Incomplete"+SGkey+"Aux.*" )
-        #SGkey = "FileBookkeepers"
-        SGkey = "CutBookkeepersFile"
+        SGkey = "FileBookkeepers"
         MSMgr.AddMetaDataItemToAllStreams( "xAOD::CutBookkeeperContainer#"+SGkey )
         MSMgr.AddMetaDataItemToAllStreams( "xAOD::CutBookkeeperAuxContainer#"+SGkey+"Aux.*" )
         MSMgr.AddMetaDataItemToAllStreams( "xAOD::CutBookkeeperContainer#Incomplete"+SGkey )
@@ -172,6 +170,7 @@ def CreateBookkeeperTool( name="CutBookkeepers" ):
   svcMgr.ToolSvc += cutflowtool
 
   # Add tool to MetaDataSvc
-  svcMgr.MetaDataSvc.MetaDataTools += [cutflowtool]
+  #svcMgr.MetaDataSvc.MetaDataTools += [cutflowtool]
 
   return
+