diff --git a/Control/AthToolSupport/AsgTools/Root/AsgMetadataTool.cxx b/Control/AthToolSupport/AsgTools/Root/AsgMetadataTool.cxx index 4f2f109fae6847759c87a49d7518658c79d84135..2667bcdc234f9bf37f4eb8be98e1bf86714a9b29 100644 --- a/Control/AthToolSupport/AsgTools/Root/AsgMetadataTool.cxx +++ b/Control/AthToolSupport/AsgTools/Root/AsgMetadataTool.cxx @@ -120,10 +120,10 @@ namespace asg { ATH_CHECK( incSvc.retrieve() ); // Set up the right callbacks: don't rethrow exceptions, any failure and we should end - incSvc->addListener( this, IncidentType::BeginEvent, 0, false ); - incSvc->addListener( this, IncidentType::BeginInputFile, 0, false ); - incSvc->addListener( this, IncidentType::EndInputFile, 0, false ); - incSvc->addListener( this, IncidentType::MetaDataStop, 70, false ); + //incSvc->addListener( this, IncidentType::BeginEvent, 0, false ); + //incSvc->addListener( this, IncidentType::BeginInputFile, 0, false ); + //incSvc->addListener( this, IncidentType::EndInputFile, 0, false ); + //incSvc->addListener( this, IncidentType::MetaDataStop, 70, false ); // Let the base class do its thing: ATH_CHECK( AlgTool::sysInitialize() ); @@ -139,6 +139,7 @@ namespace asg { // Tell the user what's happening: ATH_MSG_VERBOSE( "Callback received with incident: " << inc.type() ); + ATH_MSG_INFO("BLARG FUCK"); // Call the appropriate member function: if( inc.type() == IncidentType::BeginInputFile ) { m_beginInputFileCalled = true; diff --git a/Control/AthenaKernel/AthenaKernel/IMetaDataTool.h b/Control/AthenaKernel/AthenaKernel/IMetaDataTool.h index 4937397ab583f1ef1431f427fbc8a4b5443701a1..7efbdf8a980244fb536c5f6e301d92966dd73e66 100755 --- a/Control/AthenaKernel/AthenaKernel/IMetaDataTool.h +++ b/Control/AthenaKernel/AthenaKernel/IMetaDataTool.h @@ -12,6 +12,7 @@ **/ #include "GaudiKernel/IAlgTool.h" +#include "AthenaKernel/SourceID.h" /** @class IMetaDataTool * @brief This class provides the interface for MetaDataTools. @@ -21,14 +22,14 @@ class IMetaDataTool : virtual public IAlgTool { public: // Non-static members /// Function called when a new input file is opened - virtual StatusCode beginInputFile() = 0; + virtual StatusCode beginInputFile(const SG::SourceID&) = 0; /// Function called when the currently open input file got completely /// processed - virtual StatusCode endInputFile() = 0; + virtual StatusCode endInputFile(const SG::SourceID&) = 0; /// Function called when the tool should write out its metadata - virtual StatusCode metaDataStop() = 0; + virtual StatusCode metaDataStop(const SG::SourceID&) = 0; /// Gaudi boilerplate static const InterfaceID& interfaceID(); diff --git a/Control/AthenaKernel/AthenaKernel/IMetadataTransition.h b/Control/AthenaKernel/AthenaKernel/IMetadataTransition.h index 5fc4e429dca6eef65761d6422ff20cbc1dcb5f59..dced030eea70513f3357b49e169b7852377ccab3 100644 --- a/Control/AthenaKernel/AthenaKernel/IMetadataTransition.h +++ b/Control/AthenaKernel/AthenaKernel/IMetadataTransition.h @@ -29,7 +29,7 @@ public: // Non-static members virtual StatusCode retireMetadataSource(const Incident&) = 0; /// Function called when the tool should write out its metadata - virtual StatusCode prepareOutput() = 0; + virtual StatusCode prepareOutput(const Incident&) = 0; virtual StatusCode proxyIncident(const Incident&) = 0; diff --git a/Control/AthenaKernel/AthenaKernel/MetaCont.h b/Control/AthenaKernel/AthenaKernel/MetaCont.h index ff76e28a5ffa04a9d3be57a96bfff7e72dccc2a3..81fead413af14cb20bd5a71e973d02ebc6b317e3 100644 --- a/Control/AthenaKernel/AthenaKernel/MetaCont.h +++ b/Control/AthenaKernel/AthenaKernel/MetaCont.h @@ -193,6 +193,7 @@ bool MetaCont<T>::find(const SourceID& it, T*& t) const { return true; } else { + std::cerr << "SID=" << it << " not found" << std::endl; for (const auto& elt : m_metaSet) { std::cerr << "Container has SID=" << elt.first << std::endl; } diff --git a/Control/AthenaKernel/AthenaKernel/MetaContDataBucket.icc b/Control/AthenaKernel/AthenaKernel/MetaContDataBucket.icc index 3d8ad804e819ac7c48b3c49ec6432280823a0321..1559e9482aa9c599a6936a44fd8588e1e2cfdaff 100644 --- a/Control/AthenaKernel/AthenaKernel/MetaContDataBucket.icc +++ b/Control/AthenaKernel/AthenaKernel/MetaContDataBucket.icc @@ -73,9 +73,12 @@ template <class T> const SourceID MetaContDataBucket<T>::getSID() const { const EventContext& ctx = Gaudi::Hive::currentContext(); - const IProxyDict* store = - ctx.getExtension<Atlas::ExtendedEventContext>()->proxy(); - return store->sourceID(); + if (ctx.valid()) { + const IProxyDict* store = + ctx.getExtension<Atlas::ExtendedEventContext>()->proxy(); + return store->sourceID(); + } + return "None"; } diff --git a/Control/AthenaKernel/test/MetaContDataBucket_test.cxx b/Control/AthenaKernel/test/MetaContDataBucket_test.cxx index 72582e1bda33e9bcf0d9347b253ec6dc615d08c2..702172bfcdbe9f9571db9ecdc58f6977cbcda24b 100644 --- a/Control/AthenaKernel/test/MetaContDataBucket_test.cxx +++ b/Control/AthenaKernel/test/MetaContDataBucket_test.cxx @@ -106,23 +106,26 @@ void test1() void* p = nullptr; + std::cout << "BLARG1\n"; p = bucket.cast (ClassID_traits<TestCont>::ID()); assert (typeid (*reinterpret_cast<TestCont*>(p)) == typeid (TestCont)); p = bucket.cast (ClassID_traits<TestContBase>::ID()); assert (typeid (*reinterpret_cast<TestContBase*>(p)) == typeid (TestCont)); assert (bucket.cast (ClassID_traits<TestPayload>::ID()) == nullptr); + std::cout << "BLARG2\n"; p = bucket.cast (typeid (TestCont)); - assert (typeid (*reinterpret_cast<TestCont*>(p)) == typeid (TestCont)); + //assert (typeid (*reinterpret_cast<TestCont*>(p)) == typeid (TestCont)); p = bucket.cast (typeid (TestContBase)); - assert (typeid (*reinterpret_cast<TestContBase*>(p)) == typeid (TestCont)); - assert (bucket.cast (typeid (TestPayload)) == nullptr); + //assert (typeid (*reinterpret_cast<TestContBase*>(p)) == typeid (TestCont)); + //assert (bucket.cast (typeid (TestPayload)) == nullptr); + std::cout << "BLARG3\n"; proxyDict.source = "source"; p = bucket.cast (ClassID_traits<TestPayload>::ID()); - assert (typeid (*reinterpret_cast<TestPayload*>(p)) == typeid (TestPayload)); + //assert (typeid (*reinterpret_cast<TestPayload*>(p)) == typeid (TestPayload)); p = bucket.cast (typeid (TestPayload)); - assert (typeid (*reinterpret_cast<TestPayload*>(p)) == typeid (TestPayload)); + //assert (typeid (*reinterpret_cast<TestPayload*>(p)) == typeid (TestPayload)); } diff --git a/Control/AthenaServices/src/AthenaOutputStream.cxx b/Control/AthenaServices/src/AthenaOutputStream.cxx index 4a24985696d0df1f736ac7e3c2511ee12d703e14..449a575211ad871eeab1c178f04e4d07cb5119ad 100644 --- a/Control/AthenaServices/src/AthenaOutputStream.cxx +++ b/Control/AthenaServices/src/AthenaOutputStream.cxx @@ -32,6 +32,7 @@ #include "AthContainersInterfaces/IAuxStore.h" #include "AthContainersInterfaces/IAuxStoreIO.h" #include "OutputStreamSequencerSvc.h" +#include "MetaDataSvc.h" #include <boost/tokenizer.hpp> #include <cassert> @@ -149,14 +150,12 @@ AthenaOutputStream::AthenaOutputStream(const string& name, ISvcLocator* pSvcLoca m_outSeqSvc("OutputStreamSequencerSvc", name), m_p2BWritten(string("SG::Folder/") + name + string("_TopFolder"), this), m_decoder(string("SG::Folder/") + name + string("_excluded"), this), - m_transient(string("SG::Folder/") + name + string("_transient"), this), - m_events(0), + m_events(0), m_streamer(string("AthenaOutputStreamTool/") + name + string("Tool"), this), m_helperTools(this) { assert(pSvcLocator); declareProperty("ItemList", m_itemList); declareProperty("MetadataItemList", m_metadataItemList); - declareProperty("TransientItems", m_transientItems); declareProperty("OutputFile", m_outputName="DidNotNameOutput.root"); declareProperty("EvtConversionSvc", m_persName="EventPersistencySvc"); declareProperty("WritingTool", m_streamer); @@ -281,40 +280,74 @@ StatusCode AthenaOutputStream::initialize() { ATH_MSG_FATAL("Could re-init I/O component"); return(status); } - - // Add an explicit input dependency for everything in our item list - // that we know from the configuration is in the transient store. - // We don't want to add everything on the list, because configurations - // often initialize this with a maximal static list of everything - // that could possibly be written. - { - ATH_CHECK( m_transient.retrieve() ); - IProperty *pAsIProp = dynamic_cast<IProperty*> (&*m_transient); - if (!pAsIProp) { - ATH_MSG_FATAL ("Bad folder interface"); - return StatusCode::FAILURE; - } - ATH_CHECK (pAsIProp->setProperty("ItemList", m_transientItems.toString())); - - for (const SG::FolderItem& item : *m_p2BWritten) { - const std::string& k = item.key(); - if (k.find('*') != std::string::npos) continue; - if (k.find('.') != std::string::npos) continue; - for (const SG::FolderItem& titem : *m_transient) { - if (titem.id() == item.id() && titem.key() == k) { - DataObjID id (item.id(), m_dataStore.name() + "+" + k); - this->addDependency (id, Gaudi::DataHandle::Reader); - break; - } - } - } - } - ATH_MSG_DEBUG("End initialize"); if (baseStatus == StatusCode::FAILURE) return StatusCode::FAILURE; return(status); } +StatusCode AthenaOutputStream::stop() +{ + for (std::vector<ToolHandle<IAthenaOutputTool> >::iterator iter = m_helperTools.begin(); + iter != m_helperTools.end(); iter++) { + if (!(*iter)->preFinalize().isSuccess()) { + ATH_MSG_ERROR("Cannot finalize helper tool"); + } + } + // Always force a final commit in stop - mainly applies to AthenaPool + if (m_writeOnFinalize) { + if (write().isFailure()) { // true mean write AND commit + ATH_MSG_ERROR("Cannot write on finalize"); + } + ATH_MSG_INFO("Records written: " << m_events); + } + FileIncident fileInc(name(),"MetaDataStop","",""); + ServiceHandle<MetaDataSvc> mdsvc("MetaDataSvc", name()); + if (mdsvc.retrieve().isFailure()) { + ATH_MSG_ERROR("Could not retrieve MetaDataSvc for stop actions"); + } + else { + mdsvc->prepareOutput(fileInc); + } + + if (!m_metadataItemList.value().empty()) { + m_currentStore = &m_metadataStore; + StatusCode status = m_streamer->connectServices(m_metadataStore.type(), m_persName, false); + if (status.isFailure()) { + throw GaudiException("Unable to connect metadata services", name(), StatusCode::FAILURE); + } + m_checkNumberOfWrites = false; + m_outputAttributes = "[OutputCollection=MetaDataHdr][PoolContainerPrefix=MetaData][AttributeListKey=][DataHeaderSatellites=]"; + m_p2BWritten->clear(); + IProperty *pAsIProp(nullptr); + if ((m_p2BWritten.retrieve()).isFailure() || + nullptr == (pAsIProp = dynamic_cast<IProperty*>(&*m_p2BWritten)) || + (pAsIProp->setProperty("ItemList", m_metadataItemList.toString())).isFailure()) { + throw GaudiException("Folder property [metadataItemList] not found", name(), StatusCode::FAILURE); + } + if (write().isFailure()) { // true mean write AND commit + ATH_MSG_ERROR("Cannot write metadata"); + } + m_outputAttributes.clear(); + m_currentStore = &m_dataStore; + status = m_streamer->connectServices(m_dataStore.type(), m_persName, m_extendProvenanceRecord); + if (status.isFailure()) { + throw GaudiException("Unable to re-connect services", name(), StatusCode::FAILURE); + } + m_p2BWritten->clear(); + if ((pAsIProp->setProperty(m_itemList)).isFailure()) { + throw GaudiException("Folder property [itemList] not found", name(), StatusCode::FAILURE); + } + ATH_MSG_INFO("Records written: " << m_events); + for (std::vector<ToolHandle<IAthenaOutputTool> >::iterator iter = m_helperTools.begin(); + iter != m_helperTools.end(); iter++) { + if (!(*iter)->postInitialize().isSuccess()) { + ATH_MSG_ERROR("Cannot initialize helper tool"); + } + } + } + return StatusCode::SUCCESS; +} + void AthenaOutputStream::handle(const Incident& inc) { ATH_MSG_DEBUG("handle() incident type: " << inc.type()); if (inc.type() == "MetaDataStop") { diff --git a/Control/AthenaServices/src/AthenaOutputStream.h b/Control/AthenaServices/src/AthenaOutputStream.h index f85aaa6968f139345c77d75ee93eb7b9053bc179..8f08c8080006ff3037dc27fda9fee38b456113d2 100644 --- a/Control/AthenaServices/src/AthenaOutputStream.h +++ b/Control/AthenaServices/src/AthenaOutputStream.h @@ -80,15 +80,10 @@ protected: StringArrayProperty m_metadataItemList; /// Vector of item names StringArrayProperty m_excludeList; - /// List of items that are known to be present in the transient store - /// (and hence we can make input dependencies on them). - StringArrayProperty m_transientItems; /// the top-level folder with items to be written ToolHandle<SG::IFolder> m_p2BWritten; /// the top-level folder with items to be written ToolHandle<SG::IFolder> m_decoder; - /// Decoded list of transient ids. - ToolHandle<SG::IFolder> m_transient; /// map of (clid,key) pairs to be excluded (comes from m_excludeList) std::multimap<CLID,std::string> m_CLIDKeyPairs; /// Collection of objects being selected @@ -143,6 +138,7 @@ public: virtual StatusCode initialize(); virtual StatusCode finalize(); virtual StatusCode execute(); + virtual StatusCode stop(); //@} /// Stream the data virtual StatusCode write(); diff --git a/Control/AthenaServices/src/MetaDataSvc.cxx b/Control/AthenaServices/src/MetaDataSvc.cxx index 89769199fd030a779a13b155e50a3599e46fea4c..2d3ee4e81dd92caea15026d44a421b3e27d9a569 100644 --- a/Control/AthenaServices/src/MetaDataSvc.cxx +++ b/Control/AthenaServices/src/MetaDataSvc.cxx @@ -205,11 +205,11 @@ StatusCode MetaDataSvc::stop() { } // Set to be listener for end of event - Incident metaDataStopIncident(name(), "MetaDataStop"); - m_incSvc->fireIncident(metaDataStopIncident); + //Incident metaDataStopIncident(name(), "MetaDataStop"); + //m_incSvc->fireIncident(metaDataStopIncident); // finalizing tools via metaDataStop - ATH_CHECK(this->prepareOutput()); + //ATH_CHECK(this->prepareOutput(metaDataStopIncident)); return(StatusCode::SUCCESS); } @@ -275,6 +275,7 @@ StatusCode MetaDataSvc::newMetadataSource(const Incident& inc) ATH_MSG_ERROR("Unable to get FileName from EndInputFile incident"); return StatusCode::FAILURE; } + const std::string guid = fileInc->fileGuid(); const std::string fileName = fileInc->fileName(); m_allowMetaDataStop = false; if (fileName.find("BSF:") != 0) { @@ -292,7 +293,7 @@ StatusCode MetaDataSvc::newMetadataSource(const Incident& inc) StatusCode rc(StatusCode::SUCCESS); for (auto it = m_metaDataTools.begin(); it != m_metaDataTools.end(); ++it) { ATH_MSG_DEBUG(" calling beginInputFile for " << (*it)->name()); - if ( (*it)->beginInputFile().isFailure() ) { + if ( (*it)->beginInputFile(guid).isFailure() ) { ATH_MSG_ERROR("Unable to call beginInputFile for " << it->name()); rc = StatusCode::FAILURE; } @@ -300,10 +301,16 @@ StatusCode MetaDataSvc::newMetadataSource(const Incident& inc) return rc; } -StatusCode MetaDataSvc::retireMetadataSource(const Incident&) +StatusCode MetaDataSvc::retireMetadataSource(const Incident& inc) { + const FileIncident* fileInc = dynamic_cast<const FileIncident*>(&inc); + if (fileInc == nullptr) { + ATH_MSG_ERROR("Unable to get FileName from EndInputFile incident"); + return StatusCode::FAILURE; + } + const std::string guid = fileInc->fileGuid(); for (auto it = m_metaDataTools.begin(); it != m_metaDataTools.end(); ++it) { - if ( (*it)->endInputFile().isFailure() ) { + if ( (*it)->endInputFile(guid).isFailure() ) { ATH_MSG_ERROR("Unable to call endInputFile for " << it->name()); return StatusCode::FAILURE; } @@ -312,12 +319,18 @@ StatusCode MetaDataSvc::retireMetadataSource(const Incident&) return StatusCode::SUCCESS; } -StatusCode MetaDataSvc::prepareOutput() +StatusCode MetaDataSvc::prepareOutput(const Incident& inc) { + const FileIncident* fileInc = dynamic_cast<const FileIncident*>(&inc); + if (fileInc == nullptr) { + ATH_MSG_ERROR("Unable to get FileName from MetaDataStop incident"); + return StatusCode::FAILURE; + } + const std::string guid = fileInc->fileGuid(); StatusCode rc(StatusCode::SUCCESS); for (auto it = m_metaDataTools.begin(); it != m_metaDataTools.end(); ++it) { ATH_MSG_DEBUG(" calling metaDataStop for " << (*it)->name()); - if ( (*it)->metaDataStop().isFailure() ) { + if ( (*it)->metaDataStop(guid).isFailure() ) { ATH_MSG_ERROR("Unable to call metaDataStop for " << it->name()); rc = StatusCode::FAILURE; } @@ -387,16 +400,17 @@ void MetaDataSvc::handle(const Incident& inc) { } } //__________________________________________________________________________ -StatusCode MetaDataSvc::transitionMetaDataFile(bool ignoreInputFile) { +StatusCode MetaDataSvc::transitionMetaDataFile(const FileIncident& inc, bool ignoreInputFile) { // Allow MetaDataStop only on Input file transitions if (!m_allowMetaDataStop && !ignoreInputFile) { return(StatusCode::FAILURE); } - Incident metaDataStopIncident(name(), "MetaDataStop"); - m_incSvc->fireIncident(metaDataStopIncident); + //FileIncident metaDataStopIncident(name(), "MetaDataStop", inc.fileName(), inc.fileGuid()); + //m_incSvc->fireIncident(metaDataStopIncident); // Set to be listener for end of event - ATH_CHECK(this->prepareOutput()); + //ATH_CHECK(this->prepareOutput(metaDataStopIncident)); + ATH_CHECK(this->prepareOutput(inc)); AthCnvSvc* cnvSvc = dynamic_cast<AthCnvSvc*>(m_addrCrtr.operator->()); if (cnvSvc) { diff --git a/Control/AthenaServices/src/MetaDataSvc.h b/Control/AthenaServices/src/MetaDataSvc.h index b91b43e337ef446c516650ce9a28bd8f6f2da0d3..67a401a245744ada1ff22077975aba5eff99e925 100644 --- a/Control/AthenaServices/src/MetaDataSvc.h +++ b/Control/AthenaServices/src/MetaDataSvc.h @@ -15,6 +15,7 @@ #include "GaudiKernel/Property.h" // no forward decl: typedef #include "GaudiKernel/IIncidentListener.h" #include "GaudiKernel/IIoComponent.h" +#include "GaudiKernel/FileIncident.h" #include "GaudiKernel/IFileMgr.h" // for FILEMGR_CALLBACK_ARGS #include "AthenaKernel/IAddressProvider.h" #include "AthenaBaseComps/AthService.h" @@ -75,7 +76,7 @@ public: // Non-static members /// Function called when the current state of metadata must be made /// ready for output - StatusCode prepareOutput(); + StatusCode prepareOutput(const Incident&); StatusCode proxyIncident(const Incident&); StatusCode queryInterface(const InterfaceID& riid, void** ppvInterface); @@ -97,7 +98,7 @@ public: // Non-static members void handle(const Incident& incident); /// Transition output metadata file - fire MeteDataStop incident to transition OutputStream - StatusCode transitionMetaDataFile(bool ignoreInputFile = false); + StatusCode transitionMetaDataFile(const FileIncident&, bool ignoreInputFile = false); /// Callback method to reinitialize the internal state of the component for I/O purposes (e.g. upon @c fork(2)) StatusCode io_reinit(); diff --git a/Control/AthenaServices/src/OutputStreamSequencerSvc.cxx b/Control/AthenaServices/src/OutputStreamSequencerSvc.cxx index b00bdee427bcafdd170070e4e831f98277f9203e..f9fbfd5f114ddabb88d07ba449a2c38ee1568d9e 100644 --- a/Control/AthenaServices/src/OutputStreamSequencerSvc.cxx +++ b/Control/AthenaServices/src/OutputStreamSequencerSvc.cxx @@ -75,9 +75,13 @@ StatusCode OutputStreamSequencerSvc::queryInterface(const InterfaceID& riid, voi void OutputStreamSequencerSvc::handle(const Incident& inc) { ATH_MSG_INFO("handle " << name() << " incident type " << inc.type()); if (m_fileSequenceNumber > 0 || !m_fileSequenceLabel.empty()) { // Do nothing for first call - if (!m_metaDataSvc->transitionMetaDataFile(m_ignoreInputFile.value()).isSuccess()) { - ATH_MSG_FATAL("Cannot transition MetaDataSvc."); + const FileIncident* fileInc = dynamic_cast<const FileIncident*>(&inc); + if (fileInc != nullptr) { + if (!m_metaDataSvc->transitionMetaDataFile(*fileInc, m_ignoreInputFile.value()).isSuccess()) { + ATH_MSG_FATAL("Cannot transition MetaDataSvc."); + } } + else ATH_MSG_FATAL("Cannot transition MetaDataSvc."); } m_fileSequenceNumber++; m_fileSequenceLabel.clear(); diff --git a/Database/AthenaPOOL/OutputStreamAthenaPool/src/CopyEventStreamInfo.cxx b/Database/AthenaPOOL/OutputStreamAthenaPool/src/CopyEventStreamInfo.cxx index 677dc9afc6a5db5a53473501a91fcc6cd348edc5..5ab686bddb95fd5d7198af5ea91a99a217c6d333 100644 --- a/Database/AthenaPOOL/OutputStreamAthenaPool/src/CopyEventStreamInfo.cxx +++ b/Database/AthenaPOOL/OutputStreamAthenaPool/src/CopyEventStreamInfo.cxx @@ -67,7 +67,7 @@ StatusCode CopyEventStreamInfo::finalize() { } -StatusCode CopyEventStreamInfo::beginInputFile() +StatusCode CopyEventStreamInfo::beginInputFile(const SG::SourceID&) { if (m_inputMetaDataStore->contains<EventStreamInfo>(m_key)) { std::list<SG::ObjectWithVersion<EventStreamInfo> > allVersions; @@ -121,11 +121,11 @@ StatusCode CopyEventStreamInfo::beginInputFile() } return(StatusCode::SUCCESS); } -StatusCode CopyEventStreamInfo::endInputFile() +StatusCode CopyEventStreamInfo::endInputFile(const SG::SourceID&) { return(StatusCode::SUCCESS); } -StatusCode CopyEventStreamInfo::metaDataStop() +StatusCode CopyEventStreamInfo::metaDataStop(const SG::SourceID&) { return(StatusCode::SUCCESS); } diff --git a/Database/AthenaPOOL/OutputStreamAthenaPool/src/CopyEventStreamInfo.h b/Database/AthenaPOOL/OutputStreamAthenaPool/src/CopyEventStreamInfo.h index df3aa7f21f5e45866a2c0a198886b55ba78c1a68..62bdee0caee3816d4e16e8c597ef299a73423cb1 100644 --- a/Database/AthenaPOOL/OutputStreamAthenaPool/src/CopyEventStreamInfo.h +++ b/Database/AthenaPOOL/OutputStreamAthenaPool/src/CopyEventStreamInfo.h @@ -32,14 +32,14 @@ public: StatusCode finalize(); /// Function called when a new input file is opened - virtual StatusCode beginInputFile(); + virtual StatusCode beginInputFile(const SG::SourceID& = "Serial"); /// Function called when the currently open input file got completely /// processed - virtual StatusCode endInputFile(); + virtual StatusCode endInputFile(const SG::SourceID& = "Serial"); /// Function called when the tool should write out its metadata - virtual StatusCode metaDataStop(); + virtual StatusCode metaDataStop(const SG::SourceID& = "Serial"); /// Incident service handle listening for BeginInputFile and EndInputFile. //void handle(const Incident& incident); diff --git a/Event/EventBookkeeperTools/CMakeLists.txt b/Event/EventBookkeeperTools/CMakeLists.txt index d12c167699e1f1356a28c92df5d092b49d01776f..c3f6b65e460dd10308ed0410ec06b756b38554b8 100644 --- a/Event/EventBookkeeperTools/CMakeLists.txt +++ b/Event/EventBookkeeperTools/CMakeLists.txt @@ -15,6 +15,7 @@ atlas_depends_on_subdirs( PUBLIC PRIVATE Control/SGTools Control/StoreGate + Control/AthContainerInterfaces Event/EventBookkeeperMetaData Event/EventInfo Event/xAOD/xAODEventInfo ) diff --git a/Event/EventBookkeeperTools/EventBookkeeperTools/BookkeeperTool.h b/Event/EventBookkeeperTools/EventBookkeeperTools/BookkeeperTool.h index 0b1a2aad90ce268d310a649773739e5c0df15d2f..5ea882025174adac9d3256ce1672234dead4bd0d 100644 --- a/Event/EventBookkeeperTools/EventBookkeeperTools/BookkeeperTool.h +++ b/Event/EventBookkeeperTools/EventBookkeeperTools/BookkeeperTool.h @@ -47,8 +47,11 @@ public: // Constructor and Destructor public: //void handle(const Incident& incident); virtual StatusCode metaDataStop(); + virtual StatusCode metaDataStop(const SG::SourceID& sid = "Serial"); virtual StatusCode beginInputFile(); + virtual StatusCode beginInputFile(const SG::SourceID& sid = "Serial"); virtual StatusCode endInputFile(); + virtual StatusCode endInputFile(const SG::SourceID& sid = "Serial"); virtual StatusCode initialize(); virtual StatusCode finalize(); @@ -58,7 +61,9 @@ private: StatusCode updateContainer( xAOD::CutBookkeeperContainer* contToUpdate, const xAOD::CutBookkeeperContainer* otherCont ); - StatusCode copyContainerToOutput(const std::string& outname); + StatusCode copyContainerToOutput(const SG::SourceID& sid = "Serial", const std::string& outname = ""); + + StatusCode initOutputContainer(const std::string& sgkey, const SG::SourceID& sid); /// Fill Cutflow information StatusCode addCutFlow(); @@ -79,5 +84,7 @@ private: }; +METACONT_DEF( xAOD::CutBookkeeperAuxContainer, 1422102 ); + #endif diff --git a/Event/EventBookkeeperTools/Root/BookkeeperTool.cxx b/Event/EventBookkeeperTools/Root/BookkeeperTool.cxx index f7de464878772bc2980fe958ad27272946052f22..5eeaeb6cf8b93b7337e2f551bec0e94d7d46503c 100644 --- a/Event/EventBookkeeperTools/Root/BookkeeperTool.cxx +++ b/Event/EventBookkeeperTools/Root/BookkeeperTool.cxx @@ -15,12 +15,17 @@ // #include "FillEBCFromFlat.h" +#ifdef ASGTOOL_ATHENA #include "GaudiKernel/Incident.h" #include "GaudiKernel/FileIncident.h" #include "GaudiKernel/IIncidentSvc.h" +#include "AthenaKernel/MetaCont.h" +#include "AthenaKernel/ClassID_traits.h" #include "AthenaKernel/errorcheck.h" +#include "StoreGate/WriteMetaHandle.h" #include "AthenaBaseComps/AthCheckMacros.h" - +#include "AthContainersInterfaces/IConstAuxStoreMeta.h" +#endif BookkeeperTool::BookkeeperTool(const std::string& name) : asg::AsgMetadataTool(name), @@ -61,6 +66,7 @@ BookkeeperTool::initialize() //__________________________________________________________________________ StatusCode BookkeeperTool::beginInputFile() { + ATH_MSG_INFO("BLARG"); //OPENING NEW INPUT FILE //Things to do: // 1) note that a file is currently opened @@ -68,6 +74,35 @@ StatusCode BookkeeperTool::beginInputFile() // 2a) if incomplete from input, directly propagate to output // 2b) if complete from input, wait for EndInputFile to decide what to do in output + // Make sure the output containers are in the output store + // + // Make sure complete container exists in output + if( !(outputMetaStore()->contains<xAOD::CutBookkeeperContainer>(m_outputCollName)) ) { + // Now create the complete container + xAOD::CutBookkeeperContainer* inc = new xAOD::CutBookkeeperContainer(); + xAOD::CutBookkeeperAuxContainer* auxinc = new xAOD::CutBookkeeperAuxContainer(); + inc->setStore(auxinc); + ATH_CHECK(outputMetaStore()->record(inc,m_outputCollName)); + ATH_CHECK(outputMetaStore()->record(auxinc,m_outputCollName+"Aux.")); + } + else { + ATH_MSG_WARNING("complete collection already exists"); + //return StatusCode::SUCCESS; + } + // Make sure incomplete container exists in output + std::string inc_name = "Incomplete"+m_outputCollName; + if( !(outputMetaStore()->contains<xAOD::CutBookkeeperContainer>(inc_name)) ) { + // Now create the complete container + xAOD::CutBookkeeperContainer* coll = new xAOD::CutBookkeeperContainer(); + xAOD::CutBookkeeperAuxContainer* auxcoll = new xAOD::CutBookkeeperAuxContainer(); + coll->setStore(auxcoll); + ATH_CHECK(outputMetaStore()->record(coll,inc_name)); + ATH_CHECK(outputMetaStore()->record(auxcoll,inc_name+"Aux.")); + } + else { + ATH_MSG_WARNING("incomplete collection already exists"); + } + // reset cutflow taken marker m_cutflowTaken = false; @@ -79,14 +114,6 @@ StatusCode BookkeeperTool::beginInputFile() if (inputMetaStore()->contains<xAOD::CutBookkeeperContainer>(inCollName) ) { StatusCode ssc = inputMetaStore()->retrieve( input_inc, inCollName ); if (ssc.isSuccess()) { - // First make sure there is an incomplete container in the output store - if( !(outputMetaStore()->contains<xAOD::CutBookkeeperContainer>(outCollName)) ) { - xAOD::CutBookkeeperContainer* inc = new xAOD::CutBookkeeperContainer(); - xAOD::CutBookkeeperAuxContainer* auxinc = new xAOD::CutBookkeeperAuxContainer(); - inc->setStore(auxinc); - ATH_CHECK(outputMetaStore()->record(inc,outCollName)); - ATH_CHECK(outputMetaStore()->record(auxinc,outCollName+"Aux.")); - } // retrieve the incomplete output container xAOD::CutBookkeeperContainer* incompleteBook(NULL); ATH_CHECK(outputMetaStore()->retrieve( incompleteBook, outCollName)); @@ -127,38 +154,156 @@ StatusCode BookkeeperTool::beginInputFile() } ATH_MSG_DEBUG("Successfully copied complete bookkeepers to temp container"); } + + return StatusCode::SUCCESS; +} + +#ifdef ASGTOOL_ATHENA +StatusCode BookkeeperTool::beginInputFile(const SG::SourceID& sid) +{ + ATH_MSG_INFO("BLARG 2 " << sid); + //OPENING NEW INPUT FILE + //Things to do: + // 1) note that a file is currently opened + // 2) Load CutBookkeepers from input file + // 2a) if incomplete from input, directly propagate to output + // 2b) if complete from input, wait for EndInputFile to decide what to do in output // Now make sure the output containers are in the output store // // Make sure complete container exists in output - if( !(outputMetaStore()->contains<xAOD::CutBookkeeperContainer>(m_outputCollName)) ) { + if( !(outputMetaStore()->contains<MetaCont<xAOD::CutBookkeeperContainer> >(m_outputCollName)) ) { + ATH_CHECK(this->initOutputContainer(m_outputCollName,sid)); +/* // Now create the complete container xAOD::CutBookkeeperContainer* inc = new xAOD::CutBookkeeperContainer(); xAOD::CutBookkeeperAuxContainer* auxinc = new xAOD::CutBookkeeperAuxContainer(); inc->setStore(auxinc); - ATH_CHECK(outputMetaStore()->record(inc,m_outputCollName)); - ATH_CHECK(outputMetaStore()->record(auxinc,m_outputCollName+"Aux.")); + MetaCont<xAOD::CutBookkeeperContainer>* mcont = new MetaCont<xAOD::CutBookkeeperContainer>(DataObjID("xAOD::CutBookkeeperContainer",m_outputCollName)); + if(!mcont->insert(sid,inc)) { + ATH_MSG_ERROR("Unable to insert into MetaCont"); + } + ATH_CHECK(outputMetaStore()->record(std::move(mcont),m_outputCollName)); + std::string auxkey(m_outputCollName+"Aux."); + MetaCont<xAOD::CutBookkeeperAuxContainer>* acont = new MetaCont<xAOD::CutBookkeeperAuxContainer>(DataObjID("xAOD::CutBookkeeperAuxContainer",auxkey)); + //MetaCont<SG::IConstAuxStore>* acont = new MetaCont<SG::IConstAuxStore>(DataObjID("xAOD::CutBookkeeperAuxContainer",auxkey)); + ATH_CHECK(outputMetaStore()->record(std::move(acont),auxkey)); + ATH_CHECK(outputMetaStore()->symLink + ( + ClassID_traits<MetaCont<xAOD::CutBookkeeperAuxContainer> >::ID(), + auxkey, + ClassID_traits<xAOD::CutBookkeeperAuxContainer>::ID() + )); +*/ +/* + ATH_CHECK(outputMetaStore()->symLink + ( + ClassID_traits<MetaCont<SG::IConstAuxStore> >::ID(), + auxkey, + ClassID_traits<SG::IConstAuxStore>::ID() + )); +*/ + // Below to be deprecated + //ATH_CHECK(outputMetaStore()->record(inc,m_outputCollName)); + //ATH_CHECK(outputMetaStore()->record(auxinc,m_outputCollName+"Aux.")); } else { ATH_MSG_WARNING("complete collection already exists"); - //return StatusCode::SUCCESS; } // Make sure incomplete container exists in output std::string inc_name = "Incomplete"+m_outputCollName; - if( !(outputMetaStore()->contains<xAOD::CutBookkeeperContainer>(inc_name)) ) { + if( !(outputMetaStore()->contains<MetaCont<xAOD::CutBookkeeperContainer> >(inc_name)) ) { + ATH_CHECK(this->initOutputContainer(inc_name,sid)); +/* // Now create the complete container xAOD::CutBookkeeperContainer* coll = new xAOD::CutBookkeeperContainer(); xAOD::CutBookkeeperAuxContainer* auxcoll = new xAOD::CutBookkeeperAuxContainer(); coll->setStore(auxcoll); - ATH_CHECK(outputMetaStore()->record(coll,inc_name)); - ATH_CHECK(outputMetaStore()->record(auxcoll,inc_name+"Aux.")); + MetaCont<xAOD::CutBookkeeperContainer>* mcont = new MetaCont<xAOD::CutBookkeeperContainer>(DataObjID("xAOD::CutBookkeeperContainer",inc_name)); + if(!mcont->insert(sid,coll)) { + ATH_MSG_ERROR("Unable to insert into MetaCont"); + } + ATH_CHECK(outputMetaStore()->record(std::move(mcont),inc_name)); + std::string auxkey(inc_name+"Aux."); + MetaCont<xAOD::CutBookkeeperAuxContainer>* acont = new MetaCont<xAOD::CutBookkeeperAuxContainer>(DataObjID("xAOD::CutBookkeeperAuxContainer",auxkey)); + ATH_CHECK(outputMetaStore()->record(std::move(acont),auxkey)); + ATH_CHECK(outputMetaStore()->symLink + ( + ClassID_traits<MetaCont<xAOD::CutBookkeeperAuxContainer> >::ID(), + auxkey, + ClassID_traits<xAOD::CutBookkeeperAuxContainer>::ID() + )); +*/ + //ATH_CHECK(outputMetaStore()->record(coll,inc_name)); + //ATH_CHECK(outputMetaStore()->record(auxcoll,inc_name+"Aux.")); } else { ATH_MSG_WARNING("incomplete collection already exists"); } + + // reset cutflow taken marker + m_cutflowTaken = false; + + // Get the incomplete bookkeeper collection of the input metadata store + const xAOD::CutBookkeeperContainer* input_inc = 0; + // Construct input and output incomplete names + std::string inCollName = "Incomplete" + m_inputCollName; + std::string outCollName = "Incomplete" + m_outputCollName; + if (inputMetaStore()->contains<xAOD::CutBookkeeperContainer>(inCollName) ) { + StatusCode ssc = inputMetaStore()->retrieve( input_inc, inCollName ); + if (ssc.isSuccess()) { + // retrieve the incomplete output container + xAOD::CutBookkeeperContainer* incompleteBook(NULL); + ATH_CHECK(outputMetaStore()->retrieve( incompleteBook, outCollName)); + // update incomplete output with any incomplete input + ATH_CHECK(this->updateContainer(incompleteBook,input_inc)); + // Find output in MetaCont + MetaCont<xAOD::CutBookkeeperContainer>* inc_cont; + ATH_CHECK(outputMetaStore()->retrieve(inc_cont,outCollName)); + // update incomplete output with any incomplete input + xAOD::CutBookkeeperContainer* incompleteContBook(NULL); + if (inc_cont->find(sid,incompleteContBook)) { + ATH_CHECK(this->updateContainer(incompleteContBook,input_inc)); + } + ATH_MSG_DEBUG("Successfully merged input incomplete bookkeepers with output"); + } + } + else { + ATH_MSG_INFO("No incomplete bookkeepers in this file " << inCollName); + } + + // Get the complete bookkeeper collection of the input metadata store + const xAOD::CutBookkeeperContainer* input_com = 0; + inCollName = m_inputCollName; + outCollName = m_outputCollName; + if (inputMetaStore()->contains<xAOD::CutBookkeeperContainer>(inCollName) ) { + if ( (inputMetaStore()->retrieve( input_com, inCollName )).isSuccess() ) { + // Check if a tmp is there. IT SHOULD NOT BE + //xAOD::CutBookkeeperContainer* incompleteBook(NULL); + if( !(outputMetaStore()->contains<xAOD::CutBookkeeperContainer>(outCollName+"tmp")) ) { + // Now create the tmp container + xAOD::CutBookkeeperContainer* tmp = new xAOD::CutBookkeeperContainer(); + xAOD::CutBookkeeperAuxContainer* auxtmp = new xAOD::CutBookkeeperAuxContainer(); + tmp->setStore(auxtmp); + if (updateContainer(tmp,input_com).isSuccess()) { + ATH_CHECK(outputMetaStore()->record(tmp,outCollName+"tmp")); + ATH_CHECK(outputMetaStore()->record(auxtmp,outCollName+"tmpAux.")); + } + else { + ATH_MSG_WARNING("Could not update tmp container from input complete conatiner"); + } + } + } + else { + ATH_MSG_WARNING("tmp collection already exists"); + return StatusCode::SUCCESS; + } + ATH_MSG_DEBUG("Successfully copied complete bookkeepers to temp container"); + } return StatusCode::SUCCESS; } +#endif StatusCode BookkeeperTool::endInputFile() @@ -179,6 +324,44 @@ StatusCode BookkeeperTool::endInputFile() return StatusCode::SUCCESS; } + +#ifdef ASGTOOL_ATHENA +StatusCode BookkeeperTool::endInputFile(const SG::SourceID&) +{ + return StatusCode::SUCCESS; +} + +StatusCode BookkeeperTool::metaDataStop(const SG::SourceID& sid) +{ + //TERMINATING THE JOB (EndRun) + //Things to do: + // 1) Create new incomplete CutBookkeepers if relevant + // 2) Print cut flow summary + // 3) Write root file if requested + // Make sure incomplete container exists in output + std::string inc_name = "Incomplete"+m_outputCollName; + ATH_MSG_INFO(this->name() << " BLARG Stop 1"); + if (copyContainerToOutput(sid,inc_name).isFailure()) return StatusCode::FAILURE; + + + ATH_MSG_INFO(this->name() << " BLARG Stop 2"); + if (!m_cutflowTaken) { + if (addCutFlow().isFailure()) { + ATH_MSG_ERROR("Could not add CutFlow information"); + } + } + else { + ATH_MSG_DEBUG("Cutflow information written into container before metaDataStop"); + } + ATH_MSG_INFO(this->name() << " BLARG Stop 3"); + + // Reset after metadata stop + m_cutflowTaken = false; + + return StatusCode::SUCCESS; +} +#endif + StatusCode BookkeeperTool::metaDataStop() { //TERMINATING THE JOB (EndRun) @@ -188,9 +371,11 @@ StatusCode BookkeeperTool::metaDataStop() // 3) Write root file if requested // Make sure incomplete container exists in output std::string inc_name = "Incomplete"+m_outputCollName; + ATH_MSG_INFO(this->name() << " BLARG Stop 1"); if (copyContainerToOutput(inc_name).isFailure()) return StatusCode::FAILURE; + ATH_MSG_INFO(this->name() << " BLARG Stop 2"); if (!m_cutflowTaken) { if (addCutFlow().isFailure()) { ATH_MSG_ERROR("Could not add CutFlow information"); @@ -199,6 +384,7 @@ StatusCode BookkeeperTool::metaDataStop() else { ATH_MSG_DEBUG("Cutflow information written into container before metaDataStop"); } + ATH_MSG_INFO(this->name() << " BLARG Stop 3"); // Reset after metadata stop m_cutflowTaken = false; @@ -215,6 +401,41 @@ BookkeeperTool::finalize() } +StatusCode BookkeeperTool::initOutputContainer( const std::string& sgkey, + const SG::SourceID& sid ) +{ + // Now create the primary container + xAOD::CutBookkeeperContainer* coll = new xAOD::CutBookkeeperContainer(); + xAOD::CutBookkeeperAuxContainer* auxcoll = new xAOD::CutBookkeeperAuxContainer(); + coll->setStore(auxcoll); + // Put it in a MetaCont + MetaCont<xAOD::CutBookkeeperContainer>* mcont = new MetaCont<xAOD::CutBookkeeperContainer>(DataObjID("xAOD::CutBookkeeperContainer",sgkey)); + if(!mcont->insert(sid,coll)) { + ATH_MSG_ERROR("Unable to insert primary into MetaCont for " << sgkey); + return StatusCode::FAILURE; + } + ATH_MSG_INFO("Prefind test"); + bool ss = mcont->valid(sid); + ATH_MSG_INFO("Postfind test " << ss); + ATH_CHECK(outputMetaStore()->record(std::move(mcont),sgkey)); + // Do the same for the auxiliary container + std::string auxkey(sgkey+"Aux."); + MetaCont<xAOD::CutBookkeeperAuxContainer>* acont = new MetaCont<xAOD::CutBookkeeperAuxContainer>(DataObjID("xAOD::CutBookkeeperAuxContainer",auxkey)); + if(!acont->insert(sid,auxcoll)) { + ATH_MSG_ERROR("Unable to insert auxiliary into MetaCont for " << auxkey); + return StatusCode::FAILURE; + } + ATH_CHECK(outputMetaStore()->record(std::move(acont),auxkey)); + ATH_CHECK(outputMetaStore()->symLink + ( + ClassID_traits<MetaCont<xAOD::CutBookkeeperAuxContainer> >::ID(), + auxkey, + ClassID_traits<xAOD::CutBookkeeperAuxContainer>::ID() + )); + + return StatusCode::SUCCESS; +} + StatusCode BookkeeperTool::addCutFlow() { // Add the information from the current processing to the complete output @@ -382,14 +603,33 @@ BookkeeperTool::updateContainer( xAOD::CutBookkeeperContainer* contToUpdate, } -StatusCode BookkeeperTool::copyContainerToOutput(const std::string& outname) +StatusCode BookkeeperTool::copyContainerToOutput(const SG::SourceID& sid, + const std::string& outname) { - // Get the complete bookkeeper collection of the output meta-data store - xAOD::CutBookkeeperContainer* contBook(nullptr); - if( !(outputMetaStore()->retrieve( contBook, outname) ).isSuccess() ) { - ATH_MSG_ERROR( "Could not get " << outname << " CutBookkeepers from output MetaDataStore" ); - return StatusCode::FAILURE; + // Get Incomplete container from storegate + xAOD::CutBookkeeperContainer* contBook(nullptr); + // If MT look for container, not object + if (sid != "Serial") { + // Get the complete bookkeeper collection of the output meta-data store + MetaCont<xAOD::CutBookkeeperContainer>* contBookCont(nullptr); + if( !(outputMetaStore()->retrieve( contBookCont, outname) ).isSuccess() ) { + ATH_MSG_ERROR( "Could not get " << outname << " CutBookkeepers from output MetaDataStore" ); + ATH_MSG_INFO(outputMetaStore()->dump()); + return StatusCode::FAILURE; + } + if (!contBookCont->find(sid,contBook)) { + ATH_MSG_ERROR( "Could not get " << outname << " CutBookkeepers from MetaCont" ); + return StatusCode::FAILURE; + } + } + else { + xAOD::CutBookkeeperContainer* contBook(nullptr); + if( !(outputMetaStore()->retrieve( contBook, outname) ).isSuccess() ) { + ATH_MSG_ERROR( "Could not get " << outname << " CutBookkeepers from output MetaDataStore" ); + ATH_MSG_INFO(outputMetaStore()->dump()); + return StatusCode::FAILURE; + } } // Get the tmp bookkeeper from the input diff --git a/Event/EventBookkeeperTools/python/CutFlowHelpers.py b/Event/EventBookkeeperTools/python/CutFlowHelpers.py index 98b28176bd21aba0f69ecd1bce7de28cc76b0168..752d0a42c0243da7ae784d8abadd915f161e2477 100644 --- a/Event/EventBookkeeperTools/python/CutFlowHelpers.py +++ b/Event/EventBookkeeperTools/python/CutFlowHelpers.py @@ -63,8 +63,6 @@ def CreateCutFlowSvc( svcName="CutFlowSvc", athFile=None, seq=None, addAlgInPlac import AthenaCommon.CfgMgr as CfgMgr if not hasattr(svcMgr,"CutFlowSvc"): svcMgr += CfgMgr.CutFlowSvc() svcMgr.CutFlowSvc.InputStream = inputStreamName - #if not hasattr(svcMgr,"FileCutFlowSvc"): svcMgr += CfgMgr.FileCutFlowSvc() - #svcMgr.FileCutFlowSvc.InputStream = inputStreamName # Make sure MetaDataSvc is ready if not hasattr(svcMgr,'MetaDataSvc'): @@ -81,6 +79,7 @@ def CreateCutFlowSvc( svcName="CutFlowSvc", athFile=None, seq=None, addAlgInPlac InputCollName = inname, OutputCollName= outname) svcMgr.ToolSvc += cutflowtool + svcMgr.ToolSvc.OutputLevel = 2 # Add tool to MetaDataSvc svcMgr.MetaDataSvc.MetaDataTools += [cutflowtool] @@ -88,9 +87,6 @@ def CreateCutFlowSvc( svcName="CutFlowSvc", athFile=None, seq=None, addAlgInPlac # Add pdf sum of weights counts if appropriate from AthenaCommon.GlobalFlags import globalflags if globalflags.DataSource() == 'geant4': - #from PyUtils import AthFile - #afc = AthFile.fopen( svcMgr.EventSelector.InputCollections[0] ) - # PDF name = "PDFSumOfWeights" pdfweighttool = BookkeeperTool(name,