From 79f40d663cb5092ed0172bfcc0583237e2e550b1 Mon Sep 17 00:00:00 2001
From: Mikhail Mineev <Mikhail.Mineev@cern.ch>
Date: Tue, 13 Feb 2024 22:51:51 +0100
Subject: [PATCH 1/2] Duplicated requests to global tag map removed

---
 Database/IOVDbSvc/src/IOVDbFolder.cxx | 84 +++++++++------------------
 Database/IOVDbSvc/src/IOVDbFolder.h   |  8 +--
 Database/IOVDbSvc/src/IOVDbSvc.cxx    | 18 +++++-
 Database/IOVDbSvc/src/IOVDbSvc.h      |  3 +
 4 files changed, 51 insertions(+), 62 deletions(-)

diff --git a/Database/IOVDbSvc/src/IOVDbFolder.cxx b/Database/IOVDbSvc/src/IOVDbFolder.cxx
index c8340fac93ac..3c1f4c949757 100644
--- a/Database/IOVDbSvc/src/IOVDbFolder.cxx
+++ b/Database/IOVDbSvc/src/IOVDbFolder.cxx
@@ -74,7 +74,7 @@ IOVDbFolder::IOVDbFolder(IOVDbConn* conn,
                          IClassIDSvc* clidsvc, IIOVDbMetaDataTool* metadatatool,
                          const bool checklock, const bool outputToFile,
                          const std::string & source, const bool crestToFile,
-                         const std::string & crestServer):
+                         const std::string & crestServer, const std::string & crestTag):
   AthMessaging("IOVDbFolder"),
   p_clidSvc(clidsvc),
   p_metaDataTool(metadatatool),
@@ -85,7 +85,8 @@ IOVDbFolder::IOVDbFolder(IOVDbConn* conn,
   m_outputToFile{outputToFile},
   m_crestToFile{crestToFile},
   m_source{source},
-  m_crestServer{crestServer}
+  m_crestServer{crestServer},
+  m_crestTag{crestTag}
 {
   // set message same message level as our parent (IOVDbSvc)
   setLevel(msg.level());
@@ -150,8 +151,6 @@ IOVDbFolder::IOVDbFolder(IOVDbConn* conn,
   if (m_extensible) {
     ATH_MSG_INFO( "Extensible folder " << m_foldername );
   }
-
-
   
 }
 
@@ -236,17 +235,11 @@ IOVDbFolder::loadCache(const cool::ValidityKey vkey,
     //const std::string  jsonFolderName=sanitiseCrestTag(m_foldername);
 
     CrestFunctions cfunctions(m_crestServer);
-
-    if (m_globaltag != globalTag){
-      m_globaltag = globalTag;
-      m_cresttagmap.clear();
-      m_cresttagmap = cfunctions.getGlobalTagMap(globalTag);
-    }
-    const std::string completeTag = m_cresttagmap[m_foldername];
-    ATH_MSG_INFO("Download tag would be: "<<completeTag);
+    
+    ATH_MSG_INFO("Download tag would be: "<<m_crestTag);
 
     std::string crestPayloadType="crest-json-single-iov";
-    nlohmann::json tagProperties = cfunctions.getTagProperties(completeTag);
+    nlohmann::json tagProperties = cfunctions.getTagProperties(m_crestTag);
     if(tagProperties!=nullptr){
       if(tagProperties.contains("payloadSpec"))
         crestPayloadType=tagProperties["payloadSpec"].get<std::string>();
@@ -257,7 +250,7 @@ IOVDbFolder::loadCache(const cool::ValidityKey vkey,
     
     // Get a vector of pairs retrieved from crest
     //  <IOV_SINCE(string),HASH(string)>
-    auto crestIOVs = cfunctions.getIovsForTag(completeTag);
+    auto crestIOVs = cfunctions.getIovsForTag(m_crestTag);
     if(crestIOVs.empty()){
       ATH_MSG_WARNING("Load cache failed for " << m_foldername << ". No IOVs retrieved from the DB");
     }
@@ -369,31 +362,31 @@ IOVDbFolder::loadCache(const cool::ValidityKey vkey,
 
       nlohmann::json js =
       {
-        {"name", completeTag}
+        {"name", m_crestTag}
       };
 
       try{
         crestFSClient.createTag(js);
-        ATH_MSG_INFO("Tag " << completeTag << " saved to disk.");
+        ATH_MSG_INFO("Tag " << m_crestTag << " saved to disk.");
         ATH_MSG_INFO("CREST Dump dir = " << crest_work_dir);
       }
       catch (const std::exception& e) {
-        ATH_MSG_WARNING("Data saving for tag " << completeTag << " failed: " << e.what());
+        ATH_MSG_WARNING("Data saving for tag " << m_crestTag << " failed: " << e.what());
       }
 
       try{
-        crestFSClient.storePayloadDump(completeTag, sinceT, reply);
-        ATH_MSG_INFO("Data (payload and IOV) saved for tag " << completeTag << ".");
+        crestFSClient.storePayloadDump(m_crestTag, sinceT, reply);
+        ATH_MSG_INFO("Data (payload and IOV) saved for tag " << m_crestTag << ".");
       }
       catch (const std::exception& e) {
-        ATH_MSG_WARNING("Data (payload and IOV) saving for tag " << completeTag<<" failed; " << e.what());
+        ATH_MSG_WARNING("Data (payload and IOV) saving for tag " << m_crestTag<<" failed; " << e.what());
       }
     }
 
     //
-    if (m_crest_tag != completeTag){
-      m_crest_tag = completeTag;
-      m_tag_info = cfunctions.getTagInfo(completeTag);
+    if (m_crest_tag != m_crestTag){
+      m_crest_tag = m_crestTag;
+      m_tag_info = cfunctions.getTagInfo(m_crestTag);
     }
 
     const std::string& nodeDescription = cfunctions.getTagInfoElement(m_tag_info,"node_description");
@@ -1063,7 +1056,7 @@ IOVDbFolder::createTransientAddress(const std::vector<std::string> & symlinks){
 }
 
 std::unique_ptr<SG::TransientAddress>
-IOVDbFolder::preLoadFolder(ITagInfoMgr *tagInfoMgr , const unsigned int cacheRun, const unsigned int cacheTime, const std::string& globalTag) {
+IOVDbFolder::preLoadFolder(ITagInfoMgr *tagInfoMgr , const unsigned int cacheRun, const unsigned int cacheTime) {
   // preload Address from SG - does folder setup including COOL access
   // also set detector store location - cannot be done in constructor
   // as detector store does not exist yet in IOVDbSvc initialisation
@@ -1073,16 +1066,10 @@ IOVDbFolder::preLoadFolder(ITagInfoMgr *tagInfoMgr , const unsigned int cacheRun
   if( not m_useFileMetaData ) {
     if(m_source=="CREST"){
       CrestFunctions cfunctions(m_crestServer);
-      if (m_globaltag != globalTag){
-        m_globaltag = globalTag;
-        m_cresttagmap.clear();
-        m_cresttagmap = cfunctions.getGlobalTagMap(globalTag);
-      }
-      const std::string  & tagName=m_cresttagmap[m_foldername];
 
-      if (m_crest_tag != tagName){
-        m_crest_tag = tagName;
-        m_tag_info = cfunctions.getTagInfo(tagName);
+      if (m_crest_tag != m_crestTag){
+        m_crest_tag = m_crestTag;
+        m_tag_info = cfunctions.getTagInfo(m_crestTag);
       }
       m_folderDescription = cfunctions.getTagInfoElement(m_tag_info,"node_description");
     } else {
@@ -1107,19 +1094,8 @@ IOVDbFolder::preLoadFolder(ITagInfoMgr *tagInfoMgr , const unsigned int cacheRun
   // setup channel list and folder type
   if( not m_useFileMetaData ) {
     if(m_source=="CREST"){
-        CrestFunctions cfunctions(m_crestServer);
-        if (m_globaltag != globalTag){
-          m_globaltag = globalTag;
-          m_cresttagmap.clear();
-          m_cresttagmap = cfunctions.getGlobalTagMap(globalTag);
-        }       
-        const std::string  & crestTag=m_cresttagmap[m_foldername];
-
-        if (m_crest_tag != crestTag){
-          m_crest_tag = crestTag;
-          m_tag_info = cfunctions.getTagInfo(crestTag);
-        }
- 
+        CrestFunctions cfunctions(m_crestServer); 
+
         const std::string & payloadSpec = cfunctions.getTagInfoElement(m_tag_info,"payload_spec");  
         std::string chanList = cfunctions.getTagInfoElement(m_tag_info,"channel_list");
         std::tie(m_channums, m_channames) = cfunctions.extractChannelListFromString(chanList);
@@ -1142,7 +1118,7 @@ IOVDbFolder::preLoadFolder(ITagInfoMgr *tagInfoMgr , const unsigned int cacheRun
           int colsize = std::count(payloadSpec.begin(), payloadSpec.end(), ch);
 
 	  nlohmann::json tag_meta;
-          tag_meta["tagName"] = crestTag;
+          tag_meta["tagName"] = m_crestTag;
           tag_meta["description"] = "";
           tag_meta["chansize"] = n_size;
           tag_meta["colsize"] = colsize;
@@ -1161,11 +1137,11 @@ IOVDbFolder::preLoadFolder(ITagInfoMgr *tagInfoMgr , const unsigned int cacheRun
 
           try{
             crestFSClient.createTagMetaInfo(tag_meta);;
-            ATH_MSG_INFO("Tag meta info for " << crestTag << " saved to disk.");
+            ATH_MSG_INFO("Tag meta info for " << m_crestTag << " saved to disk.");
             ATH_MSG_INFO("CREST Dump dir = " << crest_work_dir);
           }
           catch (const std::exception& e) {
-            ATH_MSG_WARNING("Tag meta info saving for tag " << crestTag << " failed: " << e.what());
+            ATH_MSG_WARNING("Tag meta info saving for tag " << m_crestTag << " failed: " << e.what());
           }
 	} // m_crestToFile
 
@@ -1248,13 +1224,9 @@ IOVDbFolder::resolveTag(const cool::IFolderPtr& fptr,const std::string& globalTa
     return false;
   }
   if(m_source=="CREST"){
-    if (m_globaltag != globalTag){
-      m_globaltag = globalTag;
-      m_cresttagmap.clear();
-      CrestFunctions cfunctions(m_crestServer);
-      m_cresttagmap = cfunctions.getGlobalTagMap(globalTag);
-    }
-    m_tag = m_cresttagmap[m_foldername];
+
+    m_tag = m_crestTag;
+
     ATH_MSG_DEBUG( "resolveTag returns " << m_tag );
     return true;
   }
diff --git a/Database/IOVDbSvc/src/IOVDbFolder.h b/Database/IOVDbSvc/src/IOVDbFolder.h
index 9d48ab035efc..42d77c45b0ff 100644
--- a/Database/IOVDbSvc/src/IOVDbFolder.h
+++ b/Database/IOVDbSvc/src/IOVDbFolder.h
@@ -1,3 +1,4 @@
+
 /*
   Copyright (C) 2002-2021 CERN for the benefit of the ATLAS collaboration
 */
@@ -46,7 +47,7 @@ public:
               IClassIDSvc* clidsvc, IIOVDbMetaDataTool* metadatatool,
               const bool checklock, const bool outputToFile=false,
               const std::string & source="COOL_DATABASE", const bool crestToFile=false,
-              const std::string & crestServer="");
+              const std::string & crestServer="",const std::string & crestTag="");
   ~IOVDbFolder();
   
 
@@ -121,7 +122,7 @@ public:
   std::unique_ptr<SG::TransientAddress>
   preLoadFolder(ITagInfoMgr *tagInfoMgr,
                 const unsigned int cacheRun,
-                const unsigned int cacheTime,const std::string& globalTag="");
+                const unsigned int cacheTime);
 
   // print out cache
   void printCache();
@@ -291,8 +292,7 @@ private:
   const bool m_crestToFile{false};
   const std::string m_source;
   const std::string m_crestServer;
-  std::string m_globaltag = "";
-  std::map<std::string, std::string> m_cresttagmap; // pairs: COOL folder - CREST tag name
+  const std::string m_crestTag;
   std::string m_crest_tag = "";
   nlohmann::json m_tag_info = nullptr;
 };
diff --git a/Database/IOVDbSvc/src/IOVDbSvc.cxx b/Database/IOVDbSvc/src/IOVDbSvc.cxx
index e709950b0389..d9f50a9965ad 100644
--- a/Database/IOVDbSvc/src/IOVDbSvc.cxx
+++ b/Database/IOVDbSvc/src/IOVDbSvc.cxx
@@ -399,7 +399,7 @@ StatusCode IOVDbSvc::preLoadAddresses(StoreID::type storeID,tadList& tlist) {
         if (folder->conn()==pThisConnection || (folder->conn()==nullptr && doMeta)) {
           std::unique_ptr<SG::TransientAddress> tad =
             folder->preLoadFolder( &(*m_h_tagInfoMgr), m_par_cacheRun.value(),
-                                   m_par_cacheTime.value(),m_globalTag);
+                                   m_par_cacheTime.value());
           if (oldconn!=pThisConnection) {
             // close old connection if appropriate
             if (m_par_manageConnections && oldconn!=nullptr) oldconn->setInactive();
@@ -946,6 +946,13 @@ StatusCode IOVDbSvc::setupFolders() {
   // read the Folders joboptions and setup the folder list
   // no wildcards are allowed
 
+  // getting the pairs: folder name - CREST tag name:
+  if (m_par_source == "CREST"){
+    IOVDbNamespace::CrestFunctions cfunctions(m_par_crestServer);
+    m_cresttagmap.clear();
+    m_cresttagmap = cfunctions.getGlobalTagMap(m_par_globalTag);
+  }
+  
   //1. Loop through folders
   std::list<IOVDbParser> allFolderdata;
   for (const auto & thisFolder : m_par_folders.value()) {
@@ -1043,10 +1050,17 @@ StatusCode IOVDbSvc::setupFolders() {
         return StatusCode::FAILURE;
       }
     }
+    
     // create the new folder, but only if a folder for this SG key has not
     // already been requested
+
+    std::string crestTag = "";
+    if (m_par_source == "CREST"){
+      crestTag = m_cresttagmap[folderdata.folderName()];
+    }
+    
     IOVDbFolder* folder=new IOVDbFolder(conn,folderdata,msg(),&(*m_h_clidSvc), &(*m_h_metaDataTool),
-                                        m_par_checklock, m_outputToFile.value(), m_par_source, m_crestToFile.value(), m_par_crestServer);
+                                        m_par_checklock, m_outputToFile.value(), m_par_source, m_crestToFile.value(), m_par_crestServer, crestTag);
     const std::string& key=folder->key();
     if (m_foldermap.find(key)==m_foldermap.end()) {  //This check is too weak. For POOL-based folders, the SG key is in the folder description (not known at this point).
       m_foldermap[key]=folder;
diff --git a/Database/IOVDbSvc/src/IOVDbSvc.h b/Database/IOVDbSvc/src/IOVDbSvc.h
index 35172fba6907..63d3d48c44bf 100644
--- a/Database/IOVDbSvc/src/IOVDbSvc.h
+++ b/Database/IOVDbSvc/src/IOVDbSvc.h
@@ -41,6 +41,7 @@
 #include "CoolKernel/ValidityKey.h"
 
 #include "IOVDbConn.h"
+#include "CrestFunctions.h"
 
 
 #include <string>
@@ -229,6 +230,8 @@ private:
   StringProperty m_par_source{this,"Source","COOL_DATABASE","source of data as a string (default COOL_DATABASE)"};
   // CREST Server URL with host number; default is "http://crest-undertow-api.web.cern.ch"
   StringProperty m_par_crestServer{this,"crestServer","http://crest-undertow-api.web.cern.ch","CREST URL with the port number as a string (default http://crest-undertow-api.web.cern.ch)"};
+  // This map contains the pairs: COOL folder - CREST tag name 
+  std::map<std::string, std::string> m_cresttagmap;
   // Format of data; default is empty string (default for a given source)
   StringProperty m_par_format{this,"Format",{},"Format of data; default is empty string (default for a given source)"};
   // Can output to file for debugging purposes
-- 
GitLab


From ac4feabe88f5ff3c20c9f3a9e518a155c2c62e83 Mon Sep 17 00:00:00 2001
From: Mikhail Mineev <Mikhail.Mineev@cern.ch>
Date: Wed, 14 Feb 2024 10:14:39 +0100
Subject: [PATCH 2/2] small corrections in IOVDbFolder.h

---
 Database/IOVDbSvc/src/IOVDbFolder.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Database/IOVDbSvc/src/IOVDbFolder.h b/Database/IOVDbSvc/src/IOVDbFolder.h
index 42d77c45b0ff..7fad0064de73 100644
--- a/Database/IOVDbSvc/src/IOVDbFolder.h
+++ b/Database/IOVDbSvc/src/IOVDbFolder.h
@@ -1,6 +1,5 @@
-
 /*
-  Copyright (C) 2002-2021 CERN for the benefit of the ATLAS collaboration
+  Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
 */
 
 // IOVDbFolder.h
-- 
GitLab