Merge branch '21.0-CheckTmpCoolProductionOnGrid' into '21.0'

Patch for running MC TriggerDB from SQLite (attempt 2)

See merge request !39271

Trigger/TrigConfiguration/TrigConfStorage/TrigConfStorage/SessionMgr.h

@@ -48,6 +48,7 @@ namespace TrigConf {
       // setters
       void setConnectionString(const std::string & connStr) { m_connectionString = connStr; }
       void setUseFrontier(bool useFrontier) { m_useFrontier = useFrontier; }
+      void setUseSQLite(bool useSQLite) { m_useSQLite = useSQLite; }
       void setRetrialPeriod(int retrialPeriod) { m_retrialPeriod = retrialPeriod; }
       void setRetrialTimeout(int retrialTimeout) { m_retrialTimeout = retrialTimeout; }
       void setConnectionTimeout(int connectionTimeout) { m_connectionTimeout = connectionTimeout; }
@@ -61,6 +62,7 @@ namespace TrigConf {
       // accessors
       const std::string& connection() const { return m_connectionString; }
       bool useFrontier() const { return m_useFrontier; }
+      bool useSQLite() const { return m_useSQLite; }
       int retrialPeriod() const { return m_retrialPeriod; }
       int retrialTimeout() const { return m_retrialTimeout; }
       int connectionTimeout() const { return m_connectionTimeout; }
@@ -87,6 +89,7 @@ namespace TrigConf {
       std::string            m_user { "" };        ///< user name
       std::string            m_password { "" };    ///< password
       bool                   m_useFrontier { false }; ///< uses frontier instead of oracle
+      bool                   m_useSQLite { true }; ///< resolve sqlite in the replicaSorter
       int                    m_retrialPeriod {0};
       int                    m_retrialTimeout {0};
       int                    m_connectionTimeout {0};

Trigger/TrigConfiguration/TrigConfStorage/TrigConfStorage/StorageMgr.h

@@ -105,6 +105,9 @@ namespace TrigConf {
       void setUseFrontier(bool useFrontier) { m_useFrontier = useFrontier; } 
       bool useFrontier() const { return m_useFrontier; } 
 
+      void setUseSQLite(bool useSQLite) { m_useSQLite = useSQLite; } 
+      bool useSQLite() const { return m_useSQLite; } 
+
       void setRetrialPeriod(int retrialPeriod) { m_retrialPeriod = retrialPeriod; }
       void setRetrialTimeout(int retrialTimeout) { m_retrialTimeout = retrialTimeout; }
       void setConnectionTimeout(int connectionTimeout) { m_connectionTimeout = connectionTimeout; }
@@ -131,6 +134,7 @@ namespace TrigConf {
       std::string     m_user {""};    ///< user name        
       std::string     m_password {""};///< password         
       bool            m_useFrontier {false}; ///< using frontier to connect to oracle (if set, disables sqlite) 
+      bool            m_useSQLite {true}; ///< using frontier to connect to oracle (if set, disables sqlite) 
       std::ostream &  m_ostream; ///< output stream    
 
       int             m_retrialPeriod {0};

Trigger/TrigConfiguration/TrigConfStorage/src/ReplicaSorter.cxx

@@ -17,35 +17,69 @@
 
 using namespace std;
 
-TrigConf::ReplicaSorter::ReplicaSorter() :
-   m_frontiergen(false) 
+TrigConf::ReplicaSorter::ReplicaSorter()
 {
-   std::cout << "ReplicaSorter constructor" << std::endl;
    readConfig();
 }
 
 void
 TrigConf::ReplicaSorter::sort(std::vector<const coral::IDatabaseServiceDescription*>& replicaSet) {
+   // if only one replica offered, return immediately
+   // this helps for online, where explicit configuration file is given
+   // that does not match any of the standard dbreplica.config entries
+   if (replicaSet.size()<=1) return;
+
    // loop through all the offered replicas
    std::map<int,const coral::IDatabaseServiceDescription*> primap;
-   for (std::vector<const coral::IDatabaseServiceDescription*>::const_iterator 
-           itr=replicaSet.begin();itr!=replicaSet.end();++itr) {
-      const std::string conn=(**itr).connectionString();
+
+   for( const coral::IDatabaseServiceDescription* dbSrv : replicaSet) {
+      const std::string & conn = dbSrv->connectionString();
       // do not use SQLite files
-      if (conn.find("sqlite_file")==std::string::npos) {
+      if (conn.find("sqlite_file")!=std::string::npos) {
+         // include SQLite files unless they are vetoed
+         // COOL SQLIte files recognised by ALLP in connection string
+         // vetoed if use-SQlite flag not set, or pattern is found in
+         // SQLite filename
+         // Geometry SQLite files recognised by geomDB in connection string
+         if (! (( (m_useSQLite==false) && conn.find("ALLP")!=std::string::npos)))
+            {
+               // local sqlite files get -9999, DB release ones
+               // (identified with path starting / or containing DBRelease)
+               // get -999, so local one will be tried first if present
+               if (conn.find("sqlite_file:/")!=std::string::npos ||
+                   conn.find("DBRelease")!=std::string::npos) {
+                  primap[-999]=dbSrv;
+               } else {
+                  primap[-9999]=dbSrv;
+               }
+            }
+      } else {
+
+         // define priority for technologies with this server (lower = better)
+         bool veto=false;
+         int spri=5; // default for Oracle
+         if (conn.find("frontier:")!=std::string::npos) {
+            spri=3; // use frontier before oracle
+            // dont use frontier servers if disabled, or generic Frontier server
+            // is specified (via '()' in server definition) and FRONTIER_SERVER
+            // env variable is not set
+            if (!m_useFrontier ||
+                (conn.find("()")!=std::string::npos && m_useFrontierGen==false))
+               veto=true;
+         }
          // extract the server name (assuming URLs "techno://server/schema")
          // example of current conn naming scheme:  coral://127.0.0.1:3320/&oracle://ATLAS_CONFIG/ATLAS_CONF_TRIGGER_REPR
          std::string::size_type ipos0=conn.find("&");
          std::string::size_type ipos1=conn.find("://",ipos0+1);
          std::string::size_type ipos2=conn.find("/",ipos1+3);
-         if (ipos1!=std::string::npos && ipos2!=std::string::npos) {
+         if (ipos1!=std::string::npos && ipos2!=std::string::npos && !veto) {
             const std::string server=conn.substr(ipos1+3,ipos2-ipos1-3);
             // check if this server is on list of replicas to use for domain
             // if so, add it with its associated priority
             for (ServerMap::const_iterator sitr=m_servermap.begin();
                  sitr!=m_servermap.end();++sitr) {
                if (sitr->first==server) 
-                  primap[sitr->second]=*itr;
+                  primap[sitr->second+spri]=dbSrv;
             }
          }
       }
@@ -86,7 +120,7 @@ TrigConf::ReplicaSorter::readConfig() {
    if (cfrontier && strcmp(cfrontier,"")!=0) {
       std::cout << "Frontier server at " << cfrontier << " will be considered"
                 << std::endl;
-      m_frontiergen=true;
+      m_useFrontierGen=true;
    }
  
    // try to locate configuration file using pathresolver
@@ -128,7 +162,7 @@ TrigConf::ReplicaSorter::readConfig() {
                   // token is a server name
                   // only add Frontier ATLF server if FRONTIER_CLIENT set
                   if(atCERN && token=="ATONR_CONF") atCERN = false;
-                  if (token!="ATLF" || m_frontiergen) servers.push_back(token);
+                  if (token!="ATLF" || m_useFrontierGen) servers.push_back(token);
                }
             }
             iofs1=iofs2+1;
@@ -159,16 +193,13 @@ TrigConf::ReplicaSorter::readConfig() {
             }
          }
          if (useit) {
-            if(atCERN) {
-               servers.push_back("ATONR_COOL");
-               servers.push_back("ATONR_CONF");
-            }
             // assign these servers, priority based on position in list
             // and length of match of domain name
             for (unsigned int i=0;i<servers.size();++i) {
                int priority=i-100*bestlen;
                m_servermap.push_back(ServerPair(servers[i],priority));
             }
+            break;
          }
       }
    }

Trigger/TrigConfiguration/TrigConfStorage/src/ReplicaSorter.h

@@ -14,8 +14,16 @@ namespace TrigConf {
    class ReplicaSorter : virtual public coral::IReplicaSortingAlgorithm {
    public:
       ReplicaSorter();
-      void sort(std::vector<const coral::IDatabaseServiceDescription*>& 
-                replicaSet);
+      void sort(std::vector<const coral::IDatabaseServiceDescription*>& replicaSet);
+
+      void setUseFrontier(bool useFrontier) { m_useFrontier = useFrontier; }
+      void setUseFrontierGen(bool useFrontierGen) { m_useFrontierGen = useFrontierGen; }
+      void setUseSQLite(bool useSQLite) { m_useSQLite = useSQLite; }
+
+      bool useFrontier() const { return m_useFrontier; }
+      bool useFrontierGen() const { return m_useFrontierGen; }
+      bool useSQLite() const { return m_useSQLite; }
+
    private:
       bool readConfig();
       FILE* findFile(const std::string filename, const std::string pathvar);
@@ -23,7 +31,9 @@ namespace TrigConf {
       typedef std::pair<std::string,int> ServerPair;
       typedef std::vector< ServerPair > ServerMap;
       ServerMap m_servermap;
-      bool m_frontiergen;
+      bool m_useFrontierGen {false};
+      bool m_useSQLite {true};
+      bool m_useFrontier {true};
    };
 
 }

Trigger/TrigConfiguration/TrigConfStorage/src/SessionMgr.cxx

@@ -93,17 +93,20 @@ TrigConf::SessionMgr::createSession() {
 
    
    if(csc.replicaSortingAlgorithm() == nullptr) { // likely to be standalone, create our own
-      TRG_MSG_INFO("Create own ReplicaSortingAlgorithm");
+      TRG_MSG_INFO("Create own ReplicaSortingAlgorithm with useFrontier=" << (m_useFrontier ? "true" : "false") << " and useSQLite=" << ( m_useSQLite ? "true" : "false"));
       m_replicaSorter = new TrigConf::ReplicaSorter();
+      m_replicaSorter->setUseFrontier(m_useFrontier);
+      m_replicaSorter->setUseSQLite(m_useSQLite);
       csc.setReplicaSortingAlgorithm(*m_replicaSorter);
    }
 
    buildConnectionString();
    TRG_MSG_INFO("Connecting to " << m_connectionString);
    m_sessionproxy = connSvc.connect(m_connectionString, coral::AccessMode::ReadOnly);
+
    TRG_MSG_INFO("Opening session " << m_connectionString << " with " << m_retrialPeriod << "/" << m_retrialTimeout << "/" << m_connectionTimeout);
 
-   return *m_sessionproxy;   
+   return *m_sessionproxy;
 }
 
 

Trigger/TrigConfiguration/TrigConfStorage/src/test/2COOLApp.cxx

@@ -763,9 +763,9 @@ int main( int argc, char* argv[] ) {
          }
       
          try {
-            std::unique_ptr< TrigConf::IStorageMgr > sm = 
-               std::unique_ptr< TrigConf::IStorageMgr >(new TrigConf::StorageMgr(csv[0],user,passwd,log));
-
+            std::unique_ptr< TrigConf::StorageMgr > sm =
+               std::unique_ptr< TrigConf::StorageMgr >(new TrigConf::StorageMgr(csv[0],user,passwd,log));
+            sm->setUseFrontier(gConfig.UseFrontier());
             // setup the coolWriter
             TrigConf::TrigConfCoolWriter coolWriter(gConfig.CoolConnection(),log);
             std::vector<std::string>::const_iterator wfIt = gConfig.ListOfWriteFolders().begin();

Trigger/TrigConfiguration/TrigConfigSvc/TrigConfigSvc/ConfigSvcBase.h

@@ -41,6 +41,7 @@ namespace TrigConf {
       int                m_dbHLTPSKey { 0 };
       std::string        m_dbHLTPSKeySet { "[]" };
       BooleanProperty    m_useFrontier { false };
+      BooleanProperty    m_useSQLite { true };
       int                m_printMenuLevel { 1 };
       std::unique_ptr<TrigDBConnectionConfig> m_dbconfig;
 

Trigger/TrigConfiguration/TrigConfigSvc/TrigConfigSvc/HLTConfigSvc.h

@@ -39,10 +39,10 @@ namespace TrigConf {
       using AthService::AthMessaging::msg;
 
       // implementing IIHLTConfigSvc
-      const HLTChainList*    chainList() const __attribute__ ((deprecated));
+      const HLTChainList*    chainList() const;
       const HLTChainList&    chains() const;
       
-      const HLTSequenceList* sequenceList() const __attribute__ ((deprecated));
+      const HLTSequenceList* sequenceList() const;
       const HLTSequenceList& sequences() const;
       
       /*@brief constructor*/

Trigger/TrigConfiguration/TrigConfigSvc/python/TrigConfigSvcUtils.py

@@ -307,9 +307,12 @@ def getUsedTables(output, condition, schemaname, tables):
 
 
 def isRun2(cursor,schemaname):
-    import cx_Oracle
-    if not hasattr(cursor,'connection') or type(cursor.connection)!=cx_Oracle.Connection:
-        log.warning('Detection of DB schema only supported for Oracle. Will assume run-2')
+    try:
+        import cx_Oracle
+        if not hasattr(cursor,'connection') or type(cursor.connection)!=cx_Oracle.Connection:
+            log.warning('Detection of DB schema only supported for Oracle. Will assume run-2')
+            return True
+    except ImportError:
         return True
 
     owner = schemaname.rstrip('.')

Trigger/TrigConfiguration/TrigConfigSvc/src/ConfigSvcBase.cxx

@@ -42,6 +42,8 @@ ConfigSvcBase::declareCommonProperties() {
                     "The SuperMaster key");
    declareProperty( "UseFrontier",      m_useFrontier,
                     "Tries to use Frontier for accessing the TriggerDB");
+   declareProperty( "UseSQLite",      m_useSQLite,
+                    "Tries to use SQLite for accessing the TriggerDB if available");
    declareProperty( "PrintMenu",        m_printMenuLevel,
                     "Prints menu with detail level x=0..5 [default = " + boost::lexical_cast<string,int>(m_printMenuLevel) + "]");
 }
@@ -108,7 +110,7 @@ ConfigSvcBase::initStorageMgr() {
       StorageMgr * sm = new StorageMgr( connectionString, m_dbconfig->m_user, m_dbconfig->m_password );
 
       sm->setUseFrontier(m_dbconfig->m_useFrontier);
-
+      sm->setUseSQLite(m_useSQLite);
       sm->setRetrialPeriod( m_dbconfig->m_retrialPeriod );
       sm->setRetrialTimeout( m_dbconfig->m_retrialPeriod * (m_dbconfig->m_maxRetrials + 1) );
       sm->setConnectionTimeout( 0 );