Commit 2c682394 authored by Rachid Mazini's avatar Rachid Mazini
Browse files

Merge branch '21.0-CheckTmpCoolProductionOnGrid' into '21.0'

Patch for running MC TriggerDB from SQLite (attempt 2)

See merge request !39271
parents f96b9017 07722391
...@@ -48,6 +48,7 @@ namespace TrigConf { ...@@ -48,6 +48,7 @@ namespace TrigConf {
// setters // setters
void setConnectionString(const std::string & connStr) { m_connectionString = connStr; } void setConnectionString(const std::string & connStr) { m_connectionString = connStr; }
void setUseFrontier(bool useFrontier) { m_useFrontier = useFrontier; } void setUseFrontier(bool useFrontier) { m_useFrontier = useFrontier; }
void setUseSQLite(bool useSQLite) { m_useSQLite = useSQLite; }
void setRetrialPeriod(int retrialPeriod) { m_retrialPeriod = retrialPeriod; } void setRetrialPeriod(int retrialPeriod) { m_retrialPeriod = retrialPeriod; }
void setRetrialTimeout(int retrialTimeout) { m_retrialTimeout = retrialTimeout; } void setRetrialTimeout(int retrialTimeout) { m_retrialTimeout = retrialTimeout; }
void setConnectionTimeout(int connectionTimeout) { m_connectionTimeout = connectionTimeout; } void setConnectionTimeout(int connectionTimeout) { m_connectionTimeout = connectionTimeout; }
...@@ -61,6 +62,7 @@ namespace TrigConf { ...@@ -61,6 +62,7 @@ namespace TrigConf {
// accessors // accessors
const std::string& connection() const { return m_connectionString; } const std::string& connection() const { return m_connectionString; }
bool useFrontier() const { return m_useFrontier; } bool useFrontier() const { return m_useFrontier; }
bool useSQLite() const { return m_useSQLite; }
int retrialPeriod() const { return m_retrialPeriod; } int retrialPeriod() const { return m_retrialPeriod; }
int retrialTimeout() const { return m_retrialTimeout; } int retrialTimeout() const { return m_retrialTimeout; }
int connectionTimeout() const { return m_connectionTimeout; } int connectionTimeout() const { return m_connectionTimeout; }
...@@ -87,6 +89,7 @@ namespace TrigConf { ...@@ -87,6 +89,7 @@ namespace TrigConf {
std::string m_user { "" }; ///< user name std::string m_user { "" }; ///< user name
std::string m_password { "" }; ///< password std::string m_password { "" }; ///< password
bool m_useFrontier { false }; ///< uses frontier instead of oracle bool m_useFrontier { false }; ///< uses frontier instead of oracle
bool m_useSQLite { true }; ///< resolve sqlite in the replicaSorter
int m_retrialPeriod {0}; int m_retrialPeriod {0};
int m_retrialTimeout {0}; int m_retrialTimeout {0};
int m_connectionTimeout {0}; int m_connectionTimeout {0};
......
...@@ -105,6 +105,9 @@ namespace TrigConf { ...@@ -105,6 +105,9 @@ namespace TrigConf {
void setUseFrontier(bool useFrontier) { m_useFrontier = useFrontier; } void setUseFrontier(bool useFrontier) { m_useFrontier = useFrontier; }
bool useFrontier() const { return m_useFrontier; } bool useFrontier() const { return m_useFrontier; }
void setUseSQLite(bool useSQLite) { m_useSQLite = useSQLite; }
bool useSQLite() const { return m_useSQLite; }
void setRetrialPeriod(int retrialPeriod) { m_retrialPeriod = retrialPeriod; } void setRetrialPeriod(int retrialPeriod) { m_retrialPeriod = retrialPeriod; }
void setRetrialTimeout(int retrialTimeout) { m_retrialTimeout = retrialTimeout; } void setRetrialTimeout(int retrialTimeout) { m_retrialTimeout = retrialTimeout; }
void setConnectionTimeout(int connectionTimeout) { m_connectionTimeout = connectionTimeout; } void setConnectionTimeout(int connectionTimeout) { m_connectionTimeout = connectionTimeout; }
...@@ -131,6 +134,7 @@ namespace TrigConf { ...@@ -131,6 +134,7 @@ namespace TrigConf {
std::string m_user {""}; ///< user name std::string m_user {""}; ///< user name
std::string m_password {""};///< password std::string m_password {""};///< password
bool m_useFrontier {false}; ///< using frontier to connect to oracle (if set, disables sqlite) bool m_useFrontier {false}; ///< using frontier to connect to oracle (if set, disables sqlite)
bool m_useSQLite {true}; ///< using frontier to connect to oracle (if set, disables sqlite)
std::ostream & m_ostream; ///< output stream std::ostream & m_ostream; ///< output stream
int m_retrialPeriod {0}; int m_retrialPeriod {0};
......
...@@ -17,35 +17,69 @@ ...@@ -17,35 +17,69 @@
using namespace std; using namespace std;
TrigConf::ReplicaSorter::ReplicaSorter() : TrigConf::ReplicaSorter::ReplicaSorter()
m_frontiergen(false)
{ {
std::cout << "ReplicaSorter constructor" << std::endl;
readConfig(); readConfig();
} }
void void
TrigConf::ReplicaSorter::sort(std::vector<const coral::IDatabaseServiceDescription*>& replicaSet) { TrigConf::ReplicaSorter::sort(std::vector<const coral::IDatabaseServiceDescription*>& replicaSet) {
// if only one replica offered, return immediately
// this helps for online, where explicit configuration file is given
// that does not match any of the standard dbreplica.config entries
if (replicaSet.size()<=1) return;
// loop through all the offered replicas // loop through all the offered replicas
std::map<int,const coral::IDatabaseServiceDescription*> primap; std::map<int,const coral::IDatabaseServiceDescription*> primap;
for (std::vector<const coral::IDatabaseServiceDescription*>::const_iterator
itr=replicaSet.begin();itr!=replicaSet.end();++itr) { for( const coral::IDatabaseServiceDescription* dbSrv : replicaSet) {
const std::string conn=(**itr).connectionString(); const std::string & conn = dbSrv->connectionString();
// do not use SQLite files // do not use SQLite files
if (conn.find("sqlite_file")==std::string::npos) { if (conn.find("sqlite_file")!=std::string::npos) {
// include SQLite files unless they are vetoed
// COOL SQLIte files recognised by ALLP in connection string
// vetoed if use-SQlite flag not set, or pattern is found in
// SQLite filename
// Geometry SQLite files recognised by geomDB in connection string
if (! (( (m_useSQLite==false) && conn.find("ALLP")!=std::string::npos)))
{
// local sqlite files get -9999, DB release ones
// (identified with path starting / or containing DBRelease)
// get -999, so local one will be tried first if present
if (conn.find("sqlite_file:/")!=std::string::npos ||
conn.find("DBRelease")!=std::string::npos) {
primap[-999]=dbSrv;
} else {
primap[-9999]=dbSrv;
}
}
} else {
// define priority for technologies with this server (lower = better)
bool veto=false;
int spri=5; // default for Oracle
if (conn.find("frontier:")!=std::string::npos) {
spri=3; // use frontier before oracle
// dont use frontier servers if disabled, or generic Frontier server
// is specified (via '()' in server definition) and FRONTIER_SERVER
// env variable is not set
if (!m_useFrontier ||
(conn.find("()")!=std::string::npos && m_useFrontierGen==false))
veto=true;
}
// extract the server name (assuming URLs "techno://server/schema") // extract the server name (assuming URLs "techno://server/schema")
// example of current conn naming scheme: coral://127.0.0.1:3320/&oracle://ATLAS_CONFIG/ATLAS_CONF_TRIGGER_REPR // example of current conn naming scheme: coral://127.0.0.1:3320/&oracle://ATLAS_CONFIG/ATLAS_CONF_TRIGGER_REPR
std::string::size_type ipos0=conn.find("&"); std::string::size_type ipos0=conn.find("&");
std::string::size_type ipos1=conn.find("://",ipos0+1); std::string::size_type ipos1=conn.find("://",ipos0+1);
std::string::size_type ipos2=conn.find("/",ipos1+3); std::string::size_type ipos2=conn.find("/",ipos1+3);
if (ipos1!=std::string::npos && ipos2!=std::string::npos) { if (ipos1!=std::string::npos && ipos2!=std::string::npos && !veto) {
const std::string server=conn.substr(ipos1+3,ipos2-ipos1-3); const std::string server=conn.substr(ipos1+3,ipos2-ipos1-3);
// check if this server is on list of replicas to use for domain // check if this server is on list of replicas to use for domain
// if so, add it with its associated priority // if so, add it with its associated priority
for (ServerMap::const_iterator sitr=m_servermap.begin(); for (ServerMap::const_iterator sitr=m_servermap.begin();
sitr!=m_servermap.end();++sitr) { sitr!=m_servermap.end();++sitr) {
if (sitr->first==server) if (sitr->first==server)
primap[sitr->second]=*itr; primap[sitr->second+spri]=dbSrv;
} }
} }
} }
...@@ -86,7 +120,7 @@ TrigConf::ReplicaSorter::readConfig() { ...@@ -86,7 +120,7 @@ TrigConf::ReplicaSorter::readConfig() {
if (cfrontier && strcmp(cfrontier,"")!=0) { if (cfrontier && strcmp(cfrontier,"")!=0) {
std::cout << "Frontier server at " << cfrontier << " will be considered" std::cout << "Frontier server at " << cfrontier << " will be considered"
<< std::endl; << std::endl;
m_frontiergen=true; m_useFrontierGen=true;
} }
// try to locate configuration file using pathresolver // try to locate configuration file using pathresolver
...@@ -128,7 +162,7 @@ TrigConf::ReplicaSorter::readConfig() { ...@@ -128,7 +162,7 @@ TrigConf::ReplicaSorter::readConfig() {
// token is a server name // token is a server name
// only add Frontier ATLF server if FRONTIER_CLIENT set // only add Frontier ATLF server if FRONTIER_CLIENT set
if(atCERN && token=="ATONR_CONF") atCERN = false; if(atCERN && token=="ATONR_CONF") atCERN = false;
if (token!="ATLF" || m_frontiergen) servers.push_back(token); if (token!="ATLF" || m_useFrontierGen) servers.push_back(token);
} }
} }
iofs1=iofs2+1; iofs1=iofs2+1;
...@@ -159,16 +193,13 @@ TrigConf::ReplicaSorter::readConfig() { ...@@ -159,16 +193,13 @@ TrigConf::ReplicaSorter::readConfig() {
} }
} }
if (useit) { if (useit) {
if(atCERN) {
servers.push_back("ATONR_COOL");
servers.push_back("ATONR_CONF");
}
// assign these servers, priority based on position in list // assign these servers, priority based on position in list
// and length of match of domain name // and length of match of domain name
for (unsigned int i=0;i<servers.size();++i) { for (unsigned int i=0;i<servers.size();++i) {
int priority=i-100*bestlen; int priority=i-100*bestlen;
m_servermap.push_back(ServerPair(servers[i],priority)); m_servermap.push_back(ServerPair(servers[i],priority));
} }
break;
} }
} }
} }
......
...@@ -14,8 +14,16 @@ namespace TrigConf { ...@@ -14,8 +14,16 @@ namespace TrigConf {
class ReplicaSorter : virtual public coral::IReplicaSortingAlgorithm { class ReplicaSorter : virtual public coral::IReplicaSortingAlgorithm {
public: public:
ReplicaSorter(); ReplicaSorter();
void sort(std::vector<const coral::IDatabaseServiceDescription*>& void sort(std::vector<const coral::IDatabaseServiceDescription*>& replicaSet);
replicaSet);
void setUseFrontier(bool useFrontier) { m_useFrontier = useFrontier; }
void setUseFrontierGen(bool useFrontierGen) { m_useFrontierGen = useFrontierGen; }
void setUseSQLite(bool useSQLite) { m_useSQLite = useSQLite; }
bool useFrontier() const { return m_useFrontier; }
bool useFrontierGen() const { return m_useFrontierGen; }
bool useSQLite() const { return m_useSQLite; }
private: private:
bool readConfig(); bool readConfig();
FILE* findFile(const std::string filename, const std::string pathvar); FILE* findFile(const std::string filename, const std::string pathvar);
...@@ -23,7 +31,9 @@ namespace TrigConf { ...@@ -23,7 +31,9 @@ namespace TrigConf {
typedef std::pair<std::string,int> ServerPair; typedef std::pair<std::string,int> ServerPair;
typedef std::vector< ServerPair > ServerMap; typedef std::vector< ServerPair > ServerMap;
ServerMap m_servermap; ServerMap m_servermap;
bool m_frontiergen; bool m_useFrontierGen {false};
bool m_useSQLite {true};
bool m_useFrontier {true};
}; };
} }
...@@ -93,17 +93,20 @@ TrigConf::SessionMgr::createSession() { ...@@ -93,17 +93,20 @@ TrigConf::SessionMgr::createSession() {
if(csc.replicaSortingAlgorithm() == nullptr) { // likely to be standalone, create our own if(csc.replicaSortingAlgorithm() == nullptr) { // likely to be standalone, create our own
TRG_MSG_INFO("Create own ReplicaSortingAlgorithm"); TRG_MSG_INFO("Create own ReplicaSortingAlgorithm with useFrontier=" << (m_useFrontier ? "true" : "false") << " and useSQLite=" << ( m_useSQLite ? "true" : "false"));
m_replicaSorter = new TrigConf::ReplicaSorter(); m_replicaSorter = new TrigConf::ReplicaSorter();
m_replicaSorter->setUseFrontier(m_useFrontier);
m_replicaSorter->setUseSQLite(m_useSQLite);
csc.setReplicaSortingAlgorithm(*m_replicaSorter); csc.setReplicaSortingAlgorithm(*m_replicaSorter);
} }
buildConnectionString(); buildConnectionString();
TRG_MSG_INFO("Connecting to " << m_connectionString); TRG_MSG_INFO("Connecting to " << m_connectionString);
m_sessionproxy = connSvc.connect(m_connectionString, coral::AccessMode::ReadOnly); m_sessionproxy = connSvc.connect(m_connectionString, coral::AccessMode::ReadOnly);
TRG_MSG_INFO("Opening session " << m_connectionString << " with " << m_retrialPeriod << "/" << m_retrialTimeout << "/" << m_connectionTimeout); TRG_MSG_INFO("Opening session " << m_connectionString << " with " << m_retrialPeriod << "/" << m_retrialTimeout << "/" << m_connectionTimeout);
return *m_sessionproxy; return *m_sessionproxy;
} }
......
...@@ -763,9 +763,9 @@ int main( int argc, char* argv[] ) { ...@@ -763,9 +763,9 @@ int main( int argc, char* argv[] ) {
} }
try { try {
std::unique_ptr< TrigConf::IStorageMgr > sm = std::unique_ptr< TrigConf::StorageMgr > sm =
std::unique_ptr< TrigConf::IStorageMgr >(new TrigConf::StorageMgr(csv[0],user,passwd,log)); std::unique_ptr< TrigConf::StorageMgr >(new TrigConf::StorageMgr(csv[0],user,passwd,log));
sm->setUseFrontier(gConfig.UseFrontier());
// setup the coolWriter // setup the coolWriter
TrigConf::TrigConfCoolWriter coolWriter(gConfig.CoolConnection(),log); TrigConf::TrigConfCoolWriter coolWriter(gConfig.CoolConnection(),log);
std::vector<std::string>::const_iterator wfIt = gConfig.ListOfWriteFolders().begin(); std::vector<std::string>::const_iterator wfIt = gConfig.ListOfWriteFolders().begin();
......
...@@ -41,6 +41,7 @@ namespace TrigConf { ...@@ -41,6 +41,7 @@ namespace TrigConf {
int m_dbHLTPSKey { 0 }; int m_dbHLTPSKey { 0 };
std::string m_dbHLTPSKeySet { "[]" }; std::string m_dbHLTPSKeySet { "[]" };
BooleanProperty m_useFrontier { false }; BooleanProperty m_useFrontier { false };
BooleanProperty m_useSQLite { true };
int m_printMenuLevel { 1 }; int m_printMenuLevel { 1 };
std::unique_ptr<TrigDBConnectionConfig> m_dbconfig; std::unique_ptr<TrigDBConnectionConfig> m_dbconfig;
......
...@@ -39,10 +39,10 @@ namespace TrigConf { ...@@ -39,10 +39,10 @@ namespace TrigConf {
using AthService::AthMessaging::msg; using AthService::AthMessaging::msg;
// implementing IIHLTConfigSvc // implementing IIHLTConfigSvc
const HLTChainList* chainList() const __attribute__ ((deprecated)); const HLTChainList* chainList() const;
const HLTChainList& chains() const; const HLTChainList& chains() const;
const HLTSequenceList* sequenceList() const __attribute__ ((deprecated)); const HLTSequenceList* sequenceList() const;
const HLTSequenceList& sequences() const; const HLTSequenceList& sequences() const;
/*@brief constructor*/ /*@brief constructor*/
......
...@@ -307,9 +307,12 @@ def getUsedTables(output, condition, schemaname, tables): ...@@ -307,9 +307,12 @@ def getUsedTables(output, condition, schemaname, tables):
def isRun2(cursor,schemaname): def isRun2(cursor,schemaname):
import cx_Oracle try:
if not hasattr(cursor,'connection') or type(cursor.connection)!=cx_Oracle.Connection: import cx_Oracle
log.warning('Detection of DB schema only supported for Oracle. Will assume run-2') if not hasattr(cursor,'connection') or type(cursor.connection)!=cx_Oracle.Connection:
log.warning('Detection of DB schema only supported for Oracle. Will assume run-2')
return True
except ImportError:
return True return True
owner = schemaname.rstrip('.') owner = schemaname.rstrip('.')
......
...@@ -42,6 +42,8 @@ ConfigSvcBase::declareCommonProperties() { ...@@ -42,6 +42,8 @@ ConfigSvcBase::declareCommonProperties() {
"The SuperMaster key"); "The SuperMaster key");
declareProperty( "UseFrontier", m_useFrontier, declareProperty( "UseFrontier", m_useFrontier,
"Tries to use Frontier for accessing the TriggerDB"); "Tries to use Frontier for accessing the TriggerDB");
declareProperty( "UseSQLite", m_useSQLite,
"Tries to use SQLite for accessing the TriggerDB if available");
declareProperty( "PrintMenu", m_printMenuLevel, declareProperty( "PrintMenu", m_printMenuLevel,
"Prints menu with detail level x=0..5 [default = " + boost::lexical_cast<string,int>(m_printMenuLevel) + "]"); "Prints menu with detail level x=0..5 [default = " + boost::lexical_cast<string,int>(m_printMenuLevel) + "]");
} }
...@@ -108,7 +110,7 @@ ConfigSvcBase::initStorageMgr() { ...@@ -108,7 +110,7 @@ ConfigSvcBase::initStorageMgr() {
StorageMgr * sm = new StorageMgr( connectionString, m_dbconfig->m_user, m_dbconfig->m_password ); StorageMgr * sm = new StorageMgr( connectionString, m_dbconfig->m_user, m_dbconfig->m_password );
sm->setUseFrontier(m_dbconfig->m_useFrontier); sm->setUseFrontier(m_dbconfig->m_useFrontier);
sm->setUseSQLite(m_useSQLite);
sm->setRetrialPeriod( m_dbconfig->m_retrialPeriod ); sm->setRetrialPeriod( m_dbconfig->m_retrialPeriod );
sm->setRetrialTimeout( m_dbconfig->m_retrialPeriod * (m_dbconfig->m_maxRetrials + 1) ); sm->setRetrialTimeout( m_dbconfig->m_retrialPeriod * (m_dbconfig->m_maxRetrials + 1) );
sm->setConnectionTimeout( 0 ); sm->setConnectionTimeout( 0 );
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment