Commit 72f99250 authored by Joerg Stelzer's avatar Joerg Stelzer Committed by Graeme Stewart
Browse files

Fixes the setting of prescales from COOL (TrigConfStorage-03-01-11)

	* Fixes the setting of prescales from COOL
	* TrigConfStorage-03-01-11

2015-08-25 Joerg Stelzer
	* MenuLoader.cxx: further improvement of TIPs loading
	when no direct inputs are listed in the TIP mapping
	then they are taken from the list of direct thresholds
	* TrigConfStorage-03-01-10

2015-08-24 Joerg Stelzer
	* MenuLoader.cxx: prepare for loading of TIPs of direct input
	* TrigConfCool*: prepare for loading of TIPs of direct input from COOL
	* TrigConfStorage-03-01-09

2015-08-12 Joerg Stelzer
	* Setting of bg idx when loading of bunchgroups from cool, fixes bgCode calculation
	* TrigConfStorage-03-01-08

2015-07-05 Joerg Stelzer
	* Fix setting of input type correctly (ctpin or ctpcore ) when reading from COOL
...
(Long ChangeLog diff - truncated)
parent 23bc818c
......@@ -128,7 +128,9 @@ namespace TrigConf {
int readLvl1BGKey( const coral::AttributeList & al);
/** @brief Build a LVL1 PIT object from COOL payload */
TrigConf::PIT* readLvl1InputMap( const coral::AttributeList & al);
TrigConf::PIT* readLvl1InputMap( const coral::AttributeList & al); // backward compatible
TrigConf::TIP* readLvl1TIPMap( const coral::AttributeList & al);
/** @brief Build a LVL1 mon counter object from COOL payload */
std::vector<TrigConf::ThresholdMonitor*> readLvl1MonMap( const cool::IRecord & payload);
......
......@@ -21,6 +21,7 @@ namespace TrigConf {
class CTPConfig;
class Menu;
class PIT;
class TIP;
class CaloInfo;
class HLTFrame;
class PrescaleSet;
......@@ -313,8 +314,9 @@ namespace TrigConf {
* @param runNumber run number
* @param pits Vector of PIT object to be filled
*/
void readL1InputMapPayload(unsigned int runNumber,
std::vector<TrigConf::PIT*>& pits);
void readL1InputMapPayload( unsigned int runNumber,
std::vector<TrigConf::PIT*>& pits,
std::vector<TrigConf::TIP*>& tips);
/**@brief Read run-wise LVL1 monitoring configuration from COOL database
*
......
......@@ -107,17 +107,25 @@ TrigConf::HLTChainLoader::loadChains( HLTChainList& chainlist ) {
int version = row["TC.HTC_VERSION"].data<int>();
string name = rmtilde(row["TC.HTC_NAME"].data<string>());
string lower_chain_name = rmtilde(row["TC.HTC_LOWER_CHAIN_NAME"].data<string>());
// level
string level = "HLT";
float rerunps = 0;
if(isRun1()) {
level = rmtilde(row["TC.HTC_L2_OR_EF"].data<string>());
level = rmtilde(row["TC.HTC_L2_OR_EF"].data<string>());
if(level=="HL") level="HLT";
string rerunps_s = rmtilde(row["TC.HTC_RERUN_PRESCALE"].data<string>());
rerunps = boost::lexical_cast<float,string>(rerunps_s);
}
HLTChain* ch = new HLTChain( name, counter, version, level, lower_chain_name, 0, std::vector<HLTSignature*>() );
if(rerunps>=0)
ch->set_rerun_prescale(rerunps);
// rerun ps from the chain table only in run 1
if(isRun1()) {
float rerunps = 0;
string rerunps_s = rmtilde(row["TC.HTC_RERUN_PRESCALE"].data<string>());
rerunps = boost::lexical_cast<float,string>(rerunps_s);
if(rerunps>=0)
ch->set_rerun_prescale(rerunps);
}
ch->setId(chainId);
chainlist.addHLTChain(ch);
}
......
......@@ -162,7 +162,7 @@ TrigConf::HLTPrescaleSetLoader::load( HLTPrescaleSet& hltpss ) {
string pstype = row["PR.HPR_TYPE"].data<string>();
string pscondition = row["PR.HPR_CONDITION"].data<string>();
//cout << setw(4) << right << counter << " : " << pstype << " (" << pscondition << ") => " << value << endl;
TRG_MSG_DEBUG("Loaded prescales:"<< setw(4) << right << counter << " : " << pstype << " (" << pscondition << ") => " << value );
auto& thePS = hltpss.thePrescale( counter, str2lvl("HLT") );
......
......@@ -90,7 +90,7 @@ TrigConf::JobOptionTableLoader::loadHLTMasterTable(int SuperMasterKey,
bool
TrigConf::JobOptionTableLoader::loadComponentNamesAndParameters(TrigConf::JobOptionTable& JOTable, const std::set<int>& compIDsToLoad) {
unsigned int batchSize = 1600;
unsigned int batchSize = 500;
set<int>::const_iterator current = compIDsToLoad.begin();
vector< SplitParam > splitparams;
......@@ -116,7 +116,7 @@ JobOptionTableLoader::loadComponentNamesAndParameters(JobOptionTable& JOTable,
unsigned int batchSize) {
unique_ptr< coral::IQuery > q( m_session.nominalSchema().newQuery() );
q->setRowCacheSize( 1000 );
q->setRowCacheSize( 500 );
q->addToTableList ( "HLT_COMPONENT", "CP" );
q->addToTableList ( "HLT_CP_TO_PA", "CP2PA" );
......@@ -130,7 +130,7 @@ JobOptionTableLoader::loadComponentNamesAndParameters(JobOptionTable& JOTable,
cond += " AND CP2PA.HCP2PA_COMPONENT_ID = CP.HCP_ID";
cond += " AND CP2PA.HCP2PA_PARAMETER_ID = PA.HPA_ID";
q->setCondition( cond , bind );
// output and their types
coral::AttributeList attList;
attList.extend<std::string>( "CP.HCP_ALIAS" );
......@@ -288,52 +288,89 @@ TrigConf::JobOptionTableLoader::buildCond_IN_(const std::string& field, const st
}
std::set<int>
TrigConf::JobOptionTableLoader::getChildCompIDs(const std::set<int>& compIDs) {
namespace {
vector<int> createSortedVector(const set<int>& compIDs) {
vector<int> sortedVector(compIDs.size());
copy(compIDs.begin(), compIDs.end(), sortedVector.begin());
sort(sortedVector.begin(),sortedVector.end());
return sortedVector;
}
std::set<int> childCompIDs;
vector<string> concatAndSplit(const vector<int>& v, uint maxStringSize) {
// create a single long string of comma-separated numbers
vector<string> split;
string s("");s.reserve(8000);
uint count(0);
coral::ITable& table = m_session.nominalSchema().tableHandle("HLT_CP_TO_CP");
std::unique_ptr< coral::IQuery > q( table.newQuery() );
for(int i:v) {
s+=to_string(i) + ",";
count++;
if(count<1000 && s.size()<maxStringSize ) // oracle allows max 1000 items in (list)
continue;
// condition
std::stringstream ss;
ss << "HCP2CP_PARENT_COMP_ID IN (";
s.erase(s.end()-1); // erase last comma
int count=0;
std::set<int>::iterator id = compIDs.begin();
std::set<int>::iterator last = compIDs.end(); last--;
for(; id!=last; ++id) {
ss << *id;
if(++count==1000) {
ss << ") OR HCP2CP_PARENT_COMP_ID IN (";
split.push_back( s );
s="";
count=0;
} else {
ss << ",";
}
// the last entries in the loop
if(s!="") {
s.erase(s.end()-1); // erase last comma
split.push_back( s );
}
return split;
}
ss << *last << ")";
string cond(ss.str());
coral::AttributeList bind;
q->setCondition( cond, bind );
}
// output
coral::AttributeList attList;
attList.extend<int>( "HCP2CP_CHILD_COMP_ID" );
fillQuery(q.get(),attList);
std::set<int>
TrigConf::JobOptionTableLoader::getChildCompIDs(const std::set<int>& compIDs) {
// distinct
q->setDistinct();
std::set<int> childCompIDs; // the children IDs
TRG_MSG_DEBUG("Number of parents " << compIDs.size());
vector<int> sortedParentIDs = createSortedVector(compIDs);
vector<string> splitParentIDs = concatAndSplit(sortedParentIDs, 7500);
for(const string& idList : splitParentIDs) {
// condition
string cond = "HCP2CP_PARENT_COMP_ID IN (" + idList + ")";
coral::ITable& table = m_session.nominalSchema().tableHandle("HLT_CP_TO_CP");
std::unique_ptr< coral::IQuery > q( table.newQuery() );
coral::AttributeList bind;
q->setCondition( cond, bind );
// output
coral::AttributeList attList;
attList.extend<int>( "HCP2CP_CHILD_COMP_ID" );
fillQuery(q.get(),attList);
// distinct
q->setDistinct();
// execute
coral::ICursor& cursor = q->execute();
// fill output
while( cursor.next() ) {
const coral::AttributeList& row = cursor.currentRow();
int id = row["HCP2CP_CHILD_COMP_ID"].data<int>();
childCompIDs.insert(id);
}
coral::ICursor& cursor = q->execute();
while( cursor.next() ) {
const coral::AttributeList& row = cursor.currentRow();
int id = row["HCP2CP_CHILD_COMP_ID"].data<int>();
childCompIDs.insert(id);
}
// no more children, can stop recursion
......@@ -357,14 +394,8 @@ TrigConf::JobOptionTableLoader::load( TrigConfData& data ) {
try {
return load(jot);
}
catch( const coral::SchemaException& e ) {
msg() << "JobOptionTableLoader: ERROR SchemaException: " << e.what() << std::endl;
throw;
} catch( const std::exception& e ) {
msg() << "JobOptionTableLoader: ERROR std::exception: " << e.what() << std::endl;
throw;
} catch( ... ) {
msg() << "JobOptionTableLoader: ERROR any other exception" << std::endl;
catch( const std::exception& e ) {
TRG_MSG_FATAL("std::exception: " << e.what());
throw;
}
return false;
......@@ -386,7 +417,7 @@ TrigConf::JobOptionTableLoader::assembleSplitParameters( JobOptionTable& jot, co
std::string aliasname ="";
std::string paraname ="";
//ignore those we have already matched
vector< SplitParam > alreadymatched_v;
......@@ -431,10 +462,8 @@ TrigConf::JobOptionTableLoader::assembleSplitParameters( JobOptionTable& jot, co
if(actualname2==actualname && splitpar.alias==newtemp2.alias ){
mergedvalue+= newtemp2.value;
if(verbose()>1) {
msg() << "Found split parameters " << splitpar.alias << " : " << splitpar.name << " - " << newtemp2.name << std::endl;
msg() << splitpar.value << " - " << newtemp2.value << std::endl;
}
TRG_MSG_VERBOSE("Found split parameters " << splitpar.alias << " : " << splitpar.name << " - " << newtemp2.name);
TRG_MSG_VERBOSE(splitpar.value << " - " << newtemp2.value);
}
}
}
......@@ -526,8 +555,7 @@ TrigConf::JobOptionTableLoader::load( JobOptionTable& jot ) {
triggerDBSchemaVersion();
if(verbose())
TRG_MSG_INFO("Start loading job options with smk " << jot.smk());
TRG_MSG_INFO("Start loading job options with smk " << jot.smk());
startSession();
......@@ -548,8 +576,12 @@ TrigConf::JobOptionTableLoader::load( JobOptionTable& jot ) {
jot.setHltMasterTableId( masterTableID );
TRG_MSG_INFO("HLT masterkey: " << jot.hltMasterTableId());
TRG_MSG_INFO("Menu ID : " << triggerMenuID);
TRG_MSG_INFO("L2 Setup ID : " << l2SetupID);
TRG_MSG_INFO("EF Setup ID : " << efSetupID);
if(l2SetupID==efSetupID) {
TRG_MSG_INFO("Setup ID : " << l2SetupID);
} else {
TRG_MSG_INFO("L2 Setup ID : " << l2SetupID);
TRG_MSG_INFO("EF Setup ID : " << efSetupID);
}
int level = jot.triggerLevel()==0; // triggerLevel: 0 - L2, 1 - EF, 2 - HLT(combined L2+EF)
if( l2SetupID == efSetupID ||
......@@ -578,8 +610,8 @@ TrigConf::JobOptionTableLoader::load( JobOptionTable& jot ) {
commitSession();
// some stats:
msg() << "JobOptionTableLoader: Total parameters : " << jot.jobOptionVector().size() << std::endl;
TRG_MSG_INFO("JobOptionTableLoader: Total parameters : " << jot.jobOptionVector().size());
return true;
}
......
......@@ -47,9 +47,10 @@ bool TrigConf::L1TopoMenuLoader::load( TXC::L1TopoMenu& tcaTarget ) {
return true;
}
bool TrigConf::L1TopoMenuLoader::loadTopoAlgos( TXC::L1TopoMenu& tcaTarget) {
TRG_MSG_VERBOSE("Calling loadTopoAlgos");
TRG_MSG_VERBOSE("Calling loadTopoAlgos for SMK " << tcaTarget.getSMK());
try {
startSession();
......@@ -60,7 +61,7 @@ bool TrigConf::L1TopoMenuLoader::loadTopoAlgos( TXC::L1TopoMenu& tcaTarget) {
query0->addToTableList("TOPO_TRIGGER_MENU","TTM");
query0->addToTableList("TTM_TO_TA","TTM2TA");
query0->addToTableList("TOPO_ALGO","TA");
query0->setRowCacheSize(5);
query0->setRowCacheSize(1000);
//Bind list
coral::AttributeList bindList0;
......@@ -77,10 +78,12 @@ bool TrigConf::L1TopoMenuLoader::loadTopoAlgos( TXC::L1TopoMenu& tcaTarget) {
query0->setCondition( theCondition0, bindList0 );
coral::AttributeList attList0;
attList0.extend<int>( "TMT.TMT_TRIGGER_MENU_ID" );
attList0.extend<long>( "TMT.TMT_VERSION" );
//TMT.TMT_TRIGGER_MENU_ID, TMT.TMT_VERSION,TTM.TTM_NAME,TTM.TTM_VERSION,TTM.TTM_CTPLINK_ID,TA.TA_ID,TA.TA_NAME,TA.TA_OUTPUT,TA.TA_TYPE,TA.TA_BITS,TA.TA_SORT_DECI,TA.TA_ALGO_ID
//attList0.extend<int>( "TMT.TMT_TRIGGER_MENU_ID" );
//attList0.extend<long>( "TMT.TMT_VERSION" );
attList0.extend<string>( "TTM.TTM_NAME" );
attList0.extend<int>( "TTM.TTM_VERSION" );
//attList0.extend<int>( "TTM.TTM_VERSION" );
attList0.extend<int>( "TTM.TTM_CTPLINK_ID" );
attList0.extend<int>( "TA.TA_ID" );
attList0.extend<string>( "TA.TA_NAME" );
......@@ -95,13 +98,16 @@ bool TrigConf::L1TopoMenuLoader::loadTopoAlgos( TXC::L1TopoMenu& tcaTarget) {
coral::ICursor& cursor0 = query0->execute();
int ctplinkid = 0;
std::map<uint, TXC::L1TopoConfigAlg> idToAlgMap; // map of id -> alg for later bulk loading of inputs, outputs, etc.
while(cursor0.next()){
const coral::AttributeList& row0 = cursor0.currentRow();
int ta_id = row0["TA.TA_ID"].data<int>();
string ta_name = row0["TA.TA_NAME"].data<string>();
string ta_output = row0["TA.TA_OUTPUT"].data<string>();
string ta_type = row0["TA.TA_TYPE"].data<string>();
int ta_bits = row0["TA.TA_BITS"].data<int>();
//int ta_bits = row0["TA.TA_BITS"].data<int>();
string ta_sort_deci = row0["TA.TA_SORT_DECI"].data<string>();
int ta_algo_id = row0["TA.TA_ALGO_ID"].data<int>();
ctplinkid = row0["TTM.TTM_CTPLINK_ID"].data<int>();
......@@ -115,11 +121,21 @@ bool TrigConf::L1TopoMenuLoader::loadTopoAlgos( TXC::L1TopoMenu& tcaTarget) {
else alg.setAlgKind(TXC::L1TopoConfigAlg::NONE);
alg.setAlgOutput(ta_output);
alg.setAlgoID(ta_algo_id);
loadAlgInput(alg,ta_id);
loadAlgOutput(alg,ta_id,ta_bits);
loadAlgRegister(alg, ta_id);
loadAlgFixed(alg, ta_id);
tcaTarget.addAlgorithm(alg);
// loadAlgInput(alg,ta_id);
// loadAlgOutput(alg,ta_id,ta_bits);
// loadAlgRegister(alg, ta_id);
// loadAlgFixed(alg, ta_id);
idToAlgMap.insert(make_pair(ta_id,alg));
// tcaTarget.addAlgorithm(alg);
}
loadAllAlgsInput(idToAlgMap);
loadAllAlgsOutput(idToAlgMap);
loadAllAlgsRegister(idToAlgMap);
loadAllAlgsFixed(idToAlgMap);
for( auto & e : idToAlgMap ) {
tcaTarget.addAlgorithm(e.second);
}
loadTopoConfig(tcaTarget);
......@@ -132,6 +148,304 @@ bool TrigConf::L1TopoMenuLoader::loadTopoAlgos( TXC::L1TopoMenu& tcaTarget) {
return true;
}
bool TrigConf::L1TopoMenuLoader::loadAllAlgsInput( std::map<uint, TXC::L1TopoConfigAlg>& idToAlgMap ) {
TRG_MSG_VERBOSE("Calling loadAllAlgsInput");
set<uint> algIds;
for( auto & e : idToAlgMap ) {
algIds.insert(e.first);
}
string algIdsConc = "(";
bool first = true;
for( int id : algIds ) {
if(first) { first = false; } else { algIdsConc += ","; }
algIdsConc += to_string(id);
}
algIdsConc+=")";
TRG_MSG_DEBUG("" << algIds.size() << " algorithms ");
TRG_MSG_VERBOSE("Alg IDs : " << algIdsConc);
try {
startSession();
unique_ptr<coral::IQuery> query0(m_session.nominalSchema().newQuery());
query0->addToTableList("TA_TO_TI","TA2TI");
query0->addToTableList("TOPO_ALGO_INPUT","TAI");
coral::AttributeList bindList0;
string theCondition0 = "";
theCondition0 += " TA2TI.TA2TI_ALGO_ID IN " + algIdsConc;
theCondition0 += string(" AND TA2TI.TA2TI_INPUT_ID = TAI.TAI_ID");
query0->setCondition( theCondition0, bindList0 );
coral::AttributeList attList0;
attList0.extend<int>( "TA2TI.TA2TI_ALGO_ID" );
attList0.extend<string>( "TAI.TAI_NAME" );
attList0.extend<string>( "TAI.TAI_VALUE" );
attList0.extend<int>( "TAI.TAI_POSITION" );
query0->defineOutput(attList0);
fillQuery(query0.get(), attList0);
//std::cout << "Executing loadAlgInput query " << std::endl;
coral::ICursor& cursor0 = query0->execute();
uint inputCount = 0;
while(cursor0.next()){
inputCount ++;
const coral::AttributeList& row0 = cursor0.currentRow();
//std::cout << "loadAlgInput my row " << row0 << std::endl;
string tai_name = row0["TAI.TAI_NAME"].data<string>();
string tai_value = row0["TAI.TAI_VALUE"].data<string>();
int tai_pos = row0["TAI.TAI_POSITION"].data<int>();
if(tai_pos<0) tai_pos=0; // when position is missing in XML, currently the TT uploads -1 instead of 0
// fill the alg
int algoId = row0["TA2TI.TA2TI_ALGO_ID"].data<int>();
auto match = idToAlgMap.find(algoId);
TXC::L1TopoConfigAlg & alg = match->second;
alg.addInput(tai_name,tai_value,tai_pos);
}
TRG_MSG_DEBUG("" << inputCount << " inputs found");
} catch (const exception& e){
TRG_MSG_ERROR("loadAlgInput >> Standard C++ exception: " << e.what());
throw;
}
return true;
}
bool
TrigConf::L1TopoMenuLoader::loadAllAlgsOutput( std::map<uint, TXC::L1TopoConfigAlg> & idToAlgMap ) {
TRG_MSG_VERBOSE("Calling loadAllAlgsOutput");
set<uint> algIds;
for( auto & e : idToAlgMap ) {
algIds.insert(e.first);
}
string algIdsConc = "(";
bool first = true;
for( int id : algIds ) {
if(first) { first = false; } else { algIdsConc += ","; }
algIdsConc += to_string(id);
}
algIdsConc+=")";
try {
startSession();
unique_ptr<coral::IQuery> query0(m_session.nominalSchema().newQuery());
query0->addToTableList("TOPO_ALGO","TA");
query0->addToTableList("TA_TO_TO","TA2TO");
query0->addToTableList("TOPO_ALGO_OUTPUT","TAO");
//Bind list
coral::AttributeList bindList0;
// bindList0.extend<int>("taId");
// bindList0[0].data<int>() = ta_id;
string theCondition0 = "";
theCondition0 += " TA.TA_ID IN " + algIdsConc;
theCondition0 += " AND TA.TA_ID = TA2TO.TA2TO_ALGO_ID";
theCondition0 += " AND TA2TO.TA2TO_OUTPUT_ID = TAO.TAO_ID";
query0->setCondition( theCondition0, bindList0 );
coral::AttributeList attList0;
attList0.extend<int>( "TA2TO.TA2TO_ALGO_ID" );
attList0.extend<string>( "TAO.TAO_NAME" );
attList0.extend<string>( "TAO.TAO_VALUE" );
attList0.extend<string>( "TAO.TAO_BITNAME" );
attList0.extend<int>( "TAO.TAO_SELECTION" );
attList0.extend<int>( "TA.TA_BITS" );
query0->defineOutput(attList0);
fillQuery(query0.get(), attList0);
//std::cout << "Executing loadAlgOutput query " << std::endl;
coral::ICursor& cursor0 = query0->execute();
uint outputCount = 0;
while(cursor0.next()){
outputCount ++;
const coral::AttributeList& row0 = cursor0.currentRow();
//std::cout << "loadAlgOutput my row " << row0 << std::endl;
string tao_name = row0["TAO.TAO_NAME"].data<string>();
string tao_value = row0["TAO.TAO_VALUE"].data<string>();
string tao_bitname = row0["TAO.TAO_BITNAME"].data<string>();
int tao_sel = row0["TAO.TAO_SELECTION"].data<int>();
int ta_bits = row0["TA.TA_BITS"].data<int>();
// fill the alg
int algoId = row0["TA2TO.TA2TO_ALGO_ID"].data<int>();
auto match = idToAlgMap.find(algoId);
TXC::L1TopoConfigAlg & alg = match->second;
alg.addOutput(tao_name,tao_value,ta_bits,tao_bitname, tao_sel);
}
TRG_MSG_DEBUG("" << outputCount << " outputs found");
} catch (const exception& e){
TRG_MSG_ERROR("loadAlgOutput >> Standard C++ exception: " << e.what());
throw;
}
return true;
}
bool
TrigConf::L1TopoMenuLoader::loadAllAlgsRegister( std::map<uint, TXC::L1TopoConfigAlg> & idToAlgMap ) {
TRG_MSG_VERBOSE("Calling loadAllAlgsRegister");
set<uint> algIds;
for( auto & e : idToAlgMap ) {
algIds.insert(e.first);
}
string algIdsConc = "(";
bool first = true;
for( int id : algIds ) {
if(first) { first = false; } else { algIdsConc += ","; }
algIdsConc += to_string(id);
}
algIdsConc+=")";
try {
startSession();
unique_ptr<coral::IQuery> query0(m_session.nominalSchema().newQuery());
query0->addToTableList("TA_TO_TP","TA2TP");
query0->addToTableList("TOPO_PARAMETER","TP");
//Bind list
coral::AttributeList bindList0;
// bindList0.extend<int>("taId");
// bindList0[0].data<int>() = ta_id;
string theCondition0 = "";
theCondition0 += " TA2TP.TA2TP_ALGO_ID IN " + algIdsConc;
theCondition0 += " AND TA2TP.TA2TP_PARAM_ID = TP.TP_ID";
query0->setCondition( theCondition0, bindList0 );
coral::AttributeList attList0;
attList0.extend<int>( "TA2TP.TA2TP_ALGO_ID" );
attList0.extend<string>( "TP.TP_NAME" );
attList0.extend<long>( "TP.TP_VALUE" );
attList0.extend<int>( "TP.TP_POSITION" );
attList0.extend<int>( "TP.TP_SELECTION" );
query0->defineOutput(attList0);
fillQuery(query0.get(), attList0);
//std::cout << "Executing loadAlgRegister query " << std::endl;