Commit 85f67b07 authored by Peter Berta's avatar Peter Berta Committed by Atlas Nightlybuild
Browse files

Merge branch 'propagat_TE_data' into '22.0-mc20'

[ATR-24058][ATR-24141] Working to add R2 trigger TE data to R3 JSON structure to enable feature access from R22 AODs created from R2 data

See merge request atlas/athena!46419

(cherry picked from commit 89442554)

6a3c59d7 working to add R2 TE data to R3 JSON structure
b9133381 Add standalone signatures data
86dc0214 do not copy seq list, causes segfault on destruct
f09bf609 Change RAWtoALL/ESD/AOD slimming of R2 trigger navigation structures. Remove...
parent 7481564f
......@@ -20,20 +20,15 @@ if DQMonFlags.useTrigger():
from TriggerJobOpts.TriggerConfigGetter import TriggerConfigGetter
cfg = TriggerConfigGetter()
if not hasattr(ToolSvc, DQMonFlags.nameTrigDecTool().split('/')[-1]):
if rec.doTrigger():
tdt_local_logger.error('DQ Monitoring is being asked to set up the TrigDecisionTool for some reason. THIS IS A TERRIBLE IDEA AND SHOULD BE CONSIDERED A BUG!')
from AthenaMonitoring.TriggerInterface import getTrigDecisionTool
from AthenaConfiguration.OldFlags2NewFlags import getNewConfigFlags
# Translate all needed flags from old jobProperties to a new AthConfigFlag Container
ConfigFlags = getNewConfigFlags()
from AthenaConfiguration import ComponentAccumulator
ComponentAccumulator.CAtoGlobalWrapper(getTrigDecisionTool, ConfigFlags)
monTrigDecTool = getattr(ToolSvc, 'TrigDecisionTool')
else:
monTrigDecTool = getattr(ToolSvc, DQMonFlags.nameTrigDecTool().split('/')[-1])
from AthenaCommon.Configurable import Configurable
Configurable.configurableRun3Behavior+=1
from TrigDecisionTool.TrigDecisionToolConfig import getTrigDecisionTool
tdtAcc = getTrigDecisionTool(ConfigFlags)
Configurable.configurableRun3Behavior-=1
from AthenaConfiguration.ComponentAccumulator import conf2toConfigurable, appendCAtoAthena
monTrigDecTool = conf2toConfigurable(tdtAcc.getPrimary())
appendCAtoAthena( tdtAcc )
tdt_local_logger.info('Scheduled monitoring TDT %s', monTrigDecTool)
tdt_local_logger.info('Scheduling the trigger translator')
......
......@@ -18,9 +18,36 @@ TrigConf::JsonFileWriterHLT::JsonFileWriterHLT() :
{}
/// Helper function ptree key->[] to std::vector<T>
template<typename T>
std::vector<T> ToVector(const TrigConf::DataStructure& ds, const std::string& child){
using ptree = boost::property_tree::ptree;
std::vector<T> return_vector;
for( const ptree::value_type& entry : ds.data().get_child(child) ) {
return_vector.push_back( entry.second.get_value<T>() );
}
return return_vector;
}
/// Helper function ptree key->[[]] to std::vector<std::vector<T>>
template<typename T>
std::vector<std::vector<T>> ToVectorVector(const TrigConf::DataStructure& ds, const std::string& child){
using ptree = boost::property_tree::ptree;
std::vector<std::vector<T>> return_vector;
for( const ptree::value_type& outer : ds.data().get_child(child) ) {
return_vector.push_back(std::vector<T>());
for (const ptree::value_type& inner : outer.second) {
return_vector.back().push_back( inner.second.get_value<T>() );
}
}
return return_vector;
}
bool
TrigConf::JsonFileWriterHLT::writeJsonFile(const std::string & filename, const HLTMenu & menu) const
{
using ptree = boost::property_tree::ptree;
json chains({});
for ( const auto & chain : menu ) {
json jChain({});
......@@ -32,6 +59,17 @@ TrigConf::JsonFileWriterHLT::writeJsonFile(const std::string & filename, const H
jChain["groups"] = chain.groups();
jChain["streams"] = chain.streams();
jChain["seqeuncers"] = chain.sequencers();
// Optional Run2 payload
if (chain.hasChild("signature")) {
json jSig({});
jSig["counters"] = ToVector<uint32_t>(chain, "signature.counters");
jSig["logics"] = ToVector<int>(chain, "signature.logics");
jSig["labels"] = ToVector<std::string>(chain, "signature.labels");
jSig["outputTEs"] = ToVectorVector<std::string>(chain, "signature.outputTEs");
jChain["signature"] = jSig;
}
chains[chain.name()] = jChain;
}
......@@ -58,6 +96,14 @@ TrigConf::JsonFileWriterHLT::writeJsonFile(const std::string & filename, const H
j["sequencers"] = sequencers;
j["streams"] = streams;
// Optional Run2 payload
if (menu.hasChild("sequence_run2")) {
json jSequence({});
jSequence["outputTEs"] = ToVector<std::string>(menu, "sequence_run2.outputTEs");
jSequence["inputTEs"] = ToVectorVector<std::string>(menu, "sequence_run2.inputTEs");
jSequence["algorithms"] = ToVectorVector<std::string>(menu, "sequence_run2.algorithms");
j["sequence_run2"] = jSequence;
}
std::ofstream outfile(filename);
outfile << std::setw(4) << j << std::endl;
......
......@@ -81,6 +81,35 @@ bool convertHLTMenu(const TrigConf::HLTFrame* frame, TrigConf::HLTMenu& menu) {
pChain.add_child("groups", asArray(cptr->groups()));
// Signature data
// Note: This is run-2 only.
// It is propagated here to allow legacy trigger feature access.
std::vector<uint32_t> counters;
std::vector<int> logics;
std::vector<std::string> labels;
ptree outputTEs_outerArray; // outputTEs is a std::vector<std::vector<std::string>>
for(auto& signature : cptr->signatureList() ){
uint32_t cntr = signature->signature_counter();
counters.push_back(cntr);
logics.push_back(signature->logic());
labels.push_back(signature->label());
ptree outputTEs_innerArray;
for(auto& outputTE : signature->outputTEs()){
outputTEs_innerArray.push_back( ptree::value_type("", outputTE->name()) );
}
outputTEs_outerArray.push_back( ptree::value_type("", outputTEs_innerArray) );
}
ptree pSig;
pSig.add_child("counters", asArray(counters));
pSig.add_child("logics", asArray(logics));
pSig.add_child("outputTEs", outputTEs_outerArray);
pSig.add_child("labels", asArray(labels));
pChain.add_child("signature", pSig);
// End of signature data
pChains.push_back(std::make_pair(cptr->chain_name(), pChain));
}
ptree pStreams;
......@@ -100,6 +129,32 @@ bool convertHLTMenu(const TrigConf::HLTFrame* frame, TrigConf::HLTMenu& menu) {
pSequencers.add_child("missing", asArray(std::vector<std::string>({""})));
top.add_child("sequencers", pSequencers);
// Set run2 sequence information:
const TrigConf::HLTSequenceList& sequenceList = frame->getHLTSequenceList();
std::vector<std::string> outputTEs;
ptree inputTEs_outerArray; // sequenceInputTEs is a std::vector<std::vector<std::string>>
ptree algorithms_outerArray; // sequenceAlgorithms is a std::vector<std::vector<std::string>>
for(auto& seq : sequenceList){
outputTEs.push_back(seq->outputTE()->name());
ptree inputTEs_innerArray;
for(auto& input : seq->inputTEs()) {
inputTEs_innerArray.push_back( ptree::value_type("", input->name()) );
}
inputTEs_outerArray.push_back( ptree::value_type("", inputTEs_innerArray) );
ptree algorithms_innerArray;
for(const std::string& alg : seq->algorithms()) {
algorithms_innerArray.push_back( ptree::value_type("", alg) );
}
algorithms_outerArray.push_back( ptree::value_type("", algorithms_innerArray) );
}
ptree pSequence;
pSequence.add_child("outputTEs", asArray(outputTEs));
pSequence.add_child("inputTEs", inputTEs_outerArray);
pSequence.add_child("algorithms", algorithms_outerArray);
top.add_child("sequence_run2", pSequence);
menu.setData(std::move(top));
menu.setSMK(frame->smk());
return true;
......
......@@ -253,6 +253,32 @@ namespace TrigConf {
return StatusCode::SUCCESS;
}
/// Helper function ptree key->[] to std::vector<T>
template<typename T>
std::vector<T> ToVector(const TrigConf::DataStructure& ds, const std::string& child){
using ptree = boost::property_tree::ptree;
std::vector<T> return_vector;
for( const ptree::value_type& entry : ds.data().get_child(child) ) {
return_vector.push_back( entry.second.get_value<T>() );
}
return return_vector;
}
/// Helper function ptree key->[[]] to std::vector<std::vector<T>>
template<typename T>
std::vector<std::vector<T>> ToVectorVector(const TrigConf::DataStructure& ds, const std::string& child){
using ptree = boost::property_tree::ptree;
std::vector<std::vector<T>> return_vector;
for( const ptree::value_type& outer : ds.data().get_child(child) ) {
return_vector.push_back(std::vector<T>());
for (const ptree::value_type& inner : outer.second) {
return_vector.back().push_back( inner.second.get_value<T>() );
}
}
return return_vector;
}
/// Load JSON derived data into legacy structures to maintain
/// compatiblity with existing code.
///
......@@ -315,46 +341,75 @@ namespace TrigConf {
// Fill the HLT configuration:
if (loadedHlt.isInitialized()) {
for (const Chain& loadedChain : loadedHlt) {
// Figure out which level this chain is from:
std::string level = "";
if( loadedChain.name().find( "L2_" ) == 0 ) {
level = "L2";
} else if( loadedChain.name().find( "EF_" ) == 0 ) {
level = "EF";
} else if( loadedChain.name().find( "HLT_" ) == 0 ) {
level = "HLT";
} else {
msg << MSG::WARNING << "prepareTriggerMenu(...): "
<< "Couldn't figure out 'level' for chain: "
<< loadedChain.name() << endmsg;
}
// An empty signature list for the chain:
// This is not populated from JSON data
std::vector< HLTSignature* > signatures;
// Figure out which level this chain is from:
std::string level = "";
if( loadedChain.name().find( "L2_" ) == 0 ) {
level = "L2";
} else if( loadedChain.name().find( "EF_" ) == 0 ) {
level = "EF";
} else if( loadedChain.name().find( "HLT_" ) == 0 ) {
level = "HLT";
} else {
msg << MSG::WARNING << "prepareTriggerMenu(...): "
<< "Couldn't figure out 'level' for chain: "
<< loadedChain.name() << endmsg;
}
// An empty signature list for the chain:
std::vector< HLTSignature* > signatures;
// Optional Run2 payload
std::vector<uint32_t> counters;
std::vector<int> logics;
std::vector<std::vector<std::string>> outputTEs;
if (loadedChain.hasChild("signature")) {
counters = ToVector<uint32_t>(loadedChain, "signature.counters");
logics = ToVector<int>(loadedChain, "signature.logics");
outputTEs = ToVectorVector<std::string>(loadedChain, "signature.outputTEs");
}
if( msg.level() <= MSG::VERBOSE ) {
msg << MSG::VERBOSE << "chain " << loadedChain.name()
<< " has counter " << loadedChain.counter()
<< " and " << counters.size() << " signatures (runs 1,2 only)" << endmsg;
}
for( size_t sig = 0; sig < counters.size(); ++sig ) {
std::vector< HLTTriggerElement* > outTEs;
for( size_t outTEcounter = 0; outTEcounter< outputTEs[ sig ].size(); ++outTEcounter ) {
HLTTriggerElement* element = new HLTTriggerElement( outputTEs[ sig ][ outTEcounter ] );
outTEs.push_back( element );
}
HLTSignature* signature = new HLTSignature( counters[ sig ], logics[ sig ], outTEs );
signatures.push_back( signature );
if( msg.level() <= MSG::VERBOSE ) {
msg << MSG::VERBOSE << "prepared signature: " << *( signatures.back() ) << endmsg;
}
}
// Create the chain object:
HLTChain* chain = new HLTChain( loadedChain.name(),
loadedChain.counter(),
1, // Chain version not important
level,
loadedChain.l1item(), // L1 seeds (string)
-1, // Lower chain ID not important
signatures ); // Note: Empty
// Create the chain object:
HLTChain* chain = new HLTChain( loadedChain.name(),
loadedChain.counter(),
1, // Chain version not important
level,
loadedChain.l1item(), // L1 seeds (string)
-1, // Lower chain ID not important
signatures ); // Empty for R3 JSONs
chain->set_rerun_prescale( -1.0 ); // Not used in R3
chain->set_pass_through( -1.0 ); // Not used in R3
chain->set_leg_multiplicities( loadedChain.legMultiplicities() );
for (const std::string& group : loadedChain.groups()){
chain->addGroup(group);
}
if (loadedHltps.isInitialized()) {
const HLTPrescalesSet::HLTPrescale& loadedPrescale = loadedHltps.prescale( loadedChain.name() );
chain->set_prescale( loadedPrescale.prescale );
} else {
chain->set_prescale( 0 );
}
chain->set_rerun_prescale( -1.0 ); // Not used in R3
chain->set_pass_through( -1.0 ); // Not used in R3
chain->set_leg_multiplicities( loadedChain.legMultiplicities() );
for (const std::string& group : loadedChain.groups()){
chain->addGroup(group);
}
if (loadedHltps.isInitialized()) {
const HLTPrescalesSet::HLTPrescale& loadedPrescale = loadedHltps.prescale( loadedChain.name() );
chain->set_prescale( loadedPrescale.prescale );
} else {
chain->set_prescale( 0 );
}
// Add it to the list of chains:
if( ! chainList.addHLTChain( chain ) ) {
......@@ -367,7 +422,25 @@ namespace TrigConf {
}
}
// Do not add sequence info to legacy structures (format is different)
// Add sequence information if it's available (R1 or R2 menu):
if( loadedHlt.hasChild("sequence_run2") ) {
std::vector<std::string> menu_outputTEs = ToVector<std::string>(loadedHlt, "sequence_run2.outputTEs");
std::vector<std::vector<std::string>> menu_inputTEs = ToVectorVector<std::string>(loadedHlt, "sequence_run2.inputTEs");
std::vector<std::vector<std::string>> menu_algorithms = ToVectorVector<std::string>(loadedHlt, "sequence_run2.algorithms");
for( size_t i = 0; i< menu_outputTEs.size(); ++i ) {
HLTTriggerElement* outputTE = new HLTTriggerElement( menu_outputTEs[ i ] );
std::vector< HLTTriggerElement* > inputTEs;
for( size_t j = 0; j < menu_inputTEs[ i ].size(); ++j ) {
HLTTriggerElement* te = new HLTTriggerElement( menu_inputTEs[ i ][ j ] );
inputTEs.push_back( te );
}
HLTSequence* sequence = new HLTSequence( inputTEs, outputTE, menu_algorithms[ i ] );
sequenceList.addHLTSequence( sequence );
// This throws a runtime_error if it fails, which we don't need to
// handle, since this is a FATAL error anyways.
}
}
// Bunchgroup data is TODO
// Create a new BunchGroupSet object, since an existing one can't be
......
......@@ -21,6 +21,8 @@ def navigationThinningSvc (config):
svc.Actions = [ 'DropFeatures', 'Reload', 'SyncThinning', 'DropChains', 'Save', 'Restore']
if config['mode'] == 'cleanup':
svc.Actions = [ 'DropFeatures', 'Reload', 'SyncThinning', 'Save']
if config['mode'] == 'cleanup_noreload':
svc.Actions = [ 'DropFeatures', 'SyncThinning', 'Save']
if 'Print' in svc.Actions:
from AthenaCommon.Constants import DEBUG
......
......@@ -381,7 +381,10 @@ class HLTTriggerResultGetter(Configured):
edmlist = list(y.split('-')[0] for x in edm.values() for y in x) #flatten names
svc = navigationThinningSvc ({'name':'HLTNav_%s'%stream, 'mode':'cleanup',
# TimM Sep 2021: In MT the 'reload' slimming option in the R2 navigation thinning service was found to be creating
# AODs which would crash when trying to return features. We therefore remove this option by using the added 'cleanup_noreload'
# configuration, see ATR-24141 for details.
svc = navigationThinningSvc ({'name':'HLTNav_%s'%stream, 'mode':'cleanup_noreload',
'result':'HLTResult_HLT',
'features':edmlist})
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment