Commit dd958db1 authored by Frank Winklmeier's avatar Frank Winklmeier
Browse files

TrigConfigSvc: Initial implementation of new JobOptionSvc

Replace the old `TrigConf::HLTJobOptionsSvc` with the new
`TrigConf::JobOptionsSvc`. The service will ultimately be able to read
job properties from JSON (file and DB).

Unlike the old service, which completely re-implemented Gaudi's
`JobOptionsSvc` with all its internal classes (e.g. catalogue), the new
one is just a wrapper around the default `JobOptionsSvc`. Apart from the
additional features, it just forwards all other calls to
the "real" instance. This renders our own implementation of
`JobOptionsCatalogue` and `JobOptionsList` obsolete. An additional
benefit is that this new service can also be used when running from
Python.

For debugging purposes, the service can also dump its catalogue into a
JSON file (configured via the `DUMPFILE` property). However, for
production the usual way of extracting the joboptions from the athena
pickle file should be used.
parent 31b784a8
......@@ -29,23 +29,23 @@ atlas_depends_on_subdirs( PUBLIC
Trigger/TrigT1/L1Topo/L1TopoConfig )
# External dependencies:
find_package( Boost COMPONENTS filesystem thread system )
find_package( Boost )
find_package( COOL COMPONENTS CoolKernel )
find_package( ROOT COMPONENTS Core Tree MathCore Hist RIO pthread PyROOT )
find_package( ROOT COMPONENTS Hist )
find_package( cx_Oracle )
find_package( nlohmann_json )
# Component(s) in the package:
atlas_add_library( TrigConfigSvcLib
src/*.cxx
src/*.cxx src/*.h
PUBLIC_HEADERS TrigConfigSvc
PRIVATE_INCLUDE_DIRS ${ROOT_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ${COOL_INCLUDE_DIRS}
LINK_LIBRARIES AthenaBaseComps GaudiKernel TrigConfHLTData TrigConfL1Data TrigSteeringEvent StoreGateLib SGtests TrigMonitorBaseLib EventContainers AthAnalysisBaseCompsLib
PRIVATE_LINK_LIBRARIES ${ROOT_LIBRARIES} ${Boost_LIBRARIES} ${COOL_LIBRARIES} AthenaKernel AthenaPoolUtilities IOVDbDataModel EventInfo PathResolver TrigConfBase TrigConfJobOptData TrigConfStorage L1TopoConfig )
PRIVATE_INCLUDE_DIRS ${Boost_INCLUDE_DIRS} ${COOL_INCLUDE_DIRS} ${ROOT_INCLUDE_DIRS}
LINK_LIBRARIES AthenaBaseComps AthenaKernel GaudiKernel StoreGateLib TrigConfHLTData TrigConfInterfaces TrigConfL1Data
PRIVATE_LINK_LIBRARIES ${Boost_LIBRARIES} ${COOL_LIBRARIES} ${ROOT_LIBRARIES} AthAnalysisBaseCompsLib AthenaMonitoringLib AthenaPoolUtilities EventInfo IOVDbDataModel L1TopoConfig PathResolver TrigConfBase TrigConfStorage nlohmann_json::nlohmann_json )
atlas_add_component( TrigConfigSvc
src/components/*.cxx
INCLUDE_DIRS ${ROOT_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ${COOL_INCLUDE_DIRS}
LINK_LIBRARIES ${ROOT_LIBRARIES} ${Boost_LIBRARIES} ${COOL_LIBRARIES} AthenaBaseComps StoreGateLib SGtests GaudiKernel TrigConfHLTData TrigConfL1Data TrigSteeringEvent AthenaKernel AthenaPoolUtilities IOVDbDataModel EventInfo PathResolver TrigConfBase TrigConfJobOptData TrigConfStorage TrigMonitorBaseLib L1TopoConfig TrigConfigSvcLib )
LINK_LIBRARIES TrigConfigSvcLib )
# Install files from the package:
atlas_install_python_modules( python/*.py )
......
/*
Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
*/
#ifndef TrigConfigSvc_HLTJobOptionsSvc
#define TrigConfigSvc_HLTJobOptionsSvc
#include <string>
#include <vector>
#include <memory>
#include "AthenaBaseComps/AthService.h"
#include "GaudiKernel/IProperty.h"
#include "GaudiKernel/IJobOptionsSvc.h"
#include "GaudiKernel/ServiceHandle.h"
#include "TrigConfigSvc/ConfigSvcBase.h"
#include "TrigConfigSvc/JobOptionsCatalogue.h"
/**
* $Date: 2009-03-05 18:07:40 $
*
* @short Service providing the HLT trigger job properties
*
* @author Joerg Stelzer
* @version $Revision: 1.19 $
*
*/
class StatusCode;
namespace TrigConf {
class TrigDBConnectionConfig;
class JobOptionTable;
class HLTJobOptionsSvc : public extends2<ConfigSvcBase, IProperty, IJobOptionsSvc>
{
public:
// Constructor and destructor
HLTJobOptionsSvc( const std::string& name, ISvcLocator* pSvcLocator );
virtual ~HLTJobOptionsSvc();
// Service initialization method
virtual StatusCode sysInitialize();
virtual StatusCode initialize();
// Service finalization method
virtual StatusCode finalize();
// IJobOptionsSvc::setMyProperties
virtual StatusCode setMyProperties( const std::string& client,
IProperty* myInt );
/// implementation of IJobOptionsSvc::addPropertyToCatalogue
virtual StatusCode addPropertyToCatalogue( const std::string& client,
const Property& property );
/// implementation of IJobOptionsSvc::removePropertyFromCatalogue
virtual StatusCode removePropertyFromCatalogue( const std::string& client,
const std::string& name );
// implementation of IJobOptionsSvc::getProperties
virtual const std::vector<const Property*>* getProperties( const std::string& client) const;
// To avoid that the previous method shadowes the Service::getProperties()
virtual const std::vector<Property*> & getProperties() const { return Service::getProperties(); }
/// Get a property for a client
virtual const Property*
getClientProperty( const std::string& client,
const std::string& name ) const;
/// implementation of IJobOptionsSvc::getClients
virtual std::vector<std::string> getClients() const;
/** look for file 'File' into search path 'Path'
* and read it to update existing JobOptionsCatalogue
* @param File file name
* @param Path search path
* @return status code
*/
virtual StatusCode readOptions( const std::string& file ,
const std::string& path = "" ) ;
StatusCode queryInterface( const InterfaceID& riid, void** ppvIF );
// Access functions described by IHLTJobOptionsSvc:
StatusCode readDbWithMasterKey();
// IProperty implementation (needed for initialisation)
using Service::setProperty;
using Service::getProperty;
// virtual StatusCode setProperty(const Property& p);
// virtual StatusCode setProperty(std::string& s) { return Service::setProperty(s); }
// virtual StatusCode setProperty(const std::string& n, const std::string& v) { return Service::setProperty(n,v); }
// virtual StatusCode getProperty(Property *p) const;
// virtual const Property& getProperty(const std::string& name) const { return Service::getProperty(name); }
// virtual StatusCode getProperty(const std::string& n, std::string& v) const { return Service::getProperty(n,v); }
void PrintCatalogue(std::ostream & os) const;
private:
StatusCode FillCatalogue(const JobOptionTable & jot);
StatusCode readEnv();
// Properties:
std::string m_dirSearchPath;
std::string m_sourcePath;
std::string m_sourceType;
JobOptionsCatalogue m_catalogue;
}; // class HLTJobOptionsSvc
// little helper functions -- have been moved to TrigConfL1Data/HelperFunctions class
} // namespace TrigConf
#endif // TrigJobOption_HLTJobOptionsSvc_h
/*
Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
*/
// $Id: JobOptionsCatalogue.h,v 1.1 2007-07-29 13:25:43 stelzer Exp $
#ifndef GAUDISVC_JOBOPTIONSCATALOGUE_H
#define GAUDISVC_JOBOPTIONSCATALOGUE_H
#include <string>
#include <vector>
#include "GaudiKernel/StatusCode.h"
#include "GaudiKernel/Property.h"
#include "TrigConfigSvc/JobOptionsList.h"
class MsgStream;
namespace TrigConf {
/** @brief Collection of JobOptionsList 's, one for each algorithm*/
class JobOptionsCatalogue {
public:
/**@brief default constructor*/
JobOptionsCatalogue() {}
/**@brief destructor*/
virtual ~JobOptionsCatalogue();
/**@brief add property to algorithm
*
* @param[in] myAlg name of the algorithm
*
* @param[in] myOpt property to be added
*/
virtual void addOption( const std::string& myAlg, const Property* const& myOpt );
/**@brief retrieve all properties of an algorithm
*
* @param[in] myAlg name of the algorithm
*
* @param[out] myOpts reference to a property list that is filled
*
* @returns StatusCode about the success of the operation (success if it found the algorithm)
*/
virtual StatusCode optionsOf( const std::string& myAlg,
std::vector<const Property*>*& myOpts) const;
/**@brief resolves all properties
*
* @returns StatusCode if no unresolved properties were found
*/
virtual StatusCode resolveOptions(MsgStream& msgStream);
/**@brief resolves all properties
*
* @returns StatusCode if no unresolved properties were found
*/
virtual void printUnresolvedOptions(MsgStream& msgStream);
/**@brief creates vector of all algorithm names
*
* @returns vector of all algorithm names
*/
virtual std::vector<std::string> getClients() const;
private:
typedef std::vector<JobOptionsList*>::iterator algorithmsIterator;
/**@brief Lookup of a property
*
* @param which algorithm-property name in the form 'algorithm.property'
*
* @returns Property or 0 if not found
*/
const Property* getOption(const std::string& which);
/**@brief Sets the list of properties for a specified algorithm
*
* @param[in] myAlg name of the algorithm
*
* @param[out] myList reference to a JobOptionsList pointer that will be set if the algorithm name was found
*
* @returns StatusCode about the success of the operation (success if it found the algorithm)
*/
StatusCode findAlgorithm( const std::string& myAlg, JobOptionsList*& myList ) const;
// data members
std::vector<JobOptionsList*> m_algorithmoptions; ///< vector of JobOptionList 's, one for each algorithm
};
} // end of namespace TrigConf
#endif
/*
Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
*/
// JobOptionsList.h: interface for the JobOptionsList class.
//
//////////////////////////////////////////////////////////////////////
#if !defined(JOBOPTIONSLIST_H_INCLUDED_)
#define JOBOPTIONSLIST_H_INCLUDED_
#include <string>
#include <vector>
#include "GaudiKernel/Property.h"
namespace TrigConf {
/**@brief list of all properties of a single algorithm/component*/
class JobOptionsList {
public:
/**@brief constructor with data
*
* @param myAlg name of the algorithm
*
* @param myOpt first property
*/
JobOptionsList( const std::string& myAlg, const Property* const& myOpt );
/**@brief destructor
*
* All properties are deleted
*/
virtual ~JobOptionsList();
/**@brief add property to the list
*
* @param myOpt reference to the Property pointer to be added
*
* JobOptionsList then assumes ownership of the pointer
*/
virtual void addOption( const Property* const& myOpt );
/**@brief accessor to the name of the algorithm
*
* @returns name of algorithm
*/
virtual const std::string& algorithmName() const;
/**@brief accessor to the vector of properties
*
* @returns pointer to the property vector
*/
virtual const std::vector<const Property*>* properties() const;
private:
/**@brief default constructor (private)*/
JobOptionsList() {}
std::vector<const Property*> m_properties; ///< vector of job properties
std::string m_algorithmName; ///< name of the algorithm
};
} // end of namespace TrigConf
#endif // !defined(JOBOPTIONSLIST_H_INCLUDED_)
/*
Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
*/
#include <exception>
#include <vector>
#include <string>
#include <fstream>
#include "GaudiKernel/Property.h"
#include "PathResolver/PathResolver.h"
#include "boost/lexical_cast.hpp"
// Local includes:
#include "TrigConfigSvc/HLTJobOptionsSvc.h"
#include "TrigConfBase/TrigDBConnectionConfig.h"
#include "TrigConfL1Data/HelperFunctions.h"
#include "TrigConfJobOptData/JobOptionTable.h"
#include "TrigConfStorage/IJobOptionTableLoader.h"
#include "TrigConfStorage/SessionMgr.h"
#include "TrigConfStorage/StorageMgr.h"
using namespace std;
using namespace boost;
TrigConf::HLTJobOptionsSvc::HLTJobOptionsSvc( const std::string& name, ISvcLocator* pSvcLocator ) :
base_class( name, pSvcLocator ),
m_sourcePath(""),
m_sourceType("DB") // DB, FILE
{
declareProperty( "TYPE", m_sourceType );
declareProperty( "PATH", m_sourcePath );
m_outputLevel = MSG::INFO;
}
TrigConf::HLTJobOptionsSvc::~HLTJobOptionsSvc()
{}
//----------------------------------------------------------------------------
StatusCode
TrigConf::HLTJobOptionsSvc::readEnv()
{
// reads the setup from a file
// depreciated
return StatusCode::SUCCESS;
}
StatusCode
TrigConf::HLTJobOptionsSvc::sysInitialize() {
//We specifically do not call the base class sysInitialize here (ATR-15094)
//StatusCode sc = base_class::sysInitialize();
ATH_CHECK(initialize());
ATH_CHECK(setMyProperties(name(), this));
return StatusCode::SUCCESS;
}
//----------------------------------------------------------------------------
StatusCode
TrigConf::HLTJobOptionsSvc::initialize() {
// CHECK(ConfigSvcBase::initialize());
ATH_MSG_INFO("Initializing the HLTJobOptionsSvc");
decltype(m_dbconfig->m_smkey) oldsmk{0};
if(m_dbconfig) {
ATH_MSG_INFO("TrigDBConnectionConfig before initialization: " << m_dbconfig->toString());
oldsmk = m_dbconfig->m_smkey;
}
if ( m_sourceType == "FILE" ) {
// Read job options from a text file
if ( readOptions( m_sourcePath, m_dirSearchPath ).isFailure() ) {
ATH_MSG_ERROR("error reading options");
return StatusCode::FAILURE;
}
} else if( m_sourceType == "DB" ) {
// this fills in all members of TrigDBConnectionConfig that are specified
// in the path string
m_dbconfig = std::unique_ptr<TrigDBConnectionConfig>{
new TrigDBConnectionConfig(m_sourcePath)};
ATH_MSG_INFO("TrigDBConnectionConfig for initialization: " << m_dbconfig->toString());
if (m_dbconfig->m_type == TrigDBConnectionConfig::DBLookup)
m_dbconfig->m_user = m_dbconfig->m_password = "";
if(!oldsmk || oldsmk != m_dbconfig->m_smkey) {
if ( readDbWithMasterKey().isFailure() ) {
ATH_MSG_FATAL("Could not read DB with keys: " << m_dbconfig->m_smkey << "-" << m_dbconfig->m_hltkey << " - " << m_dbconfig->m_lvl1key);
return StatusCode::FAILURE;
}
} else {
ATH_MSG_INFO("HLTJobOptionsSvc holds already the configuration of SMK: " << m_dbconfig->m_smkey);
}
}
ATH_MSG_INFO("TrigDBConnectionConfig after initialization: " << m_dbconfig->toString());
return StatusCode::SUCCESS;
}
//----------------------------------------------------------------------------
StatusCode
TrigConf::HLTJobOptionsSvc::readDbWithMasterKey()
//----------------------------------------------------------------------------
{
if(m_sourceType=="FILE") {
ATH_MSG_ERROR("HLTJobOptionsSvc is configured to run from a file, cannot connect to database!");
return StatusCode::FAILURE;
}
CHECK(initStorageMgr());
SessionMgr & sessionMgr = dynamic_cast<TrigConf::StorageMgr*>(m_storageMgr)->sessionMgr();
sessionMgr.setRetrialPeriod(m_dbconfig->m_retrialPeriod);
sessionMgr.setRetrialTimeout( m_dbconfig->m_retrialPeriod * (m_dbconfig->m_maxRetrials + 1) );
sessionMgr.setConnectionTimeout( 0 );
sessionMgr.createSession();
TrigConf::JobOptionTable jot;
jot.setSMK(m_dbconfig->m_smkey);
if(!m_storageMgr->jobOptionTableLoader().load( jot ))
return StatusCode::FAILURE;
CHECK(freeStorageMgr());
//jot.print();
if( FillCatalogue(jot).isSuccess() ) {
//PrintCatalogue(msg().stream());
// Now we have to resolve still all the entries which are
// references to other Properties
if ( m_catalogue.resolveOptions(msg()).isSuccess() ) {
ATH_MSG_INFO("Job options successfully read in from " << m_dbconfig->toString());
} else {
ATH_MSG_FATAL("Job options references cannot be resolved.");
m_catalogue.printUnresolvedOptions(msg());
return StatusCode::FAILURE;
}
} else {
ATH_MSG_FATAL("Failed to read in job options");
return StatusCode::FAILURE;
}
return StatusCode::SUCCESS;
}
//----------------------------------------------------------------------------
void
TrigConf::HLTJobOptionsSvc::PrintCatalogue(ostream & os) const {
os << "++++++++++++++++++++++++++++++++++++++++++++++++++++" << endl;
os << "Job options catalogue" << endl;
uint nc(0), np(0);
for(const string& algName: m_catalogue.getClients() ) {
nc++;
os << "Client: " << algName << endl;
vector<const Property*>* pl;
m_catalogue.optionsOf( algName, pl);
for(const Property* prop: *pl) {
np++;
os << " property ";
prop->fillStream(os) << endl; // for Gaudi_V19
//(*plIt)->nameAndValueAsStream(os) << std::endl;
}
}
ATH_MSG_INFO("Job options catalogue contains " << nc << " clients with a total of " << np << " properties." << endl);
}
//----------------------------------------------------------------------------
StatusCode
TrigConf::HLTJobOptionsSvc::FillCatalogue(const JobOptionTable & jot) {
for(const JobOption& jo : jot.jobOptionVector() ) {
std::string client = jo.component_alias();
std::string propName = jo.name();
std::string propVal = jo.value();
std::vector<std::string> valArr;
bool isArrayProp = false;
if(propVal[0]=='[') { // indicates an array, start parsing
isArrayProp = true;
std::string::size_type pos = 1;
bool isStringtypeA = propVal.find('\"',pos) != std::string::npos;
bool isStringtypeB = propVal.find('\'',pos) != std::string::npos;
if(isStringtypeA||isStringtypeB) {
if(isStringtypeA){
while( (pos = propVal.find('\"',pos))!=std::string::npos) {
int posEnd = propVal.find('\"',pos+1);
std::string arrS = propVal.substr(pos,posEnd-pos+1);
pos=posEnd+1;
valArr.push_back(arrS);
}
}
if(isStringtypeB){
while( (pos = propVal.find('\'',pos))!=std::string::npos) {
int posEnd = propVal.find('\'',pos+1);
std::string arrS = propVal.substr(pos,posEnd-pos+1);
pos=posEnd+1;
valArr.push_back(arrS);
}
}
} else {
TrigConf::removeAllSpaces(propVal);
std::string propValNoBrackets = propVal.substr(1,propVal.rfind("]")-1);
valArr = split(propValNoBrackets, ",");
}
}
if( isArrayProp ) {
m_catalogue.addOption(client, new StringArrayProperty(propName, valArr));
} else {
m_catalogue.addOption(client, new StringProperty(propName, propVal));
}
}
return StatusCode::SUCCESS;
}
//----------------------------------------------------------------------------
StatusCode
TrigConf::HLTJobOptionsSvc::finalize() {
ATH_MSG_DEBUG("Finalizing");
CHECK(Service::finalize());
return StatusCode::SUCCESS;
}
//----------------------------------------------------------------------------
StatusCode
TrigConf::HLTJobOptionsSvc::queryInterface( const InterfaceID& riid, void** ppvIF )
//----------------------------------------------------------------------------
{
if ( IJobOptionsSvc::interfaceID() == riid ) {
*ppvIF = (IJobOptionsSvc*) this;
} else if ( IProperty::interfaceID() == riid ) {
*ppvIF = (IProperty*) this;
} else {
return Service::queryInterface(riid, ppvIF);
}
addRef();
return StatusCode::SUCCESS;
}