Skip to content
Snippets Groups Projects
Commit 928d8e4a authored by Gerhard Raven's avatar Gerhard Raven
Browse files

Add GitANNSvc, re-implement TCKANNSvc

parent 86c11d9d
No related branches found
No related tags found
No related merge requests found
......@@ -28,6 +28,7 @@ gaudi_add_module(HltServices
src/HltLinePersistenceSvc.cpp
src/LinePersistenceSvcCommon.cpp
src/PropertyConfigSvc.cpp
src/GitANNSvc.cpp
src/TCKANNSvc.cpp
src/TCKLinePersistenceSvc.cpp
src/cdb.cpp
......@@ -46,6 +47,8 @@ gaudi_add_module(HltServices
LHCb::HltEvent
LHCb::HltInterfaces
LHCb::LHCbKernel
nlohmann_json::nlohmann_json
PkgConfig::git2
)
gaudi_add_executable(hlttck_cdb_listkeys
......
/*****************************************************************************\
* (c) Copyright 2000-2018 CERN for the benefit of the LHCb Collaboration *
* *
* This software is distributed under the terms of the GNU General Public *
* Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". *
* *
* In applying this licence, CERN does not waive the privileges and immunities *
* granted to it by virtue of its status as an Intergovernmental Organization *
* or submit itself to any jurisdiction. *
\*****************************************************************************/
#pragma once
#include "GaudiKernel/Service.h"
#include "GaudiKernel/VectorMap.h"
#include "Kernel/IIndexedANNSvc.h"
#include "Kernel/SynchronizedValue.h"
#include <map>
#include <string>
namespace ANNSvcBase_details {
inline const std::array<std::string, 5> s_majors{"Hlt1SelectionID", "Hlt2SelectionID", "SpruceSelectionID", "InfoID",
"PackedObjectLocations"};
inline int index_major( const Gaudi::StringKey& major ) {
auto i = std::find( s_majors.begin(), s_majors.end(), major );
return i == s_majors.end() ? -1 : std::distance( s_majors.begin(), i );
}
inline auto reverse( IIndexedANNSvc::map_t const& m ) {
IIndexedANNSvc::inv_map_t imap;
for ( auto const& [k, v] : m ) {
auto [_, ok] = imap.insert( v, k );
if ( !ok ) throw GaudiException( "unable to invert map", __PRETTY_FUNCTION__, StatusCode::FAILURE );
}
return imap;
}
} // namespace ANNSvcBase_details
class ANNSvcBase : public extends<Service, IIndexedANNSvc> {
public:
using extends::extends;
map_t const& i2s( unsigned int index, const Gaudi::StringKey& major ) const override final {
auto i = ANNSvcBase_details::index_major( major );
if ( i < 0 ) throw GaudiException{"bad major key", __PRETTY_FUNCTION__, StatusCode::FAILURE};
auto ptr = m_maps[i].with_lock( [&]( std::map<unsigned int, map_t>& map ) -> map_t const* {
auto j = map.find( index );
if ( j != map.end() ) return &j->second;
auto [k, ok] = map.emplace( index, fetch( index, major ) );
return ok ? &k->second : nullptr;
} );
if ( !ptr ) throw GaudiException{"Failed to add to map", __PRETTY_FUNCTION__, StatusCode::FAILURE};
return *ptr;
}
inv_map_t const& s2i( unsigned int index, const Gaudi::StringKey& major ) const override final {
auto i = ANNSvcBase_details::index_major( major );
if ( i < 0 ) throw GaudiException{"bad major key", __PRETTY_FUNCTION__, StatusCode::FAILURE};
auto ptr = m_inv_maps[i].with_lock( [&]( std::map<unsigned int, inv_map_t>& imap ) -> inv_map_t const* {
auto j = imap.find( index );
if ( j != imap.end() ) return &j->second;
auto [k, ok] = imap.emplace( index, ANNSvcBase_details::reverse( this->i2s( index, major ) ) );
return ok ? &k->second : nullptr;
} );
if ( !ptr ) throw GaudiException{"Failed to add to map", __PRETTY_FUNCTION__, StatusCode::FAILURE};
return *ptr;
}
private:
Gaudi::Property<bool> m_allow_zero_as_key{this, "AllowZeroAsKey", false};
bool require_valid_key( unsigned key ) const {
if ( key == 0 && !m_allow_zero_as_key ) {
throw GaudiException{"Zero used as key while this is explicitly dis-allowed", __PRETTY_FUNCTION__,
StatusCode::FAILURE};
}
}
virtual map_t fetch( unsigned int tck, const Gaudi::StringKey& major ) const = 0;
mutable std::array<LHCb::cxx::SynchronizedValue<std::map<unsigned int, map_t>>, ANNSvcBase_details::s_majors.size()>
m_maps;
mutable std::array<LHCb::cxx::SynchronizedValue<std::map<unsigned int, inv_map_t>>,
ANNSvcBase_details::s_majors.size()>
m_inv_maps;
};
/*****************************************************************************\
* (c) Copyright 2000-2018 CERN for the benefit of the LHCb Collaboration *
* *
* This software is distributed under the terms of the GNU General Public *
* Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". *
* *
* In applying this licence, CERN does not waive the privileges and immunities *
* granted to it by virtue of its status as an Intergovernmental Organization *
* or submit itself to any jurisdiction. *
\*****************************************************************************/
#include "ANNSvcBase.h"
#include <fmt/format.h>
#include <git2.h>
#include <nlohmann/json.hpp>
namespace {
std::string to_string( const git_blob* blob ) {
assert( blob != nullptr );
return std::string{reinterpret_cast<const char*>( git_blob_rawcontent( blob ) ), git_blob_rawsize( blob )};
}
// TODO: this is a copy from ParamFileSvc -- find a way to avoid the code duplication...
class git {
git_repository* m_repo = nullptr;
struct object {
git_object* obj = nullptr;
operator bool() const { return obj != nullptr; }
operator git_object**() { return &obj; }
const git_blob* as_blob() const {
return git_object_type( obj ) == GIT_OBJECT_BLOB ? reinterpret_cast<const git_blob*>( obj ) : nullptr;
}
std::optional<std::string> as_string() const {
auto* blob = as_blob();
if ( !blob ) return std::nullopt; // can we set git_error_last to something usefull??
return to_string( blob );
}
~object() { git_object_free( obj ); }
};
public:
git( std::string const& path ) {
git_libgit2_init();
if ( git_repository_open_bare( &m_repo, path.c_str() ) < 0 ) { m_repo = nullptr; };
}
operator bool() const { return m_repo != nullptr; }
std::optional<std::string> read( std::string_view rev, std::string_view path ) const {
assert( m_repo != nullptr );
auto rev_path = fmt::format( "{}:{}", rev, path );
object obj;
if ( git_revparse_single( obj, m_repo, rev_path.c_str() ) < 0 ) return std::nullopt;
return obj.as_string();
}
~git() {
git_repository_free( m_repo );
git_libgit2_shutdown();
}
};
} // namespace
class GitANNSvc : public ANNSvcBase {
Gaudi::Property<std::string> m_repo{this, "Repository", {}};
Gaudi::Property<std::string> m_fname{this, "Filename", {}};
Gaudi::Property<std::map<unsigned int, std::string>> m_tck2commit{this, "KeyMapping", {}};
map_t fetch( unsigned int tck, const Gaudi::StringKey& major ) const override {
auto g = git{m_repo};
if ( !g )
throw GaudiException( "GitANNSvc: unable to open git repo: " + m_fname.value(), __PRETTY_FUNCTION__,
StatusCode::FAILURE );
auto s = g.read( m_tck2commit.value().at( tck ), m_fname );
if ( !s )
throw GaudiException( "GitANNSvc: unable to read : " + m_fname.value() + " for version " +
m_tck2commit.value().at( tck ) + " from repository " + m_repo.value(),
__PRETTY_FUNCTION__, StatusCode::FAILURE );
auto json = nlohmann::json::parse( *s );
auto j = json.find( major.str() );
if ( j == json.end() )
throw GaudiException( "JSONANNSvc: could not obtain key " + major.str() + " from file " + m_fname.value() +
" for version " + m_tck2commit.value().at( tck ) + " from repository " + m_repo.value(),
__PRETTY_FUNCTION__, StatusCode::FAILURE );
map_t m;
for ( const auto& [k, v] : j->get<std::map<unsigned int, std::string>>() ) { m.insert( k, v ); }
return m;
}
public:
using ANNSvcBase::ANNSvcBase;
};
DECLARE_COMPONENT( GitANNSvc )
......@@ -33,7 +33,7 @@ private:
"do we allow undefined, on-demand generated, key/value pairs?"};
};
DECLARE_COMPONENT( HltANNSvc )
// DECLARE_COMPONENT( HltANNSvc )
std::optional<IANNSvc::minor_value_type> HltANNSvc::handleUndefined( const major_key_type& major,
const std::string& minor ) const {
......
......@@ -8,12 +8,14 @@
* granted to it by virtue of its status as an Intergovernmental Organization *
* or submit itself to any jurisdiction. *
\*****************************************************************************/
#include "Kernel/SynchronizedValue.h"
#include <map>
#include <string>
// Define the parser
#include "ANNSvcBase.h"
#include "GaudiKernel/Service.h"
#include "GaudiKernel/VectorMap.h"
#include "GaudiKernel/ParsersFactory.h"
#include "GaudiKernel/ToStream.h"
#include "Kernel/IIndexedANNSvc.h"
#include "Kernel/IPropertyConfigSvc.h"
#include "Kernel/TCK.h"
namespace {
using additionalIDs_t = std::map<std::string, std::map<std::string, unsigned int>>;
......@@ -22,8 +24,7 @@ namespace {
"InfoID", "PackedObjectLocations"};
} // namespace
namespace Gaudi {
namespace Parsers {
namespace Gaudi::Parsers {
// Parser grammar
template <typename Iterator, typename Skipper>
......@@ -35,91 +36,100 @@ namespace Gaudi {
// Additional parser for our property
StatusCode parse( additionalIDs_t& result, const std::string& input ) {
return Gaudi::Parsers::parse_( result, input );
return parse_( result, input );
}
} // namespace Parsers
} // namespace Gaudi
} // namespace Gaudi::Parsers
// We also need to be able to print an object of our type as a string that both
// Python and our parser can understand,
#include "GaudiKernel/ToStream.h"
namespace std {
// This is an example valid for any mapping type.
ostream& operator<<( ostream& s, const additionalIDs_t& m ) {
bool first = true;
s << '{';
for ( const auto& i : m ) {
if ( first )
first = false;
else
s << ", ";
Gaudi::Utils::toStream( i.first, s ) << ": ";
Gaudi::Utils::toStream( i.second, s );
}
s << '}';
return s;
return GaudiUtils::details::ostream_joiner( s << '{', m,", ", [](ostream& os, const auto& i) {
Gaudi::Utils::toStream( i.second, Gaudi::Utils::toStream( i.first, os ) << ": " );
}) << '}';
}
} // namespace std
#include "GaudiKernel/Service.h"
#include "GaudiKernel/VectorMap.h"
#include "Kernel/IIndexedANNSvc.h"
#include "Kernel/IPropertyConfigSvc.h"
#include "Kernel/TCK.h"
namespace {
using map_t = GaudiUtils::VectorMap<unsigned int, std::string>;
using inv_map_t = GaudiUtils::VectorMap<std::string, unsigned int>;
inv_map_t reverse( map_t const& m ) {
inv_map_t imap;
for ( auto const& [k, v] : m ) {
auto [_, ok] = imap.insert( v, k );
if ( !ok ) throw GaudiException( "unable to invert map", __PRETTY_FUNCTION__, StatusCode::FAILURE );
class TCKANNSvc : public ANNSvcBase {
map_t fetch( unsigned int tck, const Gaudi::StringKey& major ) const override {
TCK _tck( tck );
_tck.normalize();
auto entry = m_cache.with_lock( [&]( std::map<TCK, const PropertyConfig*>& cache ) -> PropertyConfig const* {
auto i = cache.find( _tck );
if ( i != end( cache ) ) return i->second;
// grab properties of child from config database...
const ConfigTreeNode* tree = m_propertyConfigSvc->resolveConfigTreeNode(
ConfigTreeNodeAlias::alias_type{std::string( "TCK/" ) + _tck.str()} );
if ( !tree ) {
// If we could not resolve the (non-zero) TCK we have a problem
error() << "Requested TCK " << _tck << " could not resolved. Returning an empty map... " << endmsg;
return nullptr;
}
PropertyConfig::digest_type child = m_propertyConfigSvc->findInTree( tree->digest(), m_instanceName );
if ( child.invalid() ) {
error() << "Error finding configuration of " << m_instanceName << " for TCK " << _tck
<< " Returning an empty map... " << endmsg;
return nullptr;
}
const PropertyConfig* config = m_propertyConfigSvc->resolvePropertyConfig( child );
if ( !config ) {
error() << "Error reading configuration of " << m_instanceName << " for TCK " << _tck
<< " Returning an empty map... " << endmsg;
return nullptr;
}
auto status = cache.insert( {_tck, config} );
if ( !status.second ) {
error() << "Error updating cache for TCK " << _tck << " Returning an empty map... " << endmsg;
return nullptr;
}
return status.first->second;
} );
if ( !entry ) return {};
auto prop = std::find_if( std::begin( entry->properties() ), std::end( entry->properties() ),
[&]( const std::pair<std::string, std::string>& p ) { return major.str() == p.first; } );
if ( prop == std::end( entry->properties() ) ) {
error() << "Error finding requested major " << major << " in configuration of " << m_instanceName << " for TCK "
<< _tck << " Returning an empty map... " << endmsg;
return {};
}
return imap;
}
} // namespace
class TCKANNSvc : public extends<Service, IIndexedANNSvc> {
public:
using extends::extends;
map_t const& i2s( unsigned int index, const Gaudi::StringKey& major ) const override {
auto i = std::find( s_majors.begin(), s_majors.end(), major );
if ( i == s_majors.end() ) throw GaudiException{"bad major key", __PRETTY_FUNCTION__, StatusCode::FAILURE};
auto ptr = m_maps[std::distance( s_majors.begin(), i )].with_lock(
[&]( std::map<unsigned int, map_t>& map ) -> map_t const* {
auto j = map.find( index );
if ( j != map.end() ) return &j->second;
auto [k, ok] = map.emplace( index, fetch_from_property_config( index, major ) );
return ok ? &k->second : nullptr;
} );
if ( !ptr ) throw GaudiException{"Failed to add to map", __PRETTY_FUNCTION__, StatusCode::FAILURE};
return *ptr;
}
inv_map_t const& s2i( unsigned int index, const Gaudi::StringKey& major ) const override {
auto i = std::find( s_majors.begin(), s_majors.end(), major );
if ( i == s_majors.end() ) throw GaudiException{"bad major key", __PRETTY_FUNCTION__, StatusCode::FAILURE};
auto ptr = m_inv_maps[std::distance( s_majors.begin(), i )].with_lock(
[&]( std::map<unsigned int, inv_map_t>& imap ) -> inv_map_t const* {
auto j = imap.find( index );
if ( j != imap.end() ) return &j->second;
auto [k, ok] = imap.emplace( index, reverse( this->i2s( index, major ) ) );
return ok ? &k->second : nullptr;
} );
if ( !ptr ) throw GaudiException{"Failed to add to map", __PRETTY_FUNCTION__, StatusCode::FAILURE};
return *ptr;
// now use the property parser to do the hard work of converting the string rep into a map...
std::map<std::string, int> map;
// or extend the PropertyConfig interface so it takes a property, invokes 'fromString' on it, and returns StatusCode
// ? or an templated interface, which wraps a PropertyWithValue around it, then invokes the above, and drops the
// PropertyWithValue? std::optional< map<string,int> > = config->assign< map<string,int> >( major );
// PropertyWithValue<std::map<std::string,int>>{ major.str(), &map, false }.fromString( prop->second );
// auto status = prp.fromString( prop->second );
auto status = Gaudi::Parsers::parse( map, prop->second );
if ( status.isFailure() ) {
error() << "Error interpreting requested major " << major << " in configuration of " << m_instanceName
<< " for TCK " << _tck << " Returning an empty map... " << endmsg;
return {};
}
GaudiUtils::VectorMap<unsigned int, std::string> result;
auto additionals = m_additionals.find( major.str() );
bool ok = true;
if ( additionals != end( m_additionals ) ) {
for ( const auto& item : additionals->second ) { ok = ok && result.insert( item.second, item.first ).second; }
}
if ( !ok ) {
error() << "Duplicate entries in additional map for major " << major << " in configuration of "
<< m_instanceName << " for TCK " << _tck << " Returning an empty map... " << endmsg;
return {};
}
for ( const auto& item : map ) { ok = ok && result.insert( item.second, item.first ).second; }
if ( !ok ) {
error() << "Duplicate entries in map for major " << major << " in configuration of " << m_instanceName
<< " for TCK " << _tck << " Returning an empty map... " << endmsg;
return {};
}
return result;
}
private:
map_t fetch_from_property_config( unsigned int tck, const Gaudi::StringKey& major ) const;
// properties
Gaudi::Property<additionalIDs_t> m_additionals{
this, "AdditionalIDs", {}, [this]( auto&& ) {
......@@ -133,87 +143,8 @@ private:
Gaudi::Property<std::string> m_instanceName{this, "InstanceName", "HltANNSvc"};
mutable LHCb::cxx::SynchronizedValue<std::map<TCK, const PropertyConfig*>> m_cache; // TODO: flush cache if
// m_instanceName changes
mutable std::array<LHCb::cxx::SynchronizedValue<std::map<unsigned int, map_t>>, s_majors.size()> m_maps;
mutable std::array<LHCb::cxx::SynchronizedValue<std::map<unsigned int, inv_map_t>>, s_majors.size()> m_inv_maps;
// TODO: add properties which allow to overrule TCKs from the backend (eg. because they're not there yet...)
// TODO: add backend to read json from git repo
public:
using ANNSvcBase::ANNSvcBase;
};
DECLARE_COMPONENT( TCKANNSvc )
map_t TCKANNSvc::fetch_from_property_config( unsigned int tck, const Gaudi::StringKey& major ) const {
TCK _tck( tck );
_tck.normalize();
auto entry = m_cache.with_lock( [&]( std::map<TCK, const PropertyConfig*>& cache ) -> PropertyConfig const* {
auto i = cache.find( _tck );
if ( i != end( cache ) ) return i->second;
// grab properties of child from config database...
const ConfigTreeNode* tree = m_propertyConfigSvc->resolveConfigTreeNode(
ConfigTreeNodeAlias::alias_type{std::string( "TCK/" ) + _tck.str()} );
if ( !tree ) {
// If we could not resolve the (non-zero) TCK we have a problem
error() << "Requested TCK " << _tck << " could not resolved. Returning an empty map... " << endmsg;
return nullptr;
}
PropertyConfig::digest_type child = m_propertyConfigSvc->findInTree( tree->digest(), m_instanceName );
if ( child.invalid() ) {
error() << "Error finding configuration of " << m_instanceName << " for TCK " << _tck
<< " Returning an empty map... " << endmsg;
return nullptr;
}
const PropertyConfig* config = m_propertyConfigSvc->resolvePropertyConfig( child );
if ( !config ) {
error() << "Error reading configuration of " << m_instanceName << " for TCK " << _tck
<< " Returning an empty map... " << endmsg;
return nullptr;
}
auto status = cache.insert( {_tck, config} );
if ( !status.second ) {
error() << "Error updating cache for TCK " << _tck << " Returning an empty map... " << endmsg;
return nullptr;
}
return status.first->second;
} );
if ( !entry ) return {};
auto prop = std::find_if( std::begin( entry->properties() ), std::end( entry->properties() ),
[&]( const std::pair<std::string, std::string>& p ) { return major.str() == p.first; } );
if ( prop == std::end( entry->properties() ) ) {
error() << "Error finding requested major " << major << " in configuration of " << m_instanceName << " for TCK "
<< _tck << " Returning an empty map... " << endmsg;
return {};
}
// now use the property parser to do the hard work of converting the string rep into a map...
std::map<std::string, int> map;
// or extend the PropertyConfig interface so it takes a property, invokes 'fromString' on it, and returns StatusCode ?
// or an templated interface, which wraps a PropertyWithValue around it, then invokes the above, and drops the
// PropertyWithValue? std::optional< map<string,int> > = config->assign< map<string,int> >( major );
// PropertyWithValue<std::map<std::string,int>>{ major.str(), &map, false }.fromString( prop->second );
// auto status = prp.fromString( prop->second );
auto status = Gaudi::Parsers::parse( map, prop->second );
if ( status.isFailure() ) {
error() << "Error interpreting requested major " << major << " in configuration of " << m_instanceName
<< " for TCK " << _tck << " Returning an empty map... " << endmsg;
return {};
}
GaudiUtils::VectorMap<unsigned int, std::string> result;
auto additionals = m_additionals.find( major.str() );
bool ok = true;
if ( additionals != end( m_additionals ) ) {
for ( const auto& item : additionals->second ) { ok = ok && result.insert( item.second, item.first ).second; }
}
if ( !ok ) {
error() << "Duplicate entries in additional map for major " << major << " in configuration of " << m_instanceName
<< " for TCK " << _tck << " Returning an empty map... " << endmsg;
return {};
}
for ( const auto& item : map ) { ok = ok && result.insert( item.second, item.first ).second; }
if ( !ok ) {
error() << "Duplicate entries in map for major " << major << " in configuration of " << m_instanceName
<< " for TCK " << _tck << " Returning an empty map... " << endmsg;
return {};
}
return result;
}
......@@ -38,9 +38,10 @@ public:
/// Return the interface ID
DeclareInterfaceID( IIndexedANNSvc, 3, 0 );
virtual GaudiUtils::VectorMap<unsigned int, std::string> const& i2s( unsigned int index,
const Gaudi::StringKey& major ) const = 0;
virtual GaudiUtils::VectorMap<std::string, unsigned int> const& s2i( unsigned int index,
const Gaudi::StringKey& major ) const = 0;
using map_t = GaudiUtils::VectorMap<unsigned int, std::string>;
using inv_map_t = GaudiUtils::VectorMap<std::string, unsigned int>;
virtual map_t const& i2s( unsigned int index, const Gaudi::StringKey& major ) const = 0;
virtual inv_map_t const& s2i( unsigned int index, const Gaudi::StringKey& major ) const = 0;
};
#endif // IINDEXEDANNSVC_H
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment