add support for lumi schema to GitANNSvc

Hlt/HltServices/src/ANNSvcBase.h

@@ -11,6 +11,7 @@
 #pragma once
 #include "GaudiKernel/Service.h"
 #include "Kernel/IIndexedANNSvc.h"
+#include "Kernel/IIndexedLumiSchemaSvc.h"
 #include "Kernel/SynchronizedValue.h"
 #include <map>
 #include <string>
@@ -36,7 +37,7 @@ namespace ANNSvcBase_details {
 
 } // namespace ANNSvcBase_details
 
-class ANNSvcBase : public extends<Service, IIndexedANNSvc> {
+class ANNSvcBase : public extends<Service, IIndexedANNSvc, IIndexedLumiSchemaSvc> {
 
 public:
   using extends::extends;
@@ -67,14 +68,28 @@ public:
     return *ptr;
   }
 
+  lumi_schema_t const& lumiCounters( unsigned key, unsigned version ) const override final {
+    auto ptr = m_lumi.with_lock( [&]( std::map<std::pair<unsigned, unsigned>, lumi_schema_t>& lumi ) {
+      auto j = lumi.find( {key, version} );
+      if ( j != lumi.end() ) return &j->second;
+      auto [k, ok] = lumi.emplace( std::pair{key, version}, fetch_lumi( key, version ) );
+      return ok ? &k->second : nullptr;
+    } );
+    if ( !ptr ) throw GaudiException{"Failed to add to lumi schema map", __PRETTY_FUNCTION__, StatusCode::FAILURE};
+    return *ptr;
+  }
+
 private:
   Gaudi::Property<bool> m_allow_zero_as_key{this, "AllowZeroAsKey", false};
 
-  virtual map_t fetch( unsigned int tck, const Gaudi::StringKey& major ) const = 0;
+  virtual map_t         fetch( unsigned int tck, const Gaudi::StringKey& major ) const = 0;
+  virtual lumi_schema_t fetch_lumi( unsigned int key, unsigned int version ) const     = 0;
 
   mutable std::array<LHCb::cxx::SynchronizedValue<std::map<unsigned int, map_t>>, ANNSvcBase_details::s_majors.size()>
       m_maps;
   mutable std::array<LHCb::cxx::SynchronizedValue<std::map<unsigned int, inv_map_t>>,
                      ANNSvcBase_details::s_majors.size()>
       m_inv_maps;
+
+  mutable LHCb::cxx::SynchronizedValue<std::map<std::pair<unsigned, unsigned>, lumi_schema_t>> m_lumi;
 };

Hlt/HltServices/src/GitANNSvc.cpp

@@ -96,30 +96,29 @@ class GitANNSvc : public ANNSvcBase {
   Gaudi::Property<std::string> m_fmt{this, "RefFormatter", "{0}:ann/json/{1:.2}/{1}.json"}; // 0=tag, 1=key, 2=label
   Gaudi::Property<std::string> m_altfmt{this, "AltRefFormatter", "key-{1}:ann/json/{1:.2}/{1}.json"};
 
+  Gaudi::Property<std::map<unsigned int, std::string>> m_lumi_key2JSON{this, "LumiOverrule", {}};
+  Gaudi::Property<std::string>                         m_lumi_fmt{this, "LumiRefFormatter",
+                                          "{0}:luminosity_counters/json/{1:.2}/{1}.json"}; // 0=tag, 1=key, 2=version
+
   std::optional<std::string> fetch_from_repo( std::string const& repo, std::string const& name ) const {
     if ( repo.empty() ) error() << "no repo specified..." << endmsg;
     auto g = git{repo};
     if ( !g ) throw GaudiException( "unable to open git repo: " + repo, __PRETTY_FUNCTION__, StatusCode::FAILURE );
     if ( msgLevel( MSG::DEBUG ) ) {
-      debug() << "fetch_from_repos: trying to get " << name << " from " << repo << endmsg;
+      debug() << "fetch_from_repo: trying to get " << name << " from " << repo << endmsg;
     }
     return g.read( name );
   }
 
-  std::optional<std::string> fetch_from_repos( std::string_view fmt, std::string_view key,
-                                               std::string_view major ) const {
+  template <typename F>
+  std::optional<std::string> fetch_from_repos( F&& fmt ) const {
     for ( const auto& [repo, tag] : m_repo.value() ) {
-      if ( auto s = fetch_from_repo( repo, fmt::format( fmt, tag, key, major ) ); s ) return s;
+      if ( auto s = fetch_from_repo( repo, fmt( tag ) ); s ) return s;
     }
     return std::nullopt;
   }
 
-  std::optional<std::string> fetch_from_repos( std::string_view key, std::string_view major ) const {
-    auto s = fetch_from_repos( m_fmt, key, major );
-    return s ? s : fetch_from_repos( m_altfmt, key, major );
-  }
-
-  std::optional<nlohmann::json> fetch_json( unsigned int key, const Gaudi::StringKey& major ) const {
+  std::optional<nlohmann::json> fetch_ann_json( unsigned int key, const Gaudi::StringKey& major ) const {
     if ( auto io = m_key2JSON.find( key ); io != m_key2JSON.end() ) {
       warning() << "key " << fmt::format( "0x{:08x}", key ) << " has an explicitly configured overrule -- using that..."
                 << endmsg;
@@ -132,10 +131,11 @@ class GitANNSvc : public ANNSvcBase {
     auto id = ( i != m_key2commit.end() ? i->second : fmt::format( "{:08x}", key ) );
 
     if ( msgLevel( MSG::DEBUG ) )
-      debug() << "fetch_json( " << key << " -> " << id << " ) from repo " << m_repo.value() << endmsg;
-
-    auto s = fetch_from_repos( id, major.str() );
+      debug() << "fetch_ann_json( " << key << " -> " << id << " ) from repo " << m_repo.value() << endmsg;
 
+    auto s = fetch_from_repos( [&]( auto const& tag ) { return fmt::format( m_fmt.value(), tag, id, major.str() ); } );
+    if ( !s )
+      s = fetch_from_repos( [&]( auto const& tag ) { return fmt::format( m_altfmt.value(), tag, id, major.str() ); } );
     if ( !s ) return std::nullopt;
     auto json = nlohmann::json::parse( *s, nullptr, false );
     if ( json.is_discarded() ) return std::nullopt;
@@ -144,7 +144,7 @@ class GitANNSvc : public ANNSvcBase {
 
   template <typename Inserter>
   void fetch_( unsigned int key, const Gaudi::StringKey& major, Inserter inserter ) const {
-    auto json = fetch_json( key, major );
+    auto json = fetch_ann_json( key, major );
     if ( !json )
       throw GaudiException( fmt::format( "unable to obtain json for key {:08x} from repositories {}", key,
                                          Gaudi::Utils::toString( m_repo.value() ) ),
@@ -162,6 +162,51 @@ class GitANNSvc : public ANNSvcBase {
     return m;
   }
 
+  std::optional<nlohmann::json> fetch_lumi_json( unsigned int key ) const {
+    if ( auto io = m_lumi_key2JSON.find( key ); io != m_key2JSON.end() ) {
+      warning() << "lumi key " << fmt::format( "0x{:08x}", key )
+                << " has an explicitly configured overrule -- using that..." << endmsg;
+      auto json = nlohmann::json::parse( io->second, nullptr, false );
+      if ( json.is_discarded() ) return std::nullopt;
+      return json;
+    }
+
+    auto id = fmt::format( "{:08x}", key );
+
+    if ( msgLevel( MSG::DEBUG ) )
+      debug() << "fetch_lumi_json( " << key << " -> " << id << " ) from repo " << m_repo.value() << endmsg;
+
+    auto s = fetch_from_repos( [&]( auto const& tag ) { return fmt::format( m_lumi_fmt.value(), tag, id ); } );
+    if ( !s ) return std::nullopt;
+    auto json = nlohmann::json::parse( *s, nullptr, false );
+    if ( json.is_discarded() ) return std::nullopt;
+    return json;
+  }
+
+  lumi_schema_t fetch_lumi( unsigned int key, unsigned int version ) const override {
+    if ( msgLevel( MSG::DEBUG ) ) debug() << "fetch lumi(" << key << "," << version << ")" << endmsg;
+    auto json = fetch_lumi_json( key );
+    if ( !json )
+      throw GaudiException( fmt::format( "unable to obtain json for key {:08x} from repositories {}", key,
+                                         Gaudi::Utils::toString( m_repo.value() ) ),
+                            "GitANNSvc::fetch", StatusCode::FAILURE );
+    auto vs = json->find( "version" );
+    if ( vs == json->end() || vs->template get<unsigned>() != version ) {
+      // must have correct version!!!)
+      error() << "wrong version" << endmsg;
+      return {};
+    }
+    auto j = json->find( "CounterDefinition" );
+    if ( j == json->end() ) {
+      error() << "no counterdef" << endmsg;
+      return {};
+    }
+    lumi_schema_t m;
+    std::transform( j->begin(), j->end(), std::back_inserter( m ),
+                    []( const nlohmann::json& k ) { return IIndexedLumiSchemaSvc::from_json( k ); } );
+    return m;
+  }
+
 public:
   using ANNSvcBase::ANNSvcBase;
 };

Hlt/HltServices/src/TCKANNSvc.cpp

@@ -54,6 +54,11 @@ namespace std {
 
 class TCKANNSvc : public ANNSvcBase {
 
+  lumi_schema_t fetch_lumi( unsigned int, unsigned int ) const override {
+    throw GaudiException( "fetch_lumi not supported -- please use different service implementation",
+                          __PRETTY_FUNCTION__, StatusCode::FAILURE );
+    return {};
+  }
   map_t fetch( unsigned int tck, const Gaudi::StringKey& major ) const override {
     TCK _tck( tck );
     _tck.normalize();

Kernel/HltInterfaces/include/Kernel/IIndexedLumiSchemaSvc.h

+/*****************************************************************************\
+* (c) Copyright 2000-2018 CERN for the benefit of the LHCb Collaboration      *
+*                                                                             *
+* This software is distributed under the terms of the GNU General Public      *
+* Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   *
+*                                                                             *
+* In applying this licence, CERN does not waive the privileges and immunities *
+* granted to it by virtue of its status as an Intergovernmental Organization  *
+* or submit itself to any jurisdiction.                                       *
+\*****************************************************************************/
+#pragma once
+
+// Include files
+#include "GaudiKernel/INamedInterface.h"
+
+/** @class ITCKANNSvc ITCKANNSvc.h
+ *
+ * return the layout of a lumi bank given a key
+ *
+ *  @author Gerhard Raven
+ *  @date   2014-05-29
+ */
+
+struct IIndexedLumiSchemaSvc : extend_interfaces<INamedInterface> {
+public:
+  /// Return the interface ID
+  DeclareInterfaceID( IIndexedLumiSchemaSvc, 1, 0 );
+
+  struct CounterDefinition {
+    std::string name;
+    unsigned    size;
+    unsigned    offset;
+  };
+
+  template <typename JSON>
+  static CounterDefinition from_json( const JSON& json ) {
+    return CounterDefinition{json["name"].template get<std::string>(), json["size"].template get<unsigned>(),
+                             json["offset"].template get<unsigned>()};
+  }
+
+  using lumi_schema_t = std::vector<CounterDefinition>;
+
+  // given a bank of version `version`, and a key `key`, return which counters can be found where
+  virtual lumi_schema_t const& lumiCounters( unsigned key, unsigned version ) const = 0;
+};