.gitlab-ci.yml

@@ -7,9 +7,16 @@ variables:
   TARGET_BRANCH: master
 
   ALLEN_DATA: "/scratch/allen_data"
+
   LCG_VERSION: "LCG_101"
+
+  # LCG_PLATFORM = {LCG_SYSTEM}+{LCG_QUALIFIER}-{LCG_OPTIMIZATION}
+  LCG_SYSTEM: "x86_64_v3-centos7-clang12"
+  LCG_QUALIFIER: "cpu"
+  LCG_OPTIMIZATION: "opt"
+
   LB_NIGHTLY_SLOT: lhcb-head
-  BINARY_TAG: x86_64_v2-centos7-gcc11-opt
+  BINARY_TAG: x86_64_v2-centos7-gcc11-opt # FIXME remove if not needed
   NO_LBLOGIN: "1" # prevent lbdocker containers to start LbLogin/LbEnv
 
 
@@ -89,7 +96,7 @@ check-formatting:
     when: on_failure
     expire_in: 1 week
   needs: []
-    
+
 # On the master branch: compile docs, copy to public directory and deploy from there with gitlab pages
 pages:
   only:
@@ -152,4 +159,7 @@ test_pages:
     paths:
       - .cache/pip
 
-include: "scripts/ci/config/main.yaml"
+include:
+  - local: 'scripts/ci/config/main.yaml'
+  - project: 'lhcb-rta/reference-update-bot'
+    file: 'templates/ref-bot-ci-trigger-template.yml'

CMakeLists.txt

@@ -274,7 +274,7 @@ if(TARGET_DEVICE STREQUAL "CPU")
 
     add_library(${ARGV})
     add_library(Allen::${ARGV0} ALIAS ${ARGV0})
-    target_compile_definitions(${ARGV0} PRIVATE ${TARGET_DEFINITION})
+    target_compile_definitions(${ARGV0} PRIVATE ${TARGET_DEFINITION} ODIN_WITHOUT_GAUDI)
     install(TARGETS ${ARGV0}
       EXPORT Allen
       LIBRARY DESTINATION lib)
@@ -298,7 +298,7 @@ if(TARGET_DEVICE STREQUAL "CPU")
     endif()
 
     add_library(Allen::${ARGV0} ALIAS ${ARGV0})
-    target_compile_definitions(${ARGV0} PRIVATE ${TARGET_DEFINITION})
+    target_compile_definitions(${ARGV0} PRIVATE ${TARGET_DEFINITION} ODIN_WITHOUT_GAUDI)
     install(TARGETS ${ARGV0}
       EXPORT Allen
       DESTINATION ${destination})
@@ -345,6 +345,8 @@ elseif(TARGET_DEVICE STREQUAL "HIP")
   message(STATUS "HIP GCC TOOLCHAIN: " ${HIP_TOOLCHAIN_OPTIONS})
   set(HIPCC_OPTIONS "${HIP_RELEASE_OPTIONS} ${HIP_TOOLCHAIN_OPTIONS} -fPIC -std=c++17 --amdgpu-target=${HIP_ARCH}")
 
+  string(APPEND CMAKE_CXX_FLAGS " -Wno-gnu-anonymous-struct -Wno-nested-anon-types -Wno-deprecated-copy")
+
   # Add the origin directory
   set(CMAKE_EXE_LINKER_FLAGS "-Wl,-rpath,./")
 
@@ -371,7 +373,7 @@ elseif(TARGET_DEVICE STREQUAL "HIP")
 
     add_library(Allen::${ARGV0} ALIAS ${ARGV0})
     target_include_directories(${ARGV0} PRIVATE ${HIP_PATH}/include ${ROCM_PATH}/hsa/include)
-    target_compile_definitions(${ARGV0} PRIVATE ${TARGET_DEFINITION})
+    target_compile_definitions(${ARGV0} PRIVATE ${TARGET_DEFINITION} ODIN_WITHOUT_GAUDI)
 
     install(TARGETS ${ARGV0}
       EXPORT Allen
@@ -385,7 +387,6 @@ elseif(TARGET_DEVICE STREQUAL "HIP")
     install(TARGETS ${ARGV0} RUNTIME DESTINATION bin)
   endfunction()
 elseif(TARGET_DEVICE STREQUAL "CUDA")
-  enable_language(CUDA)
 
   set(CMAKE_CUDA_STANDARD 17)
   set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-O3 -g -DNDEBUG --generate-line-info")
@@ -400,8 +401,11 @@ elseif(TARGET_DEVICE STREQUAL "CUDA")
     set(CUDA_VERBOSE_FLAGS "")
   endif()
 
-  string(REGEX REPLACE "toolchain/.*" "toolchain" CXX_COMPILER_DIR ${CMAKE_CXX_COMPILER})
-  string(APPEND CMAKE_CUDA_FLAGS " ${DEVICE_ARCHITECTURE} -ccbin=${CXX_COMPILER_DIR} ${CUDA_TOOLCHAIN_OPTIONS} -Xcudafe --display_error_number --use_fast_math --expt-relaxed-constexpr ${CUDA_VERBOSE_FLAGS}")
+  string(APPEND CMAKE_CUDA_FLAGS " ${DEVICE_ARCHITECTURE} ${CUDA_TOOLCHAIN_OPTIONS} -Xcudafe --display_error_number --use_fast_math --expt-relaxed-constexpr ${CUDA_VERBOSE_FLAGS}")
+  if (NOT CMAKE_CUDA_HOST_COMPILER)
+    string(REGEX REPLACE "toolchain/.*" "toolchain" CXX_COMPILER_DIR ${CMAKE_CXX_COMPILER})
+    string(APPEND CMAKE_CUDA_FLAGS " -ccbin=${CXX_COMPILER_DIR} ")
+  endif()
 
   message(STATUS "CMAKE CUDA FLAGS:" ${CMAKE_CUDA_FLAGS})
   message(STATUS "Detected CUDA include directory: " ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
@@ -420,7 +424,7 @@ elseif(TARGET_DEVICE STREQUAL "CUDA")
       set_property(TARGET ${ARGV0} PROPERTY CUDA_SEPARABLE_COMPILATION ON)
     endif()
     target_include_directories(${ARGV0} PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${PROJECT_BINARY_DIR}/code_generation)
-    target_compile_definitions(${ARGV0} PRIVATE ${TARGET_DEFINITION})
+    target_compile_definitions(${ARGV0} PRIVATE ${TARGET_DEFINITION} ODIN_WITHOUT_GAUDI)
     add_library(Allen::${ARGV0} ALIAS ${ARGV0})
     install(TARGETS ${ARGV0}
       EXPORT Allen
@@ -451,7 +455,7 @@ elseif(TARGET_DEVICE STREQUAL "CUDA")
   function(allen_add_executable)
     add_executable(${ARGV})
     if(SEPARABLE_COMPILATION)
-      set_property(TARGET ${ARGV0} PROPERTY CUDA_SEPARABLE_COMPILATION ON)
+      set_property(TARGET ${ARGV0} PROPERTY CUDA_RESOLVE_DEVICE_SYMBOLS OFF)
     endif()
     target_include_directories(${ARGV0} PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
     target_compile_definitions(${ARGV0} PRIVATE ${TARGET_DEFINITION})
@@ -605,6 +609,7 @@ foreach(header
     InputProvider.h
     OutputHandler.h
     SliceUtils.h
+    FileSystem.h
     TransposeTypes.h)
   list(APPEND AllenHeaders main/include/${header})
 endforeach()
@@ -637,7 +642,6 @@ message(STATUS "CMAKE CXX FLAGS: " ${CMAKE_CXX_FLAGS})
 # Add host library
 allen_add_host_library(HostCommon STATIC
   main/src/BankTypes.cpp
-  main/src/CheckVP.cpp
   main/src/InputReader.cpp
   main/src/InputTools.cpp
   main/src/Logger.cpp
@@ -649,9 +653,9 @@ allen_add_host_library(HostCommon STATIC
   main/src/Transpose.cpp)
 
 target_link_libraries(AllenCommon INTERFACE
-  mdf AllenFS nlohmann_json::nlohmann_json cppgsl::cppgsl)
+  LHCbEvent AllenFS nlohmann_json::nlohmann_json cppgsl::cppgsl)
 target_link_libraries(HostCommon PRIVATE
-  EventModel Gear Backend mdf NonEventData AllenCommon LHCbEvent Boost::iostreams)
+  mdf EventModel Gear Backend mdf NonEventData AllenCommon LHCbEvent Boost::iostreams)
 
 allen_add_host_library(AllenLib SHARED
   main/src/Allen.cpp
@@ -661,30 +665,31 @@ allen_add_host_library(AllenLib SHARED
   main/src/RegisterConsumers.cpp
   main/src/ZMQOutputSender.cpp)
 
-
-
 target_compile_definitions(AllenLib PUBLIC ${TARGET_DEFINITION})
 
-target_link_libraries(AllenLib PUBLIC
-  Stream
-  HostCommon
-  AllenCommon
-  HostEventModel
-  TrackChecking
-  PVChecking
-  CheckClustering
-  SelChecking
-  Monitoring
-  NonEventData
-  AllenZMQ
-  HostCombiners
-  Threads::Threads
-  Backend
-  EventModel
-  Gear
-  mdf
-  LHCbEvent
-  ${CMAKE_DL_LIBS})
+target_link_libraries(AllenLib
+  PRIVATE
+    Backend
+    Stream
+    HostCommon
+    HostEventModel
+    NonEventData
+    TrackChecking
+    PVChecking
+    CheckClustering
+    SelChecking
+    Monitoring
+    HostCombiners
+    EventModel
+    Gear
+    mdf
+    LHCbEvent
+  PUBLIC
+    AllenCommon
+    AllenRuntime
+    Threads::Threads
+    AllenZMQ
+    ${CMAKE_DL_LIBS})
 
 # To propagate filesystem includes/libs
 target_link_libraries(AllenLib PRIVATE AllenFS)
@@ -700,7 +705,7 @@ endif()
 
 allen_add_executable(Allen main/src/main.cpp)
 
-target_link_libraries(Allen PRIVATE AllenLib)
+target_link_libraries(Allen PRIVATE AllenLib NonEventData)
 
 if (NOT STANDALONE)
   target_link_libraries(Allen PRIVATE fmt::fmt)
@@ -717,7 +722,7 @@ if (STANDALONE)
   find_package(Git REQUIRED)
   file(MAKE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/input/PARAM/")
   SET(PARAMFILESROOT "${CMAKE_CURRENT_SOURCE_DIR}/input/PARAM/ParamFiles/")
-  add_custom_command(  
+  add_custom_command(
     OUTPUT "${PARAMFILESROOT}"
     COMMENT "Checking out parameterization files from the LHCb stack"
     COMMAND
@@ -729,4 +734,3 @@ if (STANDALONE)
   add_compile_definitions(PARAMFILESROOTPATH=${PARAMFILESROOT})
   message(STATUS "PARAMFILESROOT set to ${PARAMFILESROOT}")
 endif()
-

Dumpers/BinaryDumpers/CMakeLists.txt

@@ -10,6 +10,7 @@ gaudi_add_library(BinaryDumpers
                     src/lib/TestUTBoards.cpp
                     src/lib/Utils.cpp
                   LINK PUBLIC
+                    AllenCommon
                     Gaudi::GaudiKernel
                     LHCb::DAQEventLib
                     LHCb::MagnetLib
@@ -43,6 +44,7 @@ gaudi_add_module(BinaryDumpersModule
                    TBB::tbb
                    Gaudi::GaudiAlgLib
                    LHCb::AssociatorsBase
+                   LHCb::LHCbAlgsLib
                    LHCb::LinkerEvent
                    LHCb::MCEvent
                    LHCb::DAQEventLib
@@ -54,7 +56,7 @@ gaudi_add_module(BinaryDumpersModule
                    LHCb::FTDetLib
                    LHCb::MuonDAQLib
                    Rec::PrKernel
-                   yaml-cpp
+                   yaml-cpp::yaml-cpp
                    BinaryDumpers)
 
 gaudi_add_tests(QMTest)

Dumpers/BinaryDumpers/include/Dumpers/PyAllenHelper.h

@@ -5,6 +5,8 @@
 #include <Dumpers/IUpdater.h>
 #include <Allen/InputProvider.h>
 
+#include <zmq/zmq.hpp>
+
 /**
  * @brief      Helper function to cast the LHCb-implementation of the Allen
  *             non-event data manager to its shared interface
@@ -26,5 +28,7 @@ struct shared_wrap {
   }
 };
 
+uintptr_t czmq_context(zmq::context_t& ctx) { return reinterpret_cast<uintptr_t>(ctx.operator void*()); }
+
 // template cast_service<Allen::NonEventData::IUpdater>;
 // template cast_service<IInputProvider>(IService* svc);

Dumpers/BinaryDumpers/include/Dumpers/Utils.h

@@ -16,13 +16,7 @@
 #include <type_traits>
 #include <vector>
 
-template<class Tuple, class F>
-void for_each(Tuple&& tup, F&& f)
-{
-  std::apply(
-    [f = std::forward<F>(f)](auto&&... args) { (std::invoke(f, std::forward<decltype(args)>(args)), ...); },
-    std::forward<Tuple>(tup));
-}
+#include <Common.h>
 
 template<class T>
 void optional_resize(T&, size_t)

Dumpers/BinaryDumpers/options/allen.py

@@ -3,7 +3,10 @@
 # (c) Copyright 2018-2021 CERN for the benefit of the LHCb Collaboration      #
 ###############################################################################
 import os
+import sys
+import zmq
 from Configurables import ApplicationMgr
+from Configurables import Gaudi__RootCnvSvc as RootCnvSvc
 from Allen.config import setup_allen_non_event_data_service
 from PyConf.application import (configure, setup_component, ComponentConfig,
                                 ApplicationOptions)
@@ -27,7 +30,7 @@ interpreter.Declare("#include <Allen/Provider.h>")
 interpreter.Declare("#include <Dumpers/PyAllenHelper.h>")
 
 sequence_default = os.path.join(os.environ['ALLEN_INSTALL_DIR'], 'constants',
-                                'hlt1_pp_default')
+                                'hlt1_pp_default.json')
 
 
 def cast_service(return_type, svc):
@@ -52,13 +55,13 @@ parser.add_argument("-v", dest="verbosity", default=3)
 parser.add_argument("-p", dest="print_memory", default=0)
 parser.add_argument("--sequence", dest="sequence", default=sequence_default)
 parser.add_argument("-s", dest="slices", default=2)
+parser.add_argument("--mdf", dest="mdf", default="")
+parser.add_argument("--mep", dest="mep", default="")
 parser.add_argument(
-    "--mdf",
-    dest="mdf",
-    default=os.path.join(
-        allen_dir, "input", "minbias", "mdf",
-        "MiniBrunel_2018_MinBias_FTv4_DIGI_retinacluster_v1.mdf"))
-parser.add_argument("--mep", dest="mep", default=None)
+    "--mep-mask-source-id-top-5",
+    action="store_true",
+    dest="mask_top5",
+    default=False)
 parser.add_argument(
     "--reuse-meps",
     action="store_true",
@@ -81,6 +84,8 @@ parser.add_argument(
     help="Add profiler start and stop calls",
 )
 parser.add_argument("--output-file", dest="output_file", default=None)
+parser.add_argument(
+    "--output-batch-size", dest="output_batch_size", default=10)
 parser.add_argument(
     "--monitoring-save-period", dest="mon_save_period", default=0)
 parser.add_argument(
@@ -108,6 +113,8 @@ parser.add_argument(
     default=300,
     help="How long to run when reusing MEPs",
 )
+parser.add_argument(
+    "--tags", dest="tags", default="dddb-20171122,sim-20180530-vc-md100")
 
 args = parser.parse_args()
 
@@ -115,16 +122,14 @@ runtime_lib = None
 if args.profile == "CUDA":
     runtime_lib = ctypes.CDLL("libcudart.so")
 
+dddb_tag, conddb_tag = args.tags.split(',')
+
 options = ApplicationOptions(_enabled=False)
 options.simulation = True
 options.data_type = 'Upgrade'
 options.input_type = 'MDF'
-options.dddb_tag = "dddb-20171122"
-options.conddb_tag = "sim-20180530-vc-md100"
-
-# tags for FEST sample from 10/2021
-# dddb_tag="dddb-20210617"
-# conddb_tag="sim-20210617-vc-md100")
+options.dddb_tag = dddb_tag
+options.conddb_tag = conddb_tag
 
 options.finalize()
 config = ComponentConfig()
@@ -132,24 +137,29 @@ config = ComponentConfig()
 # Some extra stuff for timing table
 extSvc = ["ToolSvc", "AuditorSvc", "ZeroMQSvc"]
 
-if args.mep is not None:
+# RootCnvSvc becauce it sets up a bunch of ROOT IO stuff
+rootSvc = RootCnvSvc("RootCnvSvc", EnableIncident=1)
+ApplicationMgr().ExtSvc += ["Gaudi::IODataManager/IODataManager", rootSvc]
+
+if args.mep:
     extSvc += ["AllenConfiguration", "MEPProvider"]
     from Configurables import MEPProvider, AllenConfiguration
 
     allen_conf = AllenConfiguration("AllenConfiguration")
     allen_conf.JSON = args.sequence
-    allen_conf.OutputLevel = 2
+    allen_conf.OutputLevel = 3
 
     mep_provider = MEPProvider()
     mep_provider.NSlices = args.slices
     mep_provider.EventsPerSlice = args.events_per_slice
-    mep_provider.OutputLevel = 2
+    mep_provider.OutputLevel = 3
     # Number of MEP buffers and number of transpose/offset threads
     mep_provider.BufferConfig = (10, 8)
     mep_provider.TransposeMEPs = False
     mep_provider.SplitByRun = False
     mep_provider.Source = "Files"
-    mep_dir = args.mep
+    mep_provider.MaskSourceIDTop5 = args.mask_top5
+    mep_dir = os.path.expandvars(args.mep)
     if os.path.isdir(mep_dir):
         mep_provider.Connections = sorted([
             os.path.join(mep_dir, mep_file) for mep_file in os.listdir(mep_dir)
@@ -165,8 +175,8 @@ if args.mep is not None:
     # mep_provider.BufferNUMA = [0, 1, 0, 1, 0, 1, 0, 1, 0, 1]
     mep_provider.EvtMax = -1 if args.n_events == 0 else args.n_events
 
-config.add(
-    ApplicationMgr(EvtSel="NONE", ExtSvc=ApplicationMgr().ExtSvc + extSvc))
+ApplicationMgr().EvtSel = "NONE"
+ApplicationMgr().ExtSvc += extSvc
 
 # Copeid from PyConf.application.configure_input
 config.add(
@@ -186,7 +196,10 @@ config.update(configure(options, cf_node))
 
 # Start Gaudi and get the AllenUpdater service
 gaudi = AppMgr()
-gaudi.initialize()
+sc = gaudi.initialize()
+if not sc.isSuccess():
+    sys.exit("Failed to initialize AppMgr")
+
 svc = gaudi.service("AllenUpdater", interface=gbl.IService)
 zmqSvc = gaudi.service("ZeroMQSvc", interface=gbl.IZeroMQSvc)
 
@@ -198,10 +211,11 @@ for flag, value in [("g", args.det_folder),
                     ("params", os.getenv("PARAMFILESROOT")),
                     ("n", args.n_events), ("t", args.threads),
                     ("r", args.repetitions), ("output-file", args.output_file),
+                    ("output-batch-size", args.output_batch_size),
                     ("m", args.reserve), ("v", args.verbosity),
                     ("p", args.print_memory),
                     ("sequence", os.path.expandvars(args.sequence)),
-                    ("s", args.slices), ("mdf", args.mdf),
+                    ("s", args.slices), ("mdf", os.path.expandvars(args.mdf)),
                     ("cpu-offload", args.cpu_offload),
                     ("disable-run-changes", int(not args.enable_run_changes)),
                     ("monitoring-save-period", args.mon_save_period),
@@ -221,28 +235,65 @@ else:
     provider = gbl.Allen.make_provider(options)
 output_handler = gbl.Allen.output_handler(provider, zmqSvc, options)
 
-gaudi.start()
-
 # run Allen
 gbl.allen.__release_gil__ = 1
 
+# get the libzmq context out of the zmqSvc and use it to instantiate
+# the pyzmq Context
+cpp_ctx = zmqSvc.context()
+c_ctx = gbl.czmq_context(cpp_ctx)
+ctx = zmq.Context.shadow(c_ctx)
 
-def sleep_fun():
-    print("sleeping")
-    sleep(args.runtime)
-    gaudi.stop()
-
+ctx.linger = -1
+control = ctx.socket(zmq.PAIR)
 
 if args.reuse_meps:
-    sleep_thread = Thread(target=sleep_fun)
-    sleep_thread.start()
+    gaudi.start()
+else:
+    # C++ and python strings
+    connection = "inproc:///py_allen_control"
+    con = gbl.std.string(connection)
+    control.bind(connection)
+
+
+# Call the main Allen function from a thread so we can control it from
+# the main thread
+def allen_thread():
+    if args.profile == "CUDA":
+        runtime_lib.cudaProfilerStart()
+
+    gbl.allen(options, updater, shared_wrap(gbl.IInputProvider, provider),
+              output_handler, zmqSvc, con.c_str())
+
+    if args.profile == "CUDA":
+        runtime_lib.cudaProfilerStop()
 
-if args.profile == "CUDA":
-    runtime_lib.cudaProfilerStart()
-gbl.allen(options, updater, shared_wrap(gbl.IInputProvider, provider),
-          output_handler, zmqSvc, con.c_str())
-if args.profile == "CUDA":
-    runtime_lib.cudaProfilerStop()
+
+allen_thread = Thread(target=allen_thread)
+allen_thread.start()
 
 if args.reuse_meps:
-    sleep_thread.join()
+    print("sleeping")
+    sleep(args.runtime)
+    gaudi.stop()
+else:
+    # READY
+    msg = control.recv()
+    assert (msg.decode() == "READY")
+    gaudi.start()
+    control.send(b"START")
+    msg = control.recv()
+    assert (msg.decode() == "RUNNING")
+    sleep(5)
+    control.send(b"STOP")
+    msg = control.recv()
+    assert (msg.decode() == "READY")
+    if args.profile == "CUDA":
+        runtime_lib.cudaProfilerStop()
+    gaudi.stop()
+    gaudi.finalize()
+    control.send(b"RESET")
+    msg = control.recv()
+    assert (msg.decode() == "NOT_READY")
+
+allen_thread.join()

Dumpers/BinaryDumpers/src/TransposeRawBanks.cpp

@@ -6,14 +6,20 @@
 #include <string>
 #include <unordered_map>
 #include <vector>
-#include <GaudiAlg/MergingTransformer.h>
+
 #include <AIDA/IHistogram1D.h>
+
+#include <GaudiAlg/MergingTransformer.h>
+#include <GaudiAlg/GaudiHistoAlg.h>
+#include <GaudiKernel/GaudiException.h>
+
 #include <Event/ODIN.h>
 #include <Event/RawBank.h>
 #include <Event/RawEvent.h>
-#include <GaudiAlg/GaudiHistoAlg.h>
+
 #include <Dumpers/Utils.h>
-#include <GaudiKernel/GaudiException.h>
+
+#include <BankTypes.h>
 
 template<typename T>
 using VOC = Gaudi::Functional::vector_of_const_<T>;
@@ -41,17 +47,16 @@ using VOC = Gaudi::Functional::vector_of_const_<T>;
  *  @author Roel Aaij
  *  @date   2018-08-27
  */
-class TransposeRawBanks
-  : public Gaudi::Functional::MergingTransformer<
-      std::array<std::tuple<std::vector<char>, int>, LHCb::RawBank::types().size()>(VOC<LHCb::RawEvent*> const&),
-      Gaudi::Functional::Traits::BaseClass_t<GaudiHistoAlg>> {
+class TransposeRawBanks : public Gaudi::Functional::MergingTransformer<
+                            std::array<TransposedBanks, LHCb::RawBank::types().size()>(VOC<LHCb::RawEvent*> const&),
+                            Gaudi::Functional::Traits::BaseClass_t<GaudiHistoAlg>> {
 public:
   /// Standard constructor
   TransposeRawBanks(const std::string& name, ISvcLocator* pSvcLocator);
 
   StatusCode initialize() override;
 
-  std::array<std::tuple<std::vector<char>, int>, LHCb::RawBank::types().size()> operator()(
+  std::array<TransposedBanks, LHCb::RawBank::types().size()> operator()(
     VOC<LHCb::RawEvent*> const& rawEvents) const override;
 
 private:
@@ -86,11 +91,11 @@ StatusCode TransposeRawBanks::initialize()
   return StatusCode::SUCCESS;
 }
 
-std::array<std::tuple<std::vector<char>, int>, LHCb::RawBank::types().size()> TransposeRawBanks::operator()(
+std::array<TransposedBanks, LHCb::RawBank::types().size()> TransposeRawBanks::operator()(
   VOC<LHCb::RawEvent*> const& rawEvents) const
 {
 
-  std::array<std::tuple<std::vector<char>, int>, LHCb::RawBank::types().size()> output;
+  std::array<TransposedBanks, LHCb::RawBank::types().size()> output;
   std::array<LHCb::RawBank::View, LHCb::RawBank::types().size()> rawBanks;
 
   for (auto const* rawEvent : rawEvents) {
@@ -109,12 +114,16 @@ std::array<std::tuple<std::vector<char>, int>, LHCb::RawBank::types().size()> Tr
     auto const& banks = rawBanks[bt];
     if (banks.empty()) continue;
 
-    const uint32_t number_of_rawbanks = banks.size();
+    const uint32_t nBanks = banks.size();
     uint32_t offset = 0;
 
     std::vector<uint32_t> bankOffsets;
-    std::vector<uint32_t> bankData;
     bankOffsets.push_back(0);
+    std::vector<uint32_t> bankData;
+    std::vector<uint16_t> bankSizes;
+    bankSizes.reserve(nBanks);
+    std::vector<uint8_t> bankTypes;
+    bankTypes.reserve(nBanks);
 
     for (auto& bank : banks) {
       const uint32_t sourceID = static_cast<uint32_t>(bank->sourceID());
@@ -141,12 +150,15 @@ std::array<std::tuple<std::vector<char>, int>, LHCb::RawBank::types().size()> Tr
         offset++;
       }
       bankOffsets.push_back(offset * sizeof(uint32_t));
+      bankSizes.push_back(bank->size());
+      bankTypes.push_back(static_cast<uint8_t>(bank->type()));
     }
 
     // Dumping number_of_rawbanks + 1 offsets!
     DumpUtils::Writer bank_buffer;
-    bank_buffer.write(number_of_rawbanks, bankOffsets, bankData);
-    output[bt] = std::tuple {bank_buffer.buffer(), banks[0]->version()};
+    bank_buffer.write(nBanks, bankOffsets, bankData);
+    output[bt] =
+      TransposedBanks {bank_buffer.buffer(), std::move(bankSizes), std::move(bankTypes), banks[0]->version()};
   }
   return output;
 }

Dumpers/BinaryDumpers/tests/qmtest/lhcb_geometry_allen_event_loop.qmt

@@ -15,10 +15,12 @@
   <argument name="program"><text>python</text></argument>
   <argument name="args"><set>
     <text>../../options/allen.py</text>
+    <text>--mdf</text><text>root://eoslhcb.cern.ch///eos/lhcb/wg/rta/samples/mc/RetinaClusters_FTv4_MDF/00067189_retinaCluster_no63.mdf</text>
+    <text>-n</text><text>10000</text>
   </set></argument>
   <arguement name="timeout"><integer>600</integer></arguement>
   <argument name="validator"><text>
-countErrorLines({"FATAL":0,"ERROR":0,"WARNING":2})
+countErrorLines({"FATAL":0,"ERROR":0,"WARNING":0})
 
 import re
 

Rec/Allen/Allen/AlgorithmConversionTools.h

@@ -10,6 +10,7 @@
 \*****************************************************************************/
 #pragma once
 
+#include <algorithm>
 #include <cstdio>
 #include <BackendCommon.h>
 #include <Argument.cuh>

Rec/Allen/CMakeLists.txt

@@ -9,6 +9,7 @@ sequence as part of a Gaudi/LHCb
 application
 #]========================================]
 
+if(TARGET_DEVICE STREQUAL "CPU")
 include(GenerateConfiguration)
 
 gaudi_add_module(AllenWrapper
@@ -30,7 +31,25 @@ gaudi_add_module(AllenWrapper
                    src/TestVeloClusters.cpp
                  LINK
                    AllenLib
+		   Backend
+    		   Stream
+   		   HostCommon
+   		   HostEventModel
+    		   NonEventData
+    		   TrackChecking
+    		   PVChecking
+    		   CheckClustering
+    		   SelChecking
+    		   Monitoring
+    		   HostCombiners
+    		   EventModel
+    		   Gear
+    		   mdf
+    		   LHCbEvent
                    EventModel
+		   MuonCommon
+                   EventModel
+                   Gear
                    Gaudi::GaudiAlgLib
                    LHCb::DAQEventLib
                    LHCb::DAQKernelLib
@@ -46,14 +65,30 @@ gaudi_add_module(AllenWrapper
 target_include_directories(AllenWrapper PRIVATE ${PROJECT_BINARY_DIR}/code_generation)
 add_dependencies(AllenWrapper struct_to_tuple)
 
+
 gaudi_add_module(AllenAlgorithms
                  SOURCES
                    ${WRAPPED_ALGORITHM_SOURCES}
                  LINK
                    AllenLib
+		   Backend
+    		   Stream
+   		   HostCommon
+   		   HostEventModel
+    		   NonEventData
+    		   TrackChecking
+    		   PVChecking
+    		   CheckClustering
+    		   SelChecking
+    		   Monitoring
+    		   HostCombiners
+    		   EventModel
+    		   Gear
+    		   mdf
+    		   LHCbEvent
                    EventModel
-				   MuonCommon
-				   WrapperInterface
+		   MuonCommon
+		   WrapperInterface
                    Gaudi::GaudiAlgLib
                    LHCb::DAQEventLib
                    LHCb::DAQKernelLib
@@ -78,6 +113,8 @@ endif()
 target_compile_definitions(AllenWrapper PRIVATE -D${TARGET_DEFINITION})
 target_compile_definitions(AllenAlgorithms PRIVATE -D${TARGET_DEFINITION})
 
+endif()
+
 gaudi_install(PYTHON)
 
 gaudi_add_tests(QMTest)

Rec/Allen/python/Allen/qmtest/exclusions.py

@@ -13,20 +13,27 @@ from GaudiConf.QMTest.LHCbTest import BlockSkipper
 from GaudiConf.QMTest.LHCbExclusions import preprocessor as LHCbPreprocessor
 
 remove_throughput = LineSkipper(regexps=[
+    # Processing complete messages
+    r"Processing complete",
     # Throughput messages
     r"\s*(\d+\.\d+)\s+events/s",
     r"Ran test for (\d+\.\d+)\s+seconds",
+    r"Providing banks for",
     r"Providing events in.*",
     r"Opened\s.*",
     r"Cannot read more data.*"
 ])
 
+skip_config = BlockSkipper("User ApplicationOptions",
+                           "Application Manager Configured successfully")
+
 skip_options = BlockSkipper("Requested options:", "Ignore signals to update")
 
 skip_rates = BlockSkipper("rate_validator validation:", "Inclusive:")
 
-skip_sequence = BlockSkipper("Sequence:",
-                             "Starting timer for throughput measurement")
+skip_sequence = BlockSkipper("Sequence:", "make_selreps")
+
+preprocessor_with_rates = (LHCbPreprocessor + skip_config + remove_throughput +
+                           skip_options + skip_sequence)
 
-preprocessor = (LHCbPreprocessor + remove_throughput + skip_options +
-                skip_rates + skip_sequence)
+preprocessor = preprocessor_with_rates + skip_rates

Rec/Allen/src/ProvideRuntimeOptions.cpp

@@ -21,8 +21,8 @@
 #include <Constants.cuh>
 #include <Logger.h>
 
-class ProvideRuntimeOptions final : public Gaudi::Functional::Transformer<RuntimeOptions(
-                                      std::array<std::tuple<std::vector<char>, int>, LHCb::RawBank::LastType> const&)> {
+class ProvideRuntimeOptions final
+  : public Gaudi::Functional::Transformer<RuntimeOptions(std::array<TransposedBanks, LHCb::RawBank::LastType> const&)> {
 
 public:
   /// Standard constructor
@@ -31,8 +31,7 @@ public:
   StatusCode initialize() override;
 
   /// Algorithm execution
-  RuntimeOptions operator()(
-    std::array<std::tuple<std::vector<char>, int>, LHCb::RawBank::LastType> const& allen_banks) const override;
+  RuntimeOptions operator()(std::array<TransposedBanks, LHCb::RawBank::LastType> const& allen_banks) const override;
 
 private:
   Gaudi::Property<std::string> m_monitorFile {this, "MonitorFile", "allen_monitor.root"};
@@ -57,7 +56,7 @@ StatusCode ProvideRuntimeOptions::initialize()
 }
 
 RuntimeOptions ProvideRuntimeOptions::operator()(
-  std::array<std::tuple<std::vector<char>, int>, LHCb::RawBank::LastType> const& allen_banks) const
+  std::array<TransposedBanks, LHCb::RawBank::LastType> const& allen_banks) const
 {
   const unsigned number_of_repetitions = 1;
   const bool do_check = false;

Rec/Allen/src/RunAllen.cpp

@@ -178,7 +178,7 @@ StatusCode RunAllen::finalize()
 /** Calls Allen for one event
  */
 std::tuple<bool, HostBuffers> RunAllen::operator()(
-  const std::array<std::tuple<std::vector<char>, int>, LHCb::RawBank::types().size()>& allen_banks,
+  const std::array<TransposedBanks, LHCb::RawBank::types().size()>& allen_banks,
   const LHCb::ODIN& odin) const
 {
   // Ensure non-event-data is up-to-date

Rec/Allen/src/RunAllen.h

@@ -31,10 +31,9 @@
 // STL includes
 #include <deque>
 
-class RunAllen final
-  : public Gaudi::Functional::MultiTransformerFilter<std::tuple<HostBuffers>(
-      const std::array<std::tuple<std::vector<char>, int>, LHCb::RawBank::types().size()>& allen_banks,
-      const LHCb::ODIN& odin)> {
+class RunAllen final : public Gaudi::Functional::MultiTransformerFilter<std::tuple<HostBuffers>(
+                         const std::array<TransposedBanks, LHCb::RawBank::types().size()>& allen_banks,
+                         const LHCb::ODIN& odin)> {
 public:
   /// Standard constructor
   RunAllen(const std::string& name, ISvcLocator* pSvcLocator);
@@ -47,7 +46,7 @@ public:
 
   /// Algorithm execution
   std::tuple<bool, HostBuffers> operator()(
-    const std::array<std::tuple<std::vector<char>, int>, LHCb::RawBank::types().size()>& allen_banks,
+    const std::array<TransposedBanks, LHCb::RawBank::types().size()>& allen_banks,
     const LHCb::ODIN& odin) const override;
 
 private:

Rec/Allen/src/TestUTHits.cpp

@@ -48,7 +48,7 @@ void TestUTHits::
 operator()(HostBuffers const& host_buffers, LHCb::MCHits const& mc_hits, LHCb::Pr::UT::Hits const& hit_handler) const
 {
   if (host_buffers.host_number_of_selected_events == 0) {
-    warning() << "No events from Allen. Returning" << endmsg;
+    info() << "Allen UT decoding was not run." << endmsg;
     return;
   }
 

Rec/Allen/tests/qmtest/mdf_input.qmt

@@ -16,16 +16,17 @@
   <argument name="args"><set>
     <text>${BINARYDUMPERSROOT}/options/allen.py</text>
     <text>--monitoring-filename</text><text>mdf_input_hists.root</text>
-    <text>--mdf</text><text>${ALLEN_PROJECT_ROOT}/input/minbias/mdf/MiniBrunel_2018_MinBias_FTv4_DIGI_retinacluster_v1.mdf</text>
+    <text>--mdf</text><text>root://eoslhcb.cern.ch///eos/lhcb/wg/rta/samples/mc/RetinaClusters_FTv4_MDF/00067189_retinaCluster_no63.mdf</text>
+    <text>-n</text><text>10000</text>
   </set></argument>
-  <argument name="reference"><text>../refs/allen_10ev.ref</text></argument>
+  <argument name="reference"><text>../refs/allen_event_loop.ref</text></argument>
   <argument name="timeout"><integer>600</integer></argument>
   <argument name="validator"><text>
 
 from Allen.qmtest.exclusions import preprocessor
 validateWithReference(preproc = preprocessor)
 
-countErrorLines({"FATAL":0,"ERROR":0,"WARNING":2})
+countErrorLines({"FATAL":0,"ERROR":0,"WARNING":0})
 
   </text></argument>
 </extension>

Rec/Allen/tests/refs/allen_10ev.ref

-ApplicationMgr    SUCCESS
-====================================================================================================================================
-====================================================================================================================================
-ApplicationMgr       INFO Application Manager Configured successfully
-DetectorPersist...   INFO Added successfully Conversion service:XmlCnvSvc
-DetectorDataSvc   SUCCESS Detector description database: git:/lhcb.xml
-EventClockSvc.F...   INFO Event times generated from 0 with steps of 0
-DeFTDetector         INFO Current FT geometry version =   63
-DumpFTGeometry       INFO Conditions DB is compatible with FT bank version 4, 5, 6.
-DumpMuonGeometry     INFO Registering /dd/Conditions/ReadoutConf/Muon/Cabling/M2/Cabling
-DumpMuonGeometry     INFO Registering /dd/Conditions/ReadoutConf/Muon/Cabling/M3/Cabling
-DumpMuonGeometry     INFO Registering /dd/Conditions/ReadoutConf/Muon/Cabling/M4/Cabling
-DumpMuonGeometry     INFO Registering /dd/Conditions/ReadoutConf/Muon/Cabling/M5/Cabling
-DumpMuonGeometry     INFO Number of tiles in tell1 table: 19968
-MagneticFieldSvc     INFO Map scaled by factor 1 with polarity internally used: -1 signed relative current: -1
-ToolSvc.PrUTMag...   INFO Start generation of VeloUT Bdl LUTs
-ToolSvc.PrUTMag...   INFO Generation of VeloUT Bdl LUTs finished
-ToolSvc.PrUTMag...   INFO Start generation of VeloUT deflection LUTs
-ToolSvc.PrUTMag...   INFO Generation of VeloUT deflection LUTs finished
-EventLoopMgr      WARNING Unable to locate service "EventSelector"
-EventLoopMgr      WARNING No events will be processed from external input.
-ApplicationMgr       INFO Application Manager Initialized successfully
-ApplicationMgr       INFO Application Manager Started successfully
-Using 2 input slices.
-Input complete
-Processing complete
-ApplicationMgr       INFO Application Manager Stopped successfully
-EventLoopMgr         INFO Histograms converted successfully according to request.
-ToolSvc              INFO Removing all tools created by ToolSvc
-ApplicationMgr       INFO Application Manager Finalized successfully
-ApplicationMgr       INFO Application Manager Terminated successfully

Rec/Allen/tests/refs/allen_event_loop.ref

+DetectorPersistencySvc                 INFO Added successfully Conversion service:XmlCnvSvc
+DetectorDataSvc                     SUCCESS Detector description database: git:/lhcb.xml
+EventClockSvc.FakeEventTime            INFO Event times generated from 0 with steps of 0
+MagneticFieldSvc                       INFO Map scaled by factor 1 with polarity internally used: -1 signed relative current: -1
+DumpMuonGeometry                       INFO Registering /dd/Conditions/ReadoutConf/Muon/Cabling/M2/Cabling
+DumpMuonGeometry                       INFO Registering /dd/Conditions/ReadoutConf/Muon/Cabling/M3/Cabling
+DumpMuonGeometry                       INFO Registering /dd/Conditions/ReadoutConf/Muon/Cabling/M4/Cabling
+DumpMuonGeometry                       INFO Registering /dd/Conditions/ReadoutConf/Muon/Cabling/M5/Cabling
+DumpMuonGeometry                       INFO Number of tiles in tell1 table: 19968
+ToolSvc.PrUTMagnetTool                 INFO Start generation of VeloUT Bdl LUTs
+ToolSvc.PrUTMagnetTool                 INFO Generation of VeloUT Bdl LUTs finished
+ToolSvc.PrUTMagnetTool                 INFO Start generation of VeloUT deflection LUTs
+ToolSvc.PrUTMagnetTool                 INFO Generation of VeloUT deflection LUTs finished
+HLTControlFlowMgr                      INFO Start initialization
+HLTControlFlowMgr                      INFO Will not use an EventSelector.
+HLTControlFlowMgr                      INFO Concurrency level information:
+HLTControlFlowMgr                      INFO  o Number of events slots: 1
+HLTControlFlowMgr                      INFO  o TBB thread pool size:  'ThreadPoolSize':1
+ApplicationMgr                         INFO Application Manager Initialized successfully
+Using 2 input slices.
+DeFTDetector                           INFO Current FT geometry version =   63
+ApplicationMgr                         INFO Application Manager Started successfully
+DumpFTGeometry                         INFO Conditions DB is compatible with FT bank version 4, 5, 6.
+Starting timer for throughput measurement
+Input complete
+ApplicationMgr                         INFO Application Manager Stopped successfully
+HLTControlFlowMgr                      INFO
+HLTControlFlowMgr                      INFO
+LAZY_AND: allen_non_event_data                #=1       Sum=1           Eff=|( 100.0000 +- 0.00000 )%|
+ NONLAZY_OR: allen_non_event_data_converters  #=1       Sum=1           Eff=|( 100.0000 +- 0.00000 )%|
+  DumpBeamline/DumpBeamline                   #=1       Sum=1           Eff=|( 100.0000 +- 0.00000 )%|
+  DumpCaloGeometry/DumpCaloGeometry           #=1       Sum=1           Eff=|( 100.0000 +- 0.00000 )%|
+  DumpVPGeometry/DumpVPGeometry               #=1       Sum=1           Eff=|( 100.0000 +- 0.00000 )%|
+  DumpMagneticField/DumpMagneticField         #=1       Sum=1           Eff=|( 100.0000 +- 0.00000 )%|
+  DumpFTGeometry/DumpFTGeometry               #=1       Sum=1           Eff=|( 100.0000 +- 0.00000 )%|
+ NONLAZY_OR: allen_non_event_data_producers   #=1       Sum=1           Eff=|( 100.0000 +- 0.00000 )%|
+  AllenTESProducer/AllenTESProducer           #=1       Sum=1           Eff=|( 100.0000 +- 0.00000 )%|
+  AllenTESProducer/AllenTESProducer#1         #=1       Sum=1           Eff=|( 100.0000 +- 0.00000 )%|
+  AllenTESProducer/AllenTESProducer#2         #=1       Sum=1           Eff=|( 100.0000 +- 0.00000 )%|
+  AllenTESProducer/AllenTESProducer#3         #=1       Sum=1           Eff=|( 100.0000 +- 0.00000 )%|
+  AllenTESProducer/AllenTESProducer#4         #=1       Sum=1           Eff=|( 100.0000 +- 0.00000 )%|
+HLTControlFlowMgr                      INFO Histograms converted successfully according to request.
+ToolSvc                                INFO Removing all tools created by ToolSvc
+ApplicationMgr                         INFO Application Manager Finalized successfully
+ApplicationMgr                         INFO Application Manager Terminated successfully

backend/CMakeLists.txt

@@ -3,23 +3,29 @@
 ###############################################################################
 set(backend_sources src/CPUID.cpp)
 
+add_library(AllenRuntime INTERFACE)
+install(TARGETS AllenRuntime EXPORT Allen)
+
 if(TARGET_DEVICE STREQUAL "CPU")
   list(APPEND backend_sources src/CPUBackend.cpp)
   list(APPEND backend_sources src/HalfType.cpp)
 elseif(TARGET_DEVICE STREQUAL "CUDA")
   list(APPEND backend_sources src/CUDABackend.cpp)
+  target_link_libraries(AllenRuntime INTERFACE CUDA::cudart)
 elseif(TARGET_DEVICE STREQUAL "HIP")
   list(APPEND backend_sources src/HIPBackend.cpp)
+  target_link_libraries(AllenRuntime INTERFACE ${HIP_RUNTIME_LIB})
 endif()
 
-# Backend library
-allen_add_host_library(Backend STATIC ${backend_sources})
-target_link_libraries(Backend PUBLIC Gear AllenCommon umesimd::umesimd)
-target_link_libraries(Backend PRIVATE HostCommon)
-target_include_directories(Backend PUBLIC
+target_include_directories(AllenRuntime INTERFACE
   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/Backend>
   $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
+
+# Backend library
+allen_add_host_library(Backend STATIC ${backend_sources})
+target_link_libraries(Backend PUBLIC AllenRuntime Gear AllenCommon umesimd::umesimd)
+target_link_libraries(Backend PRIVATE HostCommon)
 target_compile_definitions(Backend PUBLIC ${TARGET_DEFINITION})
 
 set(BackendHeaders)
@@ -36,11 +42,5 @@ foreach(header
   list(APPEND BackendHeaders include/${header})
 endforeach()
 
-if(TARGET_DEVICE STREQUAL "CUDA")
-  target_link_libraries(Backend PUBLIC CUDA::cudart)
-elseif(TARGET_DEVICE STREQUAL "HIP")
-  target_link_libraries(Backend PUBLIC ${HIP_RUNTIME_LIB})
-endif()
-
 install(FILES ${BackendHeaders}
         DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/Backend)

backend/include/BackendCommon.h

@@ -18,6 +18,7 @@
 #include <string>
 #include <cassert>
 #include <array>
+#include <gsl/gsl>
 #include "AllenTypeTraits.h"
 #include "BackendCommonInterface.h"
 
@@ -96,9 +97,11 @@ namespace Allen {
 #include <cmath>
 #endif
 
-// Replacement for gsl::span in device code
+// Replacement for gsl::span in device code when building with HIP,
+// gsl::span works for CUDA and CPU
 namespace Allen {
   namespace device {
+#if defined(TARGET_DEVICE_HIP)
     template<class T>
     struct span {
       T* __ptr = nullptr;
@@ -123,6 +126,9 @@ namespace Allen {
       constexpr __device__ __host__ T& operator[](int i) { return __ptr[i]; }
       constexpr __device__ __host__ const T& operator[](int i) const { return __ptr[i]; }
     };
+#else
+    using gsl::span;
+#endif
   } // namespace device
 } // namespace Allen