diff --git a/DAQ/MDF/include/MDF/IOHandlerFileRead.h b/DAQ/MDF/include/MDF/IOHandlerFileRead.h
index 25b600266da60f0edb50eb4ce5f4e41b1c040296..c6ff4ecf1a7b544f2d3643213383c1d4b13ac269 100644
--- a/DAQ/MDF/include/MDF/IOHandlerFileRead.h
+++ b/DAQ/MDF/include/MDF/IOHandlerFileRead.h
@@ -17,6 +17,7 @@
#include <Event/RawEvent.h>
+#include <Gaudi/Accumulators.h>
#include <GaudiKernel/IService.h>
#include <GaudiKernel/SmartIF.h>
#include <GaudiUtils/IIODataManager.h>
@@ -38,7 +39,8 @@ namespace LHCb::MDF {
/// constructor
template <typename Owner>
- FileReadInputHandler( Owner& owner, std::vector<std::string> const& input ) : m_input( input ) {
+ FileReadInputHandler( Owner& owner, std::vector<std::string> const& input )
+ : m_input( input ), m_eventSize{&owner, "event size statistics (KBytes)"} {
// Retrieve conversion service handling event iteration
SmartIF<IService> service = owner.service( "Gaudi::IODataManager/IODataManager" );
m_ioMgr = service.as<Gaudi::IIODataManager>();
@@ -124,6 +126,10 @@ namespace LHCb::MDF {
LHCb::RawDataConnection m_curConnection{0, ""};
/// Reference to file manager service
SmartIF<Gaudi::IIODataManager> m_ioMgr;
+ /// minimum buffer size to allocate, may evolve over time based
+ /// on the previous events encountered. It will actually double
+ /// each time an event does cannot fit in the entire fuffer
+ unsigned int m_minimumBufferSize{2000000u};
/**
* This member allow to deal with the case where an MDFHEader
@@ -133,6 +139,9 @@ namespace LHCb::MDF {
* Used only in prefetchEvents
*/
std::optional<MDFHeader> m_curHeader{};
+
+ /// counter of event sizes (KBytes)
+ mutable Gaudi::Accumulators::StatCounter<unsigned long> m_eventSize;
};
} // namespace LHCb::MDF
@@ -150,7 +159,7 @@ std::shared_ptr<LHCb::MDF::ByteBuffer> LHCb::MDF::FileReadInputHandler::prefetch
// 50K is low, this is to ensure that in most cases we will not reallocate the event vector
// We also set a minimum of 2M so that most big events can be handled anyway, even if people
// ask for very few events
- unsigned int bufferSize = std::max( nbEvents * 50000, 2000000u );
+ unsigned int bufferSize = std::max( nbEvents * 50000, m_minimumBufferSize );
auto buffer = unique_ptr_free<std::byte>{reinterpret_cast<std::byte*>( std::malloc( bufferSize ) )};
if ( nullptr == buffer ) { std::runtime_error{"Unable to allocate memory for new buffer"}; }
// create associated events vector and reserve space
@@ -173,10 +182,19 @@ std::shared_ptr<LHCb::MDF::ByteBuffer> LHCb::MDF::FileReadInputHandler::prefetch
// buffer is full, let's stop here and store the header we've read in our static storage
m_curHeader = *header;
// check that the buffer size is not too small for a single event !
+ // if it is, double it (as long as we stay < 4G)
if ( header->recordSize() > bufferSize ) {
- throw std::runtime_error{fmt::format( "Found too big event ({} bytes), not fitting into input buffers. "
- "Please consider changing ioalg_buffer_nb_events to at least {}",
- header->recordSize(), ( ( header->recordSize() + 49999 ) / 50000 ) )};
+ while ( ( header->recordSize() > m_minimumBufferSize ) && ( m_minimumBufferSize < ( 1u << 31 ) ) ) {
+ m_minimumBufferSize *= 2;
+ owner.info() << "Increasing IOHandlerFileRead's buffer's size to " << m_minimumBufferSize << " bytes"
+ << endmsg;
+ }
+ if ( m_minimumBufferSize > ( 1u << 31 ) ) {
+ throw std::runtime_error{
+ fmt::format( "Found surprisingly big event ({} bytes), we cannot allocate such a big buffer. "
+ "Giving up.",
+ header->recordSize() )};
+ }
}
break;
}
@@ -187,6 +205,7 @@ std::shared_ptr<LHCb::MDF::ByteBuffer> LHCb::MDF::FileReadInputHandler::prefetch
// now let's build the event and copy the raw banks
events.emplace_back( header, LHCb::span<std::byte>{(std::byte*)header->data(), header->size()},
header->compression() & 0xF );
+ m_eventSize += ( header->recordSize() / 1024 );
curBufPtr += header->recordSize();
}
} catch ( IO::EndOfInput& e ) {
diff --git a/Ex/IOExample/tests/options/LargeBankReadingExample.py b/Ex/IOExample/tests/options/LargeBankReadingExample.py
new file mode 100644
index 0000000000000000000000000000000000000000..b6cecb86cfac6b2d17a2eca1cc29f87159fe1054
--- /dev/null
+++ b/Ex/IOExample/tests/options/LargeBankReadingExample.py
@@ -0,0 +1,28 @@
+###############################################################################
+# (c) Copyright 2000-2018 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+from PyConf.application import ApplicationOptions, configure_input, configure, create_or_reuse_mdfIOAlg, make_odin
+from PyConf.control_flow import CompositeNode
+from PyConf.Algorithms import PrintHeader
+from Gaudi.Configuration import DEBUG
+
+options = ApplicationOptions(_enabled=False)
+options.set_input_and_conds_from_testfiledb("mdf_file_with_large_bank")
+options.print_freq = 1
+options.evt_max = 20 # evt 18 is large (3MB)
+options.input_type = 'MDF'
+config = configure_input(options)
+
+config.update(
+ configure(
+ options,
+ CompositeNode(
+ "Top",
+ [PrintHeader(name="PrintHeader", ODINLocation=make_odin())])))
diff --git a/Ex/IOExample/tests/qmtest/ioexample.qms/mdflargebankexample.qmt b/Ex/IOExample/tests/qmtest/ioexample.qms/mdflargebankexample.qmt
new file mode 100644
index 0000000000000000000000000000000000000000..06b039632eec445c8fca0f57a17800183072ecb7
--- /dev/null
+++ b/Ex/IOExample/tests/qmtest/ioexample.qms/mdflargebankexample.qmt
@@ -0,0 +1,24 @@
+<?xml version="1.0" ?><!DOCTYPE extension PUBLIC '-//QM/2.3/Extension//EN' 'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
+<!--
+ (c) Copyright 2000-2019 CERN for the benefit of the LHCb Collaboration
+
+ This software is distributed under the terms of the GNU General Public
+ Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
+
+ In applying this licence, CERN does not waive the privileges and immunities
+ granted to it by virtue of its status as an Intergovernmental Organization
+ or submit itself to any jurisdiction.
+-->
+<extension class="GaudiTest.GaudiExeTest" kind="test">
+ <argument name="program"><text>gaudirun.py</text></argument>
+ <argument name="args"><set>
+ <text>../options/LargeBankReadingExample.py</text>
+ </set></argument>
+ <argument name="reference"><text>../refs/LargebankReadingExample.ref</text></argument>
+ <argument name="validator"><text>
+from GaudiConf.QMTest.LHCbExclusions import preprocessor as LHCbPreprocessor
+from IOExample.QMTest.DetDescExclusions import detDescFixes
+preprocessor = LHCbPreprocessor + detDescFixes
+validateWithReference(preproc = preprocessor)
+ </text></argument>
+</extension>
diff --git a/Ex/IOExample/tests/refs/LargebankReadingExample.ref b/Ex/IOExample/tests/refs/LargebankReadingExample.ref
new file mode 100644
index 0000000000000000000000000000000000000000..bd401f2416c79400ef53caed4457f43001311724
--- /dev/null
+++ b/Ex/IOExample/tests/refs/LargebankReadingExample.ref
@@ -0,0 +1,105 @@
+/***** User ApplicationOptions/ApplicationOptions **************************************************
+|-append_decoding_keys_to_output_manifest = True (default: True)
+|-auditors = [] (default: [])
+|-conddb_tag = 'sim-20210617-vc-mu100' (default: '')
+|-conditions_version = 'AlignmentV14_2024_04_05' (default: '')
+|-control_flow_file = '' (default: '')
+|-data_flow_file = '' (default: '')
+|-data_type = 'Upgrade' (default: 'Upgrade')
+|-dddb_tag = 'dddb-20210617' (default: '')
+|-event_store = 'HiveWhiteBoard' (default: 'HiveWhiteBoard')
+|-evt_max = 20 (default: -1)
+|-first_evt = 0 (default: 0)
+|-force_odin = False (default: False)
+|-gaudipython_mode = False (default: False)
+|-geometry_backend = 'DD4Hep' (default: 'DD4Hep')
+|-geometry_version = 'run3/trunk' (default: '')
+|-histo_file = '' (default: '')
+|-input_files = ['mdf:root://eoslhcb.cern.ch//eos/lhcb//lhcb/swtest/MDFLargeBank/z0_with_ut.qee.mdf']
+| (default: [])
+|-input_manifest_file = '' (default: '')
+|-input_process = '' (default: '')
+|-input_raw_format = 0.5 (default: 0.5)
+|-input_stream = '' (default: '')
+|-input_type = 'MDF' (default: '')
+|-ioalg_buffer_nb_events = 40 (default: 40)
+|-lines_maker = None
+|-mdf_ioalg_name = 'IOAlgFileRead' (default: 'IOAlgFileRead')
+|-memory_pool_size = 10485760 (default: 10485760)
+|-monitoring_file = '' (default: '')
+|-msg_svc_format = '% F%35W%S %7W%R%T %0W%M' (default: '% F%35W%S %7W%R%T %0W%M')
+|-msg_svc_time_format = '%Y-%m-%d %H:%M:%S UTC' (default: '%Y-%m-%d %H:%M:%S UTC')
+|-n_event_slots = 1 (default: -1)
+|-n_threads = 1 (default: 1)
+|-ntuple_basketsize = 32000 (default: 32000)
+|-ntuple_file = '' (default: '')
+|-output_file = '' (default: '')
+|-output_level = 3 (default: 3)
+|-output_manifest_file = '' (default: '')
+|-output_type = '' (default: '')
+|-persistreco_version = 1.0 (default: 1.0)
+|-phoenix_filename = '' (default: '')
+|-preamble_algs = [] (default: [])
+|-print_freq = 1 (default: 10000)
+|-python_logging_level = 20 (default: 20)
+|-require_specific_decoding_keys = [] (default: [])
+|-root_ioalg_name = 'RootIOAlg' (default: 'RootIOAlg')
+|-root_ioalg_opts = {} (default: {})
+|-scheduler_legacy_mode = True (default: True)
+|-simulation = False (default: None)
+|-write_decoding_keys_to_git = True (default: True)
+|-write_options_to_fsr = False (default: False)
+|-xml_file_catalog = '' (default: '')
+|-xml_summary_file = '' (default: '')
+|-xml_summary_svc = 'XMLSummarySvc' (default: 'XMLSummarySvc')
+\----- (End of User ApplicationOptions/ApplicationOptions) -----------------------------------------
+ApplicationMgr SUCCESS
+====================================================================================================================================
+====================================================================================================================================
+ApplicationMgr INFO Application Manager Configured successfully
+HLTControlFlowMgr INFO Start initialization
+HLTControlFlowMgr INFO Will not use an EventSelector.
+HLTControlFlowMgr INFO Concurrency level information:
+HLTControlFlowMgr INFO o Number of events slots: 1
+HLTControlFlowMgr INFO o TBB thread pool size: 'ThreadPoolSize':1
+ApplicationMgr INFO Application Manager Initialized successfully
+ApplicationMgr INFO Application Manager Started successfully
+HLTControlFlowMgr INFO Will measure time between events 2 and 18 (stop might be some events later)
+HLTControlFlowMgr INFO Starting loop on events
+PrintHeader INFO # 1 Run 303682, Event 16370672535
+PrintHeader INFO # 2 Run 303682, Event 73647046137
+PrintHeader INFO # 3 Run 303682, Event 41481917913
+PrintHeader INFO # 4 Run 303682, Event 71402655280
+PrintHeader INFO # 5 Run 303682, Event 51194414331
+PrintHeader INFO # 6 Run 303682, Event 6751241109
+PrintHeader INFO # 7 Run 303682, Event 51371356968
+PrintHeader INFO # 8 Run 303682, Event 79445470125
+PrintHeader INFO # 9 Run 303682, Event 15755822160
+PrintHeader INFO # 10 Run 303682, Event 82872822028
+PrintHeader INFO # 11 Run 303682, Event 69490791450
+PrintHeader INFO # 12 Run 303682, Event 43919997735
+PrintHeader INFO # 13 Run 303682, Event 36545757671
+PrintHeader INFO # 14 Run 303682, Event 63942829458
+PrintHeader INFO # 15 Run 303682, Event 43062498712
+PrintHeader INFO # 16 Run 303682, Event 30579318698
+PrintHeader INFO # 17 Run 303682, Event 24529217083
+PrintHeader INFO # 18 Run 303682, Event 34404731612
+PrintHeader INFO # 19 Run 303682, Event 13215944121
+PrintHeader INFO # 20 Run 303682, Event 74425499674
+ApplicationMgr INFO Application Manager Stopped successfully
+HLTControlFlowMgr INFO
+HLTControlFlowMgr INFO StateTree: CFNode #executed #passed
+LAZY_AND: Top #=20 Sum=20 Eff=|( 100.0000 +- 0.00000 )%|
+ PrintHeader/PrintHeader #=20 Sum=20 Eff=|( 100.0000 +- 0.00000 )%|
+HLTControlFlowMgr INFO Histograms converted successfully according to request.
+ApplicationMgr INFO Application Manager Finalized successfully
+ApplicationMgr INFO Application Manager Terminated successfully
+HLTControlFlowMgr INFO Number of counters : 1
+ | Counter | # | sum | mean/eff^* | rms/err^* | min | max |
+ | "Processed events" | 20 |
+MDFIOAlg INFO Number of counters : 1
+ | Counter | # | sum | mean/eff^* | rms/err^* | min | max |
+ | "#banks in raw event" | 20 | 17057 | 852.85 | 10.011 | 848.00 | 896.00 |
+PrintHeader INFO Number of counters : 1
+ | Counter | # | sum | mean/eff^* | rms/err^* | min | max |
+ | "EventCount" | 20 |