ModuleManager.cpp 32 KB
Newer Older
1
2
3
4
5
6
7
8
/** @file
 *  @brief Interface to the core framework
 *  @copyright Copyright (c) 2017 CERN and the Corryvreckan authors.
 * This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
 * In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
 * Intergovernmental Organization or submit itself to any jurisdiction.
 */

9
// ROOT include files
10
11
12
13
#include <Math/DisplacementVector2D.h>
#include <Math/Vector2D.h>
#include <Math/Vector3D.h>
#include <TFile.h>
14
#include <TSystem.h>
15
16

// Local include files
17
18
19
#include "ModuleManager.hpp"
#include "core/utils/log.h"
#include "exceptions.h"
20

21
#include <chrono>
22
23
#include <dlfcn.h>
#include <fstream>
24
#include <iomanip>
25

26
27
#define CORRYVRECKAN_MODULE_PREFIX "libCorryvreckanModule"
#define CORRYVRECKAN_GENERATOR_FUNCTION "corryvreckan_module_generator"
28
29
#define CORRYVRECKAN_GLOBAL_FUNCTION "corryvreckan_module_is_global"
#define CORRYVRECKAN_DUT_FUNCTION "corryvreckan_module_is_dut"
30
#define CORRYVRECKAN_TYPE_FUNCTION "corryvreckan_detector_types"
31

32
33
using namespace corryvreckan;

34
// Default constructor
35
ModuleManager::ModuleManager() : m_reference(nullptr), m_terminate(false) {
Simon Spannagel's avatar
Simon Spannagel committed
36
    // New clipboard for storage:
37
    m_clipboard = std::make_shared<Clipboard>();
38
39
}

40
41
void ModuleManager::load(ConfigManager* conf_mgr) {
    conf_manager_ = conf_mgr;
42

43
    load_detectors();
44
    load_modules();
45
46
}

47
void ModuleManager::load_detectors() {
48
    Configuration& global_config = conf_manager_->getGlobalConfiguration();
49

50
    for(auto& detector_section : conf_manager_->getDetectorConfigurations()) {
51

52
        std::string name = detector_section.getName();
53

54
55
56
57
58
59
60
        // Check if we have a duplicate:
        if(std::find_if(m_detectors.begin(), m_detectors.end(), [&name](std::shared_ptr<Detector> obj) {
               return obj->name() == name;
           }) != m_detectors.end()) {
            throw InvalidValueError(
                global_config, "detectors_file", "Detector " + detector_section.getName() + " defined twice");
        }
61

62
63
        LOG_PROGRESS(STATUS, "DET_LOAD_LOOP") << "Loading detector " << name;
        auto detector = std::make_shared<Detector>(detector_section);
64

65
66
67
68
        // Check if we already found a reference plane:
        if(m_reference != nullptr && detector->isReference()) {
            throw InvalidValueError(global_config, "detectors_file", "Found more than one reference detector");
        }
69

70
71
72
        // Switch flag if we found the reference plane:
        if(detector->isReference()) {
            m_reference = detector;
73
        }
74
75
76

        // Add the new detector to the global list:
        m_detectors.push_back(detector);
77
    }
78

79
    // Check that exactly one detector is marked as reference:
Simon Spannagel's avatar
Simon Spannagel committed
80
    if(m_reference == nullptr) {
81
82
83
        throw InvalidValueError(global_config, "detectors_file", "Found no detector marked as reference");
    }

84
    LOG_PROGRESS(STATUS, "DET_LOAD_LOOP") << "Loaded " << m_detectors.size() << " detectors";
85
86

    // Finally, sort the list of detectors by z position (from lowest to highest)
87
    std::sort(m_detectors.begin(), m_detectors.end(), [](std::shared_ptr<Detector> det1, std::shared_ptr<Detector> det2) {
88
        return det1->displacement().Z() < det2->displacement().Z();
Simon Spannagel's avatar
Simon Spannagel committed
89
    });
90
}
91

92
void ModuleManager::load_modules() {
93
94
    auto& configs = conf_manager_->getModuleConfigurations();
    Configuration& global_config = conf_manager_->getGlobalConfiguration();
Simon Spannagel's avatar
Simon Spannagel committed
95
96

    // Create histogram output file
97
98
99
    global_config.setAlias("histogram_file", "histogramFile");
    std::string histogramFile = global_config.getPath("histogram_file");

100
    m_histogramFile = std::make_unique<TFile>(histogramFile.c_str(), "RECREATE");
Simon Spannagel's avatar
Simon Spannagel committed
101
    if(m_histogramFile->IsZombie()) {
102
        throw RuntimeError("Cannot create main ROOT file " + histogramFile);
Simon Spannagel's avatar
Simon Spannagel committed
103
    }
104
    m_histogramFile->cd();
Simon Spannagel's avatar
Simon Spannagel committed
105

106
    LOG(DEBUG) << "Start loading modules, have " << configs.size() << " configurations.";
Simon Spannagel's avatar
Simon Spannagel committed
107
108
    // Loop through all non-global configurations
    for(auto& config : configs) {
109
        // Load library for each module. Libraries are named (by convention + CMAKE libCorryvreckanModule Name.suffix
Simon Spannagel's avatar
Simon Spannagel committed
110
        std::string lib_name =
111
            std::string(CORRYVRECKAN_MODULE_PREFIX).append(config.getName()).append(SHARED_LIBRARY_SUFFIX);
112
        LOG_PROGRESS(STATUS, "MOD_LOAD_LOOP") << "Loading module " << config.getName();
Simon Spannagel's avatar
Simon Spannagel committed
113
114
115
116
117
118
119

        void* lib = nullptr;
        bool load_error = false;
        dlerror();
        if(loaded_libraries_.count(lib_name) == 0) {
            // If library is not loaded then try to load it first from the config
            // directories
Simon Spannagel's avatar
Simon Spannagel committed
120
121
            if(global_config.has("library_directories")) {
                std::vector<std::string> lib_paths = global_config.getPathArray("library_directories", true);
Simon Spannagel's avatar
Simon Spannagel committed
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
                for(auto& lib_path : lib_paths) {
                    std::string full_lib_path = lib_path;
                    full_lib_path += "/";
                    full_lib_path += lib_name;

                    // Check if the absolute file exists and try to load if it exists
                    std::ifstream check_file(full_lib_path);
                    if(check_file.good()) {
                        lib = dlopen(full_lib_path.c_str(), RTLD_NOW);
                        if(lib != nullptr) {
                            LOG(DEBUG) << "Found library in configuration specified directory at " << full_lib_path;
                        } else {
                            load_error = true;
                        }
                        break;
                    }
                }
            }

            // Otherwise try to load from the standard paths if not found already
            if(!load_error && lib == nullptr) {
                lib = dlopen(lib_name.c_str(), RTLD_NOW);

                if(lib != nullptr) {
                    LOG(TRACE) << "Opened library";
                    Dl_info dl_info;
                    dl_info.dli_fname = "";

                    // workaround to get the location of the library
                    int ret = dladdr(dlsym(lib, CORRYVRECKAN_GENERATOR_FUNCTION), &dl_info);
                    if(ret != 0) {
                        LOG(DEBUG) << "Found library during global search in runtime paths at " << dl_info.dli_fname;
                    } else {
                        LOG(WARNING) << "Found library during global search but could not "
                                        "deduce location, likely broken library";
                    }
                } else {
                    load_error = true;
                }
161
162
            }
        } else {
Simon Spannagel's avatar
Simon Spannagel committed
163
164
            // Otherwise just fetch it from the cache
            lib = loaded_libraries_[lib_name];
165
166
        }

Simon Spannagel's avatar
Simon Spannagel committed
167
168
169
        // If library did not load then throw exception
        if(load_error) {
            const char* lib_error = dlerror();
170

Simon Spannagel's avatar
Simon Spannagel committed
171
172
173
174
175
176
177
            // Find the name of the loaded library if it exists
            std::string lib_error_str = lib_error;
            size_t end_pos = lib_error_str.find(':');
            std::string problem_lib;
            if(end_pos != std::string::npos) {
                problem_lib = lib_error_str.substr(0, end_pos);
            }
178

Simon Spannagel's avatar
Simon Spannagel committed
179
180
181
182
183
184
185
186
187
188
            // FIXME is checking the error in this way portable?
            if(lib_error != nullptr && std::strstr(lib_error, "cannot allocate memory in static TLS block") != nullptr) {
                LOG(ERROR) << "Library could not be loaded: not enough thread local storage "
                              "available"
                           << std::endl
                           << "Try one of below workarounds:" << std::endl
                           << "- Rerun library with the environmental variable LD_PRELOAD='" << problem_lib << "'"
                           << std::endl
                           << "- Recompile the library " << problem_lib << " with tls-model=global-dynamic";
            } else if(lib_error != nullptr && std::strstr(lib_error, "cannot open shared object file") != nullptr &&
189
                      problem_lib.find(CORRYVRECKAN_MODULE_PREFIX) == std::string::npos) {
Simon Spannagel's avatar
Simon Spannagel committed
190
191
192
193
194
195
196
197
198
199
200
201
                LOG(ERROR) << "Library could not be loaded: one of its dependencies is missing" << std::endl
                           << "The name of the missing library is " << problem_lib << std::endl
                           << "Please make sure the library is properly initialized and try "
                              "again";
            } else {
                LOG(ERROR) << "Library could not be loaded: it is not available" << std::endl
                           << " - Did you enable the library during building? " << std::endl
                           << " - Did you spell the library name correctly (case-sensitive)? ";
                if(lib_error != nullptr) {
                    LOG(DEBUG) << "Detailed error: " << lib_error;
                }
            }
202

203
            throw corryvreckan::DynamicLibraryError("Error loading " + config.getName());
Simon Spannagel's avatar
Simon Spannagel committed
204
205
206
        }
        // Remember that this library was loaded
        loaded_libraries_[lib_name] = lib;
207

208
        // Check if this module is produced once, or once per detector
209
210
        void* globalFunction = dlsym(loaded_libraries_[lib_name], CORRYVRECKAN_GLOBAL_FUNCTION);
        void* dutFunction = dlsym(loaded_libraries_[lib_name], CORRYVRECKAN_DUT_FUNCTION);
211
        void* typeFunction = dlsym(loaded_libraries_[lib_name], CORRYVRECKAN_TYPE_FUNCTION);
212

213
        // If the global function was not found, throw an error
214
        if(globalFunction == nullptr || dutFunction == nullptr || typeFunction == nullptr) {
215
216
            LOG(ERROR) << "Module library is invalid or outdated: required interface function not found!";
            throw corryvreckan::DynamicLibraryError(config.getName());
217
218
        }

219
220
221
222
223
        bool global = reinterpret_cast<bool (*)()>(globalFunction)();      // NOLINT
        bool dut_only = reinterpret_cast<bool (*)()>(dutFunction)();       // NOLINT
        char* type_tokens = reinterpret_cast<char* (*)()>(typeFunction)(); // NOLINT

        std::vector<std::string> types = get_type_vector(type_tokens);
224

Simon Spannagel's avatar
Simon Spannagel committed
225
226
227
        // Add the global internal parameters to the configuration
        std::string global_dir = gSystem->pwd();
        config.set<std::string>("_global_dir", global_dir);
228

229
        // Merge the global configuration into the modules config:
230
231
        config.merge(global_config);

232
        // Create the modules from the library
233
        std::vector<std::pair<ModuleIdentifier, Module*>> mod_list;
234
        if(global) {
235
            mod_list.emplace_back(create_unique_module(loaded_libraries_[lib_name], config));
236
        } else {
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
            mod_list = create_detector_modules(loaded_libraries_[lib_name], config, dut_only, types);
        }

        // Loop through all created instantiations
        for(auto& id_mod : mod_list) {
            // FIXME: This convert the module to an unique pointer. Check that this always works and we can do this earlier
            std::unique_ptr<Module> mod(id_mod.second);
            ModuleIdentifier identifier = id_mod.first;

            // Check if the unique instantiation already exists
            auto iter = id_to_module_.find(identifier);
            if(iter != id_to_module_.end()) {
                // Unique name exists, check if its needs to be replaced
                if(identifier.getPriority() < iter->first.getPriority()) {
                    // Priority of new instance is higher, replace the instance
                    LOG(TRACE) << "Replacing model instance " << iter->first.getUniqueName()
                               << " with instance with higher priority.";

                    module_execution_time_.erase(iter->second->get());
                    iter->second = m_modules.erase(iter->second);
                    iter = id_to_module_.erase(iter);
                } else {
                    // Priority is equal, raise an error
                    if(identifier.getPriority() == iter->first.getPriority()) {
                        throw AmbiguousInstantiationError(config.getName());
                    }
                    // Priority is lower, do not add this module to the run list
                    continue;
                }
266
            }
267
268
269
270
271
272
273
274

            // Save the identifier in the module
            mod->set_identifier(identifier);
            mod->setReference(m_reference);

            // Add the new module to the run list
            m_modules.emplace_back(std::move(mod));
            id_to_module_[identifier] = --m_modules.end();
275
        }
Simon Spannagel's avatar
Simon Spannagel committed
276
    }
277

278
    LOG_PROGRESS(STATUS, "MOD_LOAD_LOOP") << "Loaded " << m_modules.size() << " module instances";
279
280
}

281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
std::vector<std::string> ModuleManager::get_type_vector(char* type_tokens) {
    std::vector<std::string> types;

    std::stringstream tokenstream(type_tokens);
    while(tokenstream.good()) {
        std::string token;
        getline(tokenstream, token, ',');
        if(token.empty()) {
            continue;
        }
        std::transform(token.begin(), token.end(), token.begin(), ::tolower);
        types.push_back(token);
    }
    return types;
}

297
298
299
300
301
302
303
std::shared_ptr<Detector> ModuleManager::get_detector(std::string name) {
    auto it = find_if(
        m_detectors.begin(), m_detectors.end(), [&name](std::shared_ptr<Detector> obj) { return obj->name() == name; });
    return (it != m_detectors.end() ? (*it) : nullptr);
}

std::pair<ModuleIdentifier, Module*> ModuleManager::create_unique_module(void* library, Configuration config) {
Simon Spannagel's avatar
Simon Spannagel committed
304
305
306
    // Create the identifier
    ModuleIdentifier identifier(config.getName(), "", 0);

307
308
309
310
311
312
313
314
    // Return error if user tried to specialize the unique module:
    if(config.has("name")) {
        throw InvalidValueError(config, "name", "unique modules cannot be specialized using the \"name\" keyword.");
    }
    if(config.has("type")) {
        throw InvalidValueError(config, "type", "unique modules cannot be specialized using the \"type\" keyword.");
    }

Simon Spannagel's avatar
Simon Spannagel committed
315
316
    LOG(TRACE) << "Creating module " << identifier.getUniqueName() << ", using generator \""
               << CORRYVRECKAN_GENERATOR_FUNCTION << "\"";
Simon Spannagel's avatar
Simon Spannagel committed
317
318
319
320
321

    // Get the generator function for this module
    void* generator = dlsym(library, CORRYVRECKAN_GENERATOR_FUNCTION);
    // If the generator function was not found, throw an error
    if(generator == nullptr) {
322
        LOG(ERROR) << "Module library is invalid or outdated: required "
Simon Spannagel's avatar
Simon Spannagel committed
323
                      "interface function not found!";
324
        throw corryvreckan::RuntimeError("Error instantiating module from " + config.getName());
Simon Spannagel's avatar
Simon Spannagel committed
325
    }
326

327
328
329
    // Create and add module instance config
    Configuration& instance_config = conf_manager_->addInstanceConfiguration(identifier, config);

Simon Spannagel's avatar
Simon Spannagel committed
330
    // Convert to correct generator function
331
332
    auto module_generator =
        reinterpret_cast<Module* (*)(Configuration, std::vector<std::shared_ptr<Detector>>)>(generator); // NOLINT
333

334
335
336
    // Set the log section header
    std::string old_section_name = Log::getSection();
    std::string section_name = "C:";
337
    section_name += identifier.getUniqueName();
338
339
    Log::setSection(section_name);
    // Set module specific log settings
340
    auto old_settings = set_module_before(identifier.getUniqueName(), instance_config);
341
    // Build module
342
    Module* module = module_generator(instance_config, m_detectors);
343
344
    // Reset log
    Log::setSection(old_section_name);
345
    set_module_after(old_settings);
346

347
    // Return the module to the analysis
Simon Spannagel's avatar
Simon Spannagel committed
348
    return std::make_pair(identifier, module);
349
350
}

Simon Spannagel's avatar
Simon Spannagel committed
351
std::vector<std::pair<ModuleIdentifier, Module*>>
352
ModuleManager::create_detector_modules(void* library, Configuration config, bool dut_only, std::vector<std::string> types) {
353
354
355
356
357
358
359
360
361
362
363
364
365
    LOG(TRACE) << "Creating instantiations for module " << config.getName() << ", using generator \""
               << CORRYVRECKAN_GENERATOR_FUNCTION << "\"";

    // Get the generator function for this module
    void* generator = dlsym(library, CORRYVRECKAN_GENERATOR_FUNCTION);
    // If the generator function was not found, throw an error
    if(generator == nullptr) {
        LOG(ERROR) << "Module library is invalid or outdated: required "
                      "interface function not found!";
        throw corryvreckan::RuntimeError("Error instantiating module from " + config.getName());
    }

    // Convert to correct generator function
366
    auto module_generator = reinterpret_cast<Module* (*)(Configuration, std::shared_ptr<Detector>)>(generator); // NOLINT
367
    auto module_base_name = config.getName();
368
369

    // Figure out which detectors should run on this module:
370
371
372
373
374
375
376
377
378
379
380
    bool instances_created = false;
    std::vector<std::pair<std::shared_ptr<Detector>, ModuleIdentifier>> instantiations;

    // Create all names first with highest priority
    std::set<std::string> module_names;
    if(config.has("name")) {
        std::vector<std::string> names = config.getArray<std::string>("name");
        for(auto& name : names) {
            auto det = get_detector(name);
            if(det == nullptr) {
                continue;
381
            }
382
383
384
385
386

            LOG(TRACE) << "Preparing \"name\" instance for " << det->name();
            instantiations.emplace_back(det, ModuleIdentifier(module_base_name, det->name(), 0));
            // Save the name (to not instantiate it again later)
            module_names.insert(name);
387
        }
388
        instances_created = !names.empty();
389
390
    }

391
392
393
394
395
396
397
    // Then create all types that are not yet name instantiated
    if(config.has("type")) {
        // Prepare type list from configuration:
        std::vector<std::string> ctypes = config.getArray<std::string>("type");
        for(auto& type : ctypes) {
            std::transform(type.begin(), type.end(), type.begin(), ::tolower);
        }
398

399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
        // Check that this is possible at all:
        std::vector<std::string> intersection;
        std::set_intersection(ctypes.begin(), ctypes.end(), types.begin(), types.end(), std::back_inserter(intersection));
        if(!types.empty() && intersection.empty()) {
            throw InvalidInstantiationError(module_base_name, "type conflict");
        }

        for(auto& det : m_detectors) {
            auto detectortype = det->type();
            std::transform(detectortype.begin(), detectortype.end(), detectortype.begin(), ::tolower);

            // Skip all that were already added by name
            if(module_names.find(det->name()) != module_names.end()) {
                continue;
            }
            for(auto& type : ctypes) {
                // Skip all with wrong type
                if(detectortype != type) {
                    continue;
                }

                LOG(TRACE) << "Preparing \"type\" instance for " << det->name();
                instantiations.emplace_back(det, ModuleIdentifier(module_base_name, det->name(), 1));
422
423
            }
        }
424
425
426
427
428
429
430
431
432
        instances_created = !ctypes.empty();
    }

    // Create for all detectors if no name / type provided
    if(!instances_created) {
        for(auto& det : m_detectors) {
            LOG(TRACE) << "Preparing \"other\" instance for " << det->name();
            instantiations.emplace_back(det, ModuleIdentifier(module_base_name, det->name(), 2));
        }
433
434
    }

Simon Spannagel's avatar
Simon Spannagel committed
435
    std::vector<std::pair<ModuleIdentifier, Module*>> modules;
436
437
438
439
    for(const auto& instance : instantiations) {
        auto detector = instance.first;
        auto identifier = instance.second;

440
441
        // If this should only be instantiated for DUTs, skip otherwise:
        if(dut_only && !detector->isDUT()) {
Simon Spannagel's avatar
Simon Spannagel committed
442
            LOG(TRACE) << "Skipping instantiation \"" << identifier.getUniqueName() << "\", detector is no DUT";
443
444
            continue;
        }
445
446
447
448
449
450
451
452

        // Do not instantiate module if detector type is not mentioned as supported:
        auto detectortype = detector->type();
        std::transform(detectortype.begin(), detectortype.end(), detectortype.begin(), ::tolower);
        if(!types.empty() && std::find(types.begin(), types.end(), detectortype) == types.end()) {
            LOG(TRACE) << "Skipping instantiation \"" << identifier.getUniqueName() << "\", detector type mismatch";
            continue;
        }
453
        LOG(DEBUG) << "Creating instantiation \"" << identifier.getUniqueName() << "\"";
454

455
456
457
        // Create and add module instance config
        Configuration& instance_config = conf_manager_->addInstanceConfiguration(instance.second, config);

458
459
460
        // Set the log section header
        std::string old_section_name = Log::getSection();
        std::string section_name = "C:";
Simon Spannagel's avatar
Simon Spannagel committed
461
        section_name += identifier.getUniqueName();
462
463
        Log::setSection(section_name);
        // Set module specific log settings
464
        auto old_settings = set_module_before(identifier.getUniqueName(), instance_config);
465
        // Build module
466
        modules.emplace_back(identifier, module_generator(instance_config, detector));
467
468
469
470
471
472
473
474
475
        // Reset log
        Log::setSection(old_section_name);
        set_module_after(old_settings);
    }

    // Return the list of modules to the analysis
    return modules;
}

476
// Run the analysis loop - this initialises, runs and finalises all modules
477
void ModuleManager::run() {
478
    Configuration& global_config = conf_manager_->getGlobalConfiguration();
Simon Spannagel's avatar
Simon Spannagel committed
479

480
    // Check if we have an event or track limit:
481
482
    auto number_of_events = global_config.get<int>("number_of_events", -1);
    auto number_of_tracks = global_config.get<int>("number_of_tracks", -1);
483
    auto run_time = global_config.get<double>("run_time", static_cast<double>(Units::convert(-1.0, "s")));
484

485
    // Loop over all events, running each module on each "event"
Simon Spannagel's avatar
Simon Spannagel committed
486
    LOG(STATUS) << "========================| Event loop |========================";
487
    m_events = 0;
488
489
    m_tracks = 0;

Simon Spannagel's avatar
Simon Spannagel committed
490
491
    while(1) {
        bool run = true;
492

Simon Spannagel's avatar
Simon Spannagel committed
493
        // Check if we have reached the maximum number of events
494

Simon Spannagel's avatar
Simon Spannagel committed
495
496
497
        if(number_of_events > -1 && m_events >= number_of_events)
            break;

498
        if(run_time > 0.0 && (m_clipboard->get_persistent("eventStart")) >= run_time)
499
500
            break;

501
502
503
504
        // Check if we have reached the maximum number of tracks
        if(number_of_tracks > -1 && m_tracks >= number_of_tracks)
            break;

505
506
        // Run all modules
        for(auto& module : m_modules) {
507
508
509
            // Get current time
            auto start = std::chrono::steady_clock::now();

510
511
512
            // Set run module section header
            std::string old_section_name = Log::getSection();
            std::string section_name = "R:";
Simon Spannagel's avatar
Simon Spannagel committed
513
            section_name += module->getUniqueName();
514
515
            Log::setSection(section_name);
            // Set module specific settings
Simon Spannagel's avatar
Simon Spannagel committed
516
            auto old_settings = set_module_before(module->getUniqueName(), module->getConfig());
Simon Spannagel's avatar
Simon Spannagel committed
517
            // Change to the output file directory
518
519
            module->getROOTDirectory()->cd();

520
            StatusCode check = module->run(m_clipboard);
521

522
523
            // Reset logging
            Log::setSection(old_section_name);
524
            set_module_after(old_settings);
525
526
527

            // Update execution time
            auto end = std::chrono::steady_clock::now();
528
            module_execution_time_[module.get()] += static_cast<std::chrono::duration<long double>>(end - start).count();
529

530
531
            if(check == StatusCode::DeadTime) {
                // If status code indicates dead time, just silently continue with next event:
532
                break;
533
534
535
536
537
538
            } else if(check == StatusCode::Failure) {
                // If the status code indicates failure, break immediately and finish:
                run = false;
                break;
            } else if(check == StatusCode::EndRun) {
                // If the returned status code asks for end-of-run, finish module list and finish:
Simon Spannagel's avatar
Simon Spannagel committed
539
                run = false;
540
            }
Simon Spannagel's avatar
Simon Spannagel committed
541
542
        }

543
        // Print statistics:
544
        Tracks* tracks = reinterpret_cast<Tracks*>(m_clipboard->get("tracks"));
545
        m_tracks += (tracks == nullptr ? 0 : static_cast<int>(tracks->size()));
546

547
        if(m_events % 100 == 0) {
548
549
550
551
552
553
554
555
556

            auto kilo_or_mega = [](const double& input) {
                bool mega = (input > 1e6 ? true : false);
                auto value = (mega ? input * 1e-6 : input * 1e-3);
                std::stringstream output;
                output << std::fixed << std::setprecision(mega ? 2 : 1) << value << (mega ? "M" : "k");
                return output.str();
            };

557
            LOG_PROGRESS(STATUS, "event_loop")
558
                << "Ev: " << kilo_or_mega(m_events) << " Tr: " << kilo_or_mega(m_tracks) << " (" << std::setprecision(3)
559
                << (static_cast<double>(m_tracks) / m_events) << "/ev)"
560
561
562
                << (m_clipboard->has_persistent("eventStart")
                        ? " t = " + Units::display(m_clipboard->get_persistent("eventStart"), {"ns", "us", "ms", "s"})
                        : "");
563
564
        }

Simon Spannagel's avatar
Simon Spannagel committed
565
566
        // Clear objects from this iteration from the clipboard
        m_clipboard->clear();
567

568
        // Check if any of the modules return a value saying it should stop
569
        if(!run) {
Simon Spannagel's avatar
Simon Spannagel committed
570
            break;
571
572
        }

Simon Spannagel's avatar
Simon Spannagel committed
573
        // Increment event number
574
        m_events++;
575
576
577

        // Check for user termination and stop the event loop:
        if(m_terminate) {
578
            break;
579
        }
580
581
582
    }
}

583
void ModuleManager::terminate() {
584
585
586
    m_terminate = true;
}

587
// Initalise all modules
588
void ModuleManager::initialiseAll() {
589
590
591
    // Loop over all modules and initialise them
    LOG(STATUS) << "=================| Initialising modules |==================";
    for(auto& module : m_modules) {
592
593
594
        // Pass the config manager to this instance
        module->set_config_manager(conf_manager_);

595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
        // Create main ROOT directory for this module class if it does not exists yet
        LOG(TRACE) << "Creating and accessing ROOT directory";
        std::string module_name = module->getConfig().getName();
        auto directory = m_histogramFile->GetDirectory(module_name.c_str());
        if(directory == nullptr) {
            directory = m_histogramFile->mkdir(module_name.c_str());
            if(directory == nullptr) {
                throw RuntimeError("Cannot create or access overall ROOT directory for module " + module_name);
            }
        }
        directory->cd();

        // Create local directory for this instance
        TDirectory* local_directory = nullptr;
        if(module->get_identifier().getIdentifier().empty()) {
            local_directory = directory;
        } else {
            local_directory = directory->mkdir(module->get_identifier().getIdentifier().c_str());
            if(local_directory == nullptr) {
                throw RuntimeError("Cannot create or access local ROOT directory for module " + module->getUniqueName());
            }
        }

        // Change to the directory and save it in the module
        local_directory->cd();
        module->set_ROOT_directory(local_directory);

622
623
624
        // Set init module section header
        std::string old_section_name = Log::getSection();
        std::string section_name = "I:";
Simon Spannagel's avatar
Simon Spannagel committed
625
        section_name += module->getUniqueName();
626
627
        Log::setSection(section_name);
        // Set module specific settings
Simon Spannagel's avatar
Simon Spannagel committed
628
        auto old_settings = set_module_before(module->getUniqueName(), module->getConfig());
629
630
        // Change to our ROOT directory
        module->getROOTDirectory()->cd();
631

632
        LOG_PROGRESS(STATUS, "MOD_INIT_LOOP") << "Initialising \"" << module->getUniqueName() << "\"";
633
634
        // Initialise the module
        module->initialise();
635
636
637

        // Reset logging
        Log::setSection(old_section_name);
638
        set_module_after(old_settings);
Simon Spannagel's avatar
Simon Spannagel committed
639
    }
640
641
}

642
// Finalise all modules
643
void ModuleManager::finaliseAll() {
644
    Configuration& global_config = conf_manager_->getGlobalConfiguration();
Simon Spannagel's avatar
Simon Spannagel committed
645

646
647
648
    // Loop over all modules and finalise them
    LOG(STATUS) << "===================| Finalising modules |===================";
    for(auto& module : m_modules) {
649
650
        // Set init module section header
        std::string old_section_name = Log::getSection();
Simon Spannagel's avatar
Simon Spannagel committed
651
        std::string section_name = "F:";
Simon Spannagel's avatar
Simon Spannagel committed
652
        section_name += module->getUniqueName();
653
654
        Log::setSection(section_name);
        // Set module specific settings
Simon Spannagel's avatar
Simon Spannagel committed
655
        auto old_settings = set_module_before(module->getUniqueName(), module->getConfig());
656
657
        // Change to our ROOT directory
        module->getROOTDirectory()->cd();
658

659
660
        // Finalise the module
        module->finalise();
661

662
663
664
        // Store all ROOT objects:
        module->getROOTDirectory()->Write();

665
666
667
        // Remove the pointer to the ROOT directory after finalizing
        module->set_ROOT_directory(nullptr);

668
669
        // Reset logging
        Log::setSection(old_section_name);
670
        set_module_after(old_settings);
Simon Spannagel's avatar
Simon Spannagel committed
671
    }
Simon Spannagel's avatar
Simon Spannagel committed
672

Simon Spannagel's avatar
Simon Spannagel committed
673
674
    // Write the output histogram file
    m_histogramFile->Close();
675

676
    LOG(STATUS) << "Wrote histogram output file to " << global_config.getPath("histogramFile");
677

678
679
680
681
682
683
684
685
686
687
    // Write out update detectors file:
    if(global_config.has("detectors_file_updated")) {
        std::string file_name = global_config.getPath("detectors_file_updated");
        // Check if the file exists
        std::ofstream file(file_name);
        if(!file) {
            throw ConfigFileUnavailableError(file_name);
        }

        ConfigReader final_detectors;
688
        for(auto& detector : m_detectors) {
689
690
691
692
693
694
695
            final_detectors.addConfiguration(detector->getConfiguration());
        }

        final_detectors.write(file);
        LOG(STATUS) << "Wrote updated detector configuration to " << file_name;
    }

Simon Spannagel's avatar
Simon Spannagel committed
696
697
    // Check the timing for all events
    timing();
698
699
}

700
// Display timing statistics for each module, over all events and per event
701
void ModuleManager::timing() {
702
    LOG(STATUS) << "===============| Wall-clock timing (seconds) |================";
703
    for(auto& module : m_modules) {
Simon Spannagel's avatar
Simon Spannagel committed
704
        auto identifier = module->get_identifier().getIdentifier();
705
        LOG(STATUS) << std::setw(20) << module->getConfig().getName() << (identifier.empty() ? "   " : " : ")
Simon Spannagel's avatar
Simon Spannagel committed
706
                    << std::setw(10) << identifier << "  --  " << std::fixed << std::setprecision(5)
707
708
                    << module_execution_time_[module.get()] << "s = " << std::setprecision(6)
                    << 1000 * module_execution_time_[module.get()] / m_events << "ms/evt";
Simon Spannagel's avatar
Simon Spannagel committed
709
    }
710
    LOG(STATUS) << "==============================================================";
711
}
712
713

// Helper functions to set the module specific log settings if necessary
714
std::tuple<LogLevel, LogFormat> ModuleManager::set_module_before(const std::string&, const Configuration& config) {
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
    // Set new log level if necessary
    LogLevel prev_level = Log::getReportingLevel();
    if(config.has("log_level")) {
        std::string log_level_string = config.get<std::string>("log_level");
        std::transform(log_level_string.begin(), log_level_string.end(), log_level_string.begin(), ::toupper);
        try {
            LogLevel log_level = Log::getLevelFromString(log_level_string);
            if(log_level != prev_level) {
                LOG(TRACE) << "Local log level is set to " << log_level_string;
                Log::setReportingLevel(log_level);
            }
        } catch(std::invalid_argument& e) {
            throw InvalidValueError(config, "log_level", e.what());
        }
    }

    // Set new log format if necessary
    LogFormat prev_format = Log::getFormat();
    if(config.has("log_format")) {
        std::string log_format_string = config.get<std::string>("log_format");
        std::transform(log_format_string.begin(), log_format_string.end(), log_format_string.begin(), ::toupper);
        try {
            LogFormat log_format = Log::getFormatFromString(log_format_string);
            if(log_format != prev_format) {
                LOG(TRACE) << "Local log format is set to " << log_format_string;
                Log::setFormat(log_format);
            }
        } catch(std::invalid_argument& e) {
            throw InvalidValueError(config, "log_format", e.what());
        }
    }

    return std::make_tuple(prev_level, prev_format);
}
749
void ModuleManager::set_module_after(std::tuple<LogLevel, LogFormat> prev) {
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
    // Reset the previous log level
    LogLevel cur_level = Log::getReportingLevel();
    LogLevel old_level = std::get<0>(prev);
    if(cur_level != old_level) {
        Log::setReportingLevel(old_level);
        LOG(TRACE) << "Reset log level to global level of " << Log::getStringFromLevel(old_level);
    }

    // Reset the previous log format
    LogFormat cur_format = Log::getFormat();
    LogFormat old_format = std::get<1>(prev);
    if(cur_format != old_format) {
        Log::setFormat(old_format);
        LOG(TRACE) << "Reset log format to global level of " << Log::getStringFromFormat(old_format);
    }
}