ModuleManager.cpp 33.8 KB
Newer Older
1
2
3
4
5
6
7
8
/** @file
 *  @brief Interface to the core framework
 *  @copyright Copyright (c) 2017 CERN and the Corryvreckan authors.
 * This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
 * In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
 * Intergovernmental Organization or submit itself to any jurisdiction.
 */

9
// ROOT include files
10
11
12
13
#include <Math/DisplacementVector2D.h>
#include <Math/Vector2D.h>
#include <Math/Vector3D.h>
#include <TFile.h>
14
#include <TSystem.h>
15
16

// Local include files
17
#include "ModuleManager.hpp"
18
#include "core/utils/file.h"
19
20
#include "core/utils/log.h"
#include "exceptions.h"
21

22
#include <chrono>
23
24
#include <dlfcn.h>
#include <fstream>
25
#include <iomanip>
26

27
28
#define CORRYVRECKAN_MODULE_PREFIX "libCorryvreckanModule"
#define CORRYVRECKAN_GENERATOR_FUNCTION "corryvreckan_module_generator"
29
30
#define CORRYVRECKAN_GLOBAL_FUNCTION "corryvreckan_module_is_global"
#define CORRYVRECKAN_DUT_FUNCTION "corryvreckan_module_is_dut"
31
#define CORRYVRECKAN_TYPE_FUNCTION "corryvreckan_detector_types"
32

33
34
using namespace corryvreckan;

35
// Default constructor
36
ModuleManager::ModuleManager() : m_reference(nullptr), m_terminate(false) {
Simon Spannagel's avatar
Simon Spannagel committed
37
    // New clipboard for storage:
38
    m_clipboard = std::make_shared<Clipboard>();
39
40
}

41
42
void ModuleManager::load(ConfigManager* conf_mgr) {
    conf_manager_ = conf_mgr;
43

44
    load_detectors();
45
    load_modules();
46
47
}

48
void ModuleManager::load_detectors() {
49
    Configuration& global_config = conf_manager_->getGlobalConfiguration();
50

51
    for(auto& detector_section : conf_manager_->getDetectorConfigurations()) {
52

53
        std::string name = detector_section.getName();
54

55
56
57
58
59
60
61
        // Check if we have a duplicate:
        if(std::find_if(m_detectors.begin(), m_detectors.end(), [&name](std::shared_ptr<Detector> obj) {
               return obj->name() == name;
           }) != m_detectors.end()) {
            throw InvalidValueError(
                global_config, "detectors_file", "Detector " + detector_section.getName() + " defined twice");
        }
62

63
64
        LOG_PROGRESS(STATUS, "DET_LOAD_LOOP") << "Loading detector " << name;
        auto detector = std::make_shared<Detector>(detector_section);
65

66
67
68
69
        // Check if we already found a reference plane:
        if(m_reference != nullptr && detector->isReference()) {
            throw InvalidValueError(global_config, "detectors_file", "Found more than one reference detector");
        }
70

71
72
73
        // Switch flag if we found the reference plane:
        if(detector->isReference()) {
            m_reference = detector;
74
        }
75
76
77

        // Add the new detector to the global list:
        m_detectors.push_back(detector);
78
    }
79

80
    // Check that exactly one detector is marked as reference:
Simon Spannagel's avatar
Simon Spannagel committed
81
    if(m_reference == nullptr) {
82
83
84
        throw InvalidValueError(global_config, "detectors_file", "Found no detector marked as reference");
    }

85
    LOG_PROGRESS(STATUS, "DET_LOAD_LOOP") << "Loaded " << m_detectors.size() << " detectors";
86
87

    // Finally, sort the list of detectors by z position (from lowest to highest)
88
    std::sort(m_detectors.begin(), m_detectors.end(), [](std::shared_ptr<Detector> det1, std::shared_ptr<Detector> det2) {
89
        return det1->displacement().Z() < det2->displacement().Z();
Simon Spannagel's avatar
Simon Spannagel committed
90
    });
91
}
92

93
void ModuleManager::load_modules() {
94
95
    auto& configs = conf_manager_->getModuleConfigurations();
    Configuration& global_config = conf_manager_->getGlobalConfiguration();
Simon Spannagel's avatar
Simon Spannagel committed
96

97
    // (Re)create the main ROOT file
98
    global_config.setAlias("histogram_file", "histogramFile");
99
100
    auto path = std::string(gSystem->pwd()) + "/" + global_config.get<std::string>("histogram_file", "histograms");
    path = corryvreckan::add_file_extension(path, "root");
101

102
103
104
105
106
107
108
109
    if(corryvreckan::path_is_file(path)) {
        if(global_config.get<bool>("deny_overwrite", false)) {
            throw RuntimeError("Overwriting of existing main ROOT file " + path + " denied");
        }
        LOG(WARNING) << "Main ROOT file " << path << " exists and will be overwritten.";
        corryvreckan::remove_file(path);
    }
    m_histogramFile = std::make_unique<TFile>(path.c_str(), "RECREATE");
Simon Spannagel's avatar
Simon Spannagel committed
110
    if(m_histogramFile->IsZombie()) {
111
        throw RuntimeError("Cannot create main ROOT file " + path);
Simon Spannagel's avatar
Simon Spannagel committed
112
    }
113
    m_histogramFile->cd();
Simon Spannagel's avatar
Simon Spannagel committed
114

115
116
117
    // Update the histogram file path:
    global_config.set("histogram_file", path);

118
    LOG(DEBUG) << "Start loading modules, have " << configs.size() << " configurations.";
Simon Spannagel's avatar
Simon Spannagel committed
119
120
    // Loop through all non-global configurations
    for(auto& config : configs) {
121
        // Load library for each module. Libraries are named (by convention + CMAKE libCorryvreckanModule Name.suffix
Simon Spannagel's avatar
Simon Spannagel committed
122
        std::string lib_name =
123
            std::string(CORRYVRECKAN_MODULE_PREFIX).append(config.getName()).append(SHARED_LIBRARY_SUFFIX);
124
        LOG_PROGRESS(STATUS, "MOD_LOAD_LOOP") << "Loading module " << config.getName();
Simon Spannagel's avatar
Simon Spannagel committed
125
126
127
128
129
130
131

        void* lib = nullptr;
        bool load_error = false;
        dlerror();
        if(loaded_libraries_.count(lib_name) == 0) {
            // If library is not loaded then try to load it first from the config
            // directories
Simon Spannagel's avatar
Simon Spannagel committed
132
133
            if(global_config.has("library_directories")) {
                std::vector<std::string> lib_paths = global_config.getPathArray("library_directories", true);
Simon Spannagel's avatar
Simon Spannagel committed
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
                for(auto& lib_path : lib_paths) {
                    std::string full_lib_path = lib_path;
                    full_lib_path += "/";
                    full_lib_path += lib_name;

                    // Check if the absolute file exists and try to load if it exists
                    std::ifstream check_file(full_lib_path);
                    if(check_file.good()) {
                        lib = dlopen(full_lib_path.c_str(), RTLD_NOW);
                        if(lib != nullptr) {
                            LOG(DEBUG) << "Found library in configuration specified directory at " << full_lib_path;
                        } else {
                            load_error = true;
                        }
                        break;
                    }
                }
            }

            // Otherwise try to load from the standard paths if not found already
            if(!load_error && lib == nullptr) {
                lib = dlopen(lib_name.c_str(), RTLD_NOW);

                if(lib != nullptr) {
                    LOG(TRACE) << "Opened library";
                    Dl_info dl_info;
                    dl_info.dli_fname = "";

                    // workaround to get the location of the library
                    int ret = dladdr(dlsym(lib, CORRYVRECKAN_GENERATOR_FUNCTION), &dl_info);
                    if(ret != 0) {
                        LOG(DEBUG) << "Found library during global search in runtime paths at " << dl_info.dli_fname;
                    } else {
                        LOG(WARNING) << "Found library during global search but could not "
                                        "deduce location, likely broken library";
                    }
                } else {
                    load_error = true;
                }
173
174
            }
        } else {
Simon Spannagel's avatar
Simon Spannagel committed
175
176
            // Otherwise just fetch it from the cache
            lib = loaded_libraries_[lib_name];
177
178
        }

Simon Spannagel's avatar
Simon Spannagel committed
179
180
181
        // If library did not load then throw exception
        if(load_error) {
            const char* lib_error = dlerror();
182

Simon Spannagel's avatar
Simon Spannagel committed
183
184
185
186
187
188
189
            // Find the name of the loaded library if it exists
            std::string lib_error_str = lib_error;
            size_t end_pos = lib_error_str.find(':');
            std::string problem_lib;
            if(end_pos != std::string::npos) {
                problem_lib = lib_error_str.substr(0, end_pos);
            }
190

Simon Spannagel's avatar
Simon Spannagel committed
191
192
193
194
195
196
197
198
199
200
            // FIXME is checking the error in this way portable?
            if(lib_error != nullptr && std::strstr(lib_error, "cannot allocate memory in static TLS block") != nullptr) {
                LOG(ERROR) << "Library could not be loaded: not enough thread local storage "
                              "available"
                           << std::endl
                           << "Try one of below workarounds:" << std::endl
                           << "- Rerun library with the environmental variable LD_PRELOAD='" << problem_lib << "'"
                           << std::endl
                           << "- Recompile the library " << problem_lib << " with tls-model=global-dynamic";
            } else if(lib_error != nullptr && std::strstr(lib_error, "cannot open shared object file") != nullptr &&
201
                      problem_lib.find(CORRYVRECKAN_MODULE_PREFIX) == std::string::npos) {
Simon Spannagel's avatar
Simon Spannagel committed
202
203
204
205
206
207
208
209
210
211
212
213
                LOG(ERROR) << "Library could not be loaded: one of its dependencies is missing" << std::endl
                           << "The name of the missing library is " << problem_lib << std::endl
                           << "Please make sure the library is properly initialized and try "
                              "again";
            } else {
                LOG(ERROR) << "Library could not be loaded: it is not available" << std::endl
                           << " - Did you enable the library during building? " << std::endl
                           << " - Did you spell the library name correctly (case-sensitive)? ";
                if(lib_error != nullptr) {
                    LOG(DEBUG) << "Detailed error: " << lib_error;
                }
            }
214

215
            throw corryvreckan::DynamicLibraryError("Error loading " + config.getName());
Simon Spannagel's avatar
Simon Spannagel committed
216
217
218
        }
        // Remember that this library was loaded
        loaded_libraries_[lib_name] = lib;
219

220
        // Check if this module is produced once, or once per detector
221
222
        void* globalFunction = dlsym(loaded_libraries_[lib_name], CORRYVRECKAN_GLOBAL_FUNCTION);
        void* dutFunction = dlsym(loaded_libraries_[lib_name], CORRYVRECKAN_DUT_FUNCTION);
223
        void* typeFunction = dlsym(loaded_libraries_[lib_name], CORRYVRECKAN_TYPE_FUNCTION);
224

225
        // If the global function was not found, throw an error
226
        if(globalFunction == nullptr || dutFunction == nullptr || typeFunction == nullptr) {
227
228
            LOG(ERROR) << "Module library is invalid or outdated: required interface function not found!";
            throw corryvreckan::DynamicLibraryError(config.getName());
229
230
        }

231
232
233
234
235
        bool global = reinterpret_cast<bool (*)()>(globalFunction)();      // NOLINT
        bool dut_only = reinterpret_cast<bool (*)()>(dutFunction)();       // NOLINT
        char* type_tokens = reinterpret_cast<char* (*)()>(typeFunction)(); // NOLINT

        std::vector<std::string> types = get_type_vector(type_tokens);
236

Simon Spannagel's avatar
Simon Spannagel committed
237
238
239
        // Add the global internal parameters to the configuration
        std::string global_dir = gSystem->pwd();
        config.set<std::string>("_global_dir", global_dir);
240

241
        // Create the modules from the library
242
        std::vector<std::pair<ModuleIdentifier, Module*>> mod_list;
243
        if(global) {
244
            mod_list.emplace_back(create_unique_module(loaded_libraries_[lib_name], config));
245
        } else {
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
            mod_list = create_detector_modules(loaded_libraries_[lib_name], config, dut_only, types);
        }

        // Loop through all created instantiations
        for(auto& id_mod : mod_list) {
            // FIXME: This convert the module to an unique pointer. Check that this always works and we can do this earlier
            std::unique_ptr<Module> mod(id_mod.second);
            ModuleIdentifier identifier = id_mod.first;

            // Check if the unique instantiation already exists
            auto iter = id_to_module_.find(identifier);
            if(iter != id_to_module_.end()) {
                // Unique name exists, check if its needs to be replaced
                if(identifier.getPriority() < iter->first.getPriority()) {
                    // Priority of new instance is higher, replace the instance
                    LOG(TRACE) << "Replacing model instance " << iter->first.getUniqueName()
                               << " with instance with higher priority.";

                    module_execution_time_.erase(iter->second->get());
                    iter->second = m_modules.erase(iter->second);
                    iter = id_to_module_.erase(iter);
                } else {
                    // Priority is equal, raise an error
                    if(identifier.getPriority() == iter->first.getPriority()) {
                        throw AmbiguousInstantiationError(config.getName());
                    }
                    // Priority is lower, do not add this module to the run list
                    continue;
                }
275
            }
276
277
278
279
280
281
282
283

            // Save the identifier in the module
            mod->set_identifier(identifier);
            mod->setReference(m_reference);

            // Add the new module to the run list
            m_modules.emplace_back(std::move(mod));
            id_to_module_[identifier] = --m_modules.end();
284
        }
Simon Spannagel's avatar
Simon Spannagel committed
285
    }
286

287
    LOG_PROGRESS(STATUS, "MOD_LOAD_LOOP") << "Loaded " << m_modules.size() << " module instances";
288
289
}

290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
std::vector<std::string> ModuleManager::get_type_vector(char* type_tokens) {
    std::vector<std::string> types;

    std::stringstream tokenstream(type_tokens);
    while(tokenstream.good()) {
        std::string token;
        getline(tokenstream, token, ',');
        if(token.empty()) {
            continue;
        }
        std::transform(token.begin(), token.end(), token.begin(), ::tolower);
        types.push_back(token);
    }
    return types;
}

306
307
308
309
310
311
312
std::shared_ptr<Detector> ModuleManager::get_detector(std::string name) {
    auto it = find_if(
        m_detectors.begin(), m_detectors.end(), [&name](std::shared_ptr<Detector> obj) { return obj->name() == name; });
    return (it != m_detectors.end() ? (*it) : nullptr);
}

std::pair<ModuleIdentifier, Module*> ModuleManager::create_unique_module(void* library, Configuration config) {
Simon Spannagel's avatar
Simon Spannagel committed
313
314
315
    // Create the identifier
    ModuleIdentifier identifier(config.getName(), "", 0);

316
317
318
319
320
321
322
323
    // Return error if user tried to specialize the unique module:
    if(config.has("name")) {
        throw InvalidValueError(config, "name", "unique modules cannot be specialized using the \"name\" keyword.");
    }
    if(config.has("type")) {
        throw InvalidValueError(config, "type", "unique modules cannot be specialized using the \"type\" keyword.");
    }

Simon Spannagel's avatar
Simon Spannagel committed
324
325
    LOG(TRACE) << "Creating module " << identifier.getUniqueName() << ", using generator \""
               << CORRYVRECKAN_GENERATOR_FUNCTION << "\"";
Simon Spannagel's avatar
Simon Spannagel committed
326
327
328
329
330

    // Get the generator function for this module
    void* generator = dlsym(library, CORRYVRECKAN_GENERATOR_FUNCTION);
    // If the generator function was not found, throw an error
    if(generator == nullptr) {
331
        LOG(ERROR) << "Module library is invalid or outdated: required "
Simon Spannagel's avatar
Simon Spannagel committed
332
                      "interface function not found!";
333
        throw corryvreckan::RuntimeError("Error instantiating module from " + config.getName());
Simon Spannagel's avatar
Simon Spannagel committed
334
    }
335

336
337
338
    // Create and add module instance config
    Configuration& instance_config = conf_manager_->addInstanceConfiguration(identifier, config);

339
340
341
342
343
344
345
    // Specialize instance configuration
    auto output_dir = instance_config.get<std::string>("_global_dir");
    output_dir += "/";
    std::string path_mod_name = identifier.getUniqueName();
    std::replace(path_mod_name.begin(), path_mod_name.end(), ':', '_');
    output_dir += path_mod_name;

Simon Spannagel's avatar
Simon Spannagel committed
346
    // Convert to correct generator function
347
348
    auto module_generator =
        reinterpret_cast<Module* (*)(Configuration, std::vector<std::shared_ptr<Detector>>)>(generator); // NOLINT
349

350
351
352
    // Set the log section header
    std::string old_section_name = Log::getSection();
    std::string section_name = "C:";
353
    section_name += identifier.getUniqueName();
354
355
    Log::setSection(section_name);
    // Set module specific log settings
356
    auto old_settings = set_module_before(identifier.getUniqueName(), instance_config);
357
    // Build module
358
    Module* module = module_generator(instance_config, m_detectors);
359
360
    // Reset log
    Log::setSection(old_section_name);
361
    set_module_after(old_settings);
362

363
364
365
366
    // Set the module directory afterwards to catch invalid access in constructor
    module->get_configuration().set<std::string>("_output_dir", output_dir);

    // Return the module to the ModuleManager
Simon Spannagel's avatar
Simon Spannagel committed
367
    return std::make_pair(identifier, module);
368
369
}

Simon Spannagel's avatar
Simon Spannagel committed
370
std::vector<std::pair<ModuleIdentifier, Module*>>
371
ModuleManager::create_detector_modules(void* library, Configuration config, bool dut_only, std::vector<std::string> types) {
372
373
374
375
376
377
378
379
380
381
382
383
384
    LOG(TRACE) << "Creating instantiations for module " << config.getName() << ", using generator \""
               << CORRYVRECKAN_GENERATOR_FUNCTION << "\"";

    // Get the generator function for this module
    void* generator = dlsym(library, CORRYVRECKAN_GENERATOR_FUNCTION);
    // If the generator function was not found, throw an error
    if(generator == nullptr) {
        LOG(ERROR) << "Module library is invalid or outdated: required "
                      "interface function not found!";
        throw corryvreckan::RuntimeError("Error instantiating module from " + config.getName());
    }

    // Convert to correct generator function
385
    auto module_generator = reinterpret_cast<Module* (*)(Configuration, std::shared_ptr<Detector>)>(generator); // NOLINT
386
    auto module_base_name = config.getName();
387
388

    // Figure out which detectors should run on this module:
389
390
391
392
393
394
395
396
397
398
399
    bool instances_created = false;
    std::vector<std::pair<std::shared_ptr<Detector>, ModuleIdentifier>> instantiations;

    // Create all names first with highest priority
    std::set<std::string> module_names;
    if(config.has("name")) {
        std::vector<std::string> names = config.getArray<std::string>("name");
        for(auto& name : names) {
            auto det = get_detector(name);
            if(det == nullptr) {
                continue;
400
            }
401
402
403
404
405

            LOG(TRACE) << "Preparing \"name\" instance for " << det->name();
            instantiations.emplace_back(det, ModuleIdentifier(module_base_name, det->name(), 0));
            // Save the name (to not instantiate it again later)
            module_names.insert(name);
406
        }
407
        instances_created = !names.empty();
408
409
    }

410
411
412
413
414
415
416
    // Then create all types that are not yet name instantiated
    if(config.has("type")) {
        // Prepare type list from configuration:
        std::vector<std::string> ctypes = config.getArray<std::string>("type");
        for(auto& type : ctypes) {
            std::transform(type.begin(), type.end(), type.begin(), ::tolower);
        }
417

418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
        // Check that this is possible at all:
        std::vector<std::string> intersection;
        std::set_intersection(ctypes.begin(), ctypes.end(), types.begin(), types.end(), std::back_inserter(intersection));
        if(!types.empty() && intersection.empty()) {
            throw InvalidInstantiationError(module_base_name, "type conflict");
        }

        for(auto& det : m_detectors) {
            auto detectortype = det->type();
            std::transform(detectortype.begin(), detectortype.end(), detectortype.begin(), ::tolower);

            // Skip all that were already added by name
            if(module_names.find(det->name()) != module_names.end()) {
                continue;
            }
            for(auto& type : ctypes) {
                // Skip all with wrong type
                if(detectortype != type) {
                    continue;
                }

                LOG(TRACE) << "Preparing \"type\" instance for " << det->name();
                instantiations.emplace_back(det, ModuleIdentifier(module_base_name, det->name(), 1));
441
442
            }
        }
443
444
445
446
447
448
449
450
451
        instances_created = !ctypes.empty();
    }

    // Create for all detectors if no name / type provided
    if(!instances_created) {
        for(auto& det : m_detectors) {
            LOG(TRACE) << "Preparing \"other\" instance for " << det->name();
            instantiations.emplace_back(det, ModuleIdentifier(module_base_name, det->name(), 2));
        }
452
453
    }

Simon Spannagel's avatar
Simon Spannagel committed
454
    std::vector<std::pair<ModuleIdentifier, Module*>> modules;
455
456
457
458
    for(const auto& instance : instantiations) {
        auto detector = instance.first;
        auto identifier = instance.second;

459
460
        // If this should only be instantiated for DUTs, skip otherwise:
        if(dut_only && !detector->isDUT()) {
Simon Spannagel's avatar
Simon Spannagel committed
461
            LOG(TRACE) << "Skipping instantiation \"" << identifier.getUniqueName() << "\", detector is no DUT";
462
463
            continue;
        }
464
465
466
467
468
469
470
471

        // Do not instantiate module if detector type is not mentioned as supported:
        auto detectortype = detector->type();
        std::transform(detectortype.begin(), detectortype.end(), detectortype.begin(), ::tolower);
        if(!types.empty() && std::find(types.begin(), types.end(), detectortype) == types.end()) {
            LOG(TRACE) << "Skipping instantiation \"" << identifier.getUniqueName() << "\", detector type mismatch";
            continue;
        }
472
        LOG(DEBUG) << "Creating instantiation \"" << identifier.getUniqueName() << "\"";
473

474
475
476
        // Create and add module instance config
        Configuration& instance_config = conf_manager_->addInstanceConfiguration(instance.second, config);

477
478
479
480
481
482
483
        // Add internal module config
        auto output_dir = instance_config.get<std::string>("_global_dir");
        output_dir += "/";
        std::string path_mod_name = instance.second.getUniqueName();
        std::replace(path_mod_name.begin(), path_mod_name.end(), ':', '/');
        output_dir += path_mod_name;

484
485
486
        // Set the log section header
        std::string old_section_name = Log::getSection();
        std::string section_name = "C:";
Simon Spannagel's avatar
Simon Spannagel committed
487
        section_name += identifier.getUniqueName();
488
489
        Log::setSection(section_name);
        // Set module specific log settings
490
        auto old_settings = set_module_before(identifier.getUniqueName(), instance_config);
491
        // Build module
492
        Module* module = module_generator(instance_config, detector);
493
494
495
        // Reset log
        Log::setSection(old_section_name);
        set_module_after(old_settings);
496
497
498
499
500

        // Set the module directory afterwards to catch invalid access in constructor
        module->get_configuration().set<std::string>("_output_dir", output_dir);

        modules.emplace_back(identifier, module);
501
502
503
504
505
506
    }

    // Return the list of modules to the analysis
    return modules;
}

507
// Run the analysis loop - this initialises, runs and finalises all modules
508
void ModuleManager::run() {
509
    Configuration& global_config = conf_manager_->getGlobalConfiguration();
Simon Spannagel's avatar
Simon Spannagel committed
510

511
    // Check if we have an event or track limit:
512
513
    auto number_of_events = global_config.get<int>("number_of_events", -1);
    auto number_of_tracks = global_config.get<int>("number_of_tracks", -1);
514
    auto run_time = global_config.get<double>("run_time", static_cast<double>(Units::convert(-1.0, "s")));
515

516
    // Loop over all events, running each module on each "event"
Simon Spannagel's avatar
Simon Spannagel committed
517
    LOG(STATUS) << "========================| Event loop |========================";
518
    m_events = 0;
519
520
    m_tracks = 0;

Simon Spannagel's avatar
Simon Spannagel committed
521
522
    while(1) {
        bool run = true;
523

524
525
        // Run all modules
        for(auto& module : m_modules) {
526
527
528
            // Get current time
            auto start = std::chrono::steady_clock::now();

529
530
531
            // Set run module section header
            std::string old_section_name = Log::getSection();
            std::string section_name = "R:";
Simon Spannagel's avatar
Simon Spannagel committed
532
            section_name += module->getUniqueName();
533
534
            Log::setSection(section_name);
            // Set module specific settings
Simon Spannagel's avatar
Simon Spannagel committed
535
            auto old_settings = set_module_before(module->getUniqueName(), module->getConfig());
Simon Spannagel's avatar
Simon Spannagel committed
536
            // Change to the output file directory
537
538
            module->getROOTDirectory()->cd();

539
            StatusCode check = module->run(m_clipboard);
540

541
542
            // Reset logging
            Log::setSection(old_section_name);
543
            set_module_after(old_settings);
544
545
546

            // Update execution time
            auto end = std::chrono::steady_clock::now();
547
            module_execution_time_[module.get()] += static_cast<std::chrono::duration<long double>>(end - start).count();
548

549
550
            if(check == StatusCode::DeadTime) {
                // If status code indicates dead time, just silently continue with next event:
551
                break;
552
553
554
555
556
557
            } else if(check == StatusCode::Failure) {
                // If the status code indicates failure, break immediately and finish:
                run = false;
                break;
            } else if(check == StatusCode::EndRun) {
                // If the returned status code asks for end-of-run, finish module list and finish:
Simon Spannagel's avatar
Simon Spannagel committed
558
                run = false;
559
            }
Simon Spannagel's avatar
Simon Spannagel committed
560
561
        }

562
563
564
565
566
        // Check if we have reached the maximum number of events
        if(number_of_events > -1 && m_events >= number_of_events) {
            break;
        }

567
        if(m_clipboard->isEventDefined() && run_time > 0.0 && m_clipboard->getEvent()->start() >= run_time) {
568
569
570
571
572
573
574
575
            break;
        }

        // Check if we have reached the maximum number of tracks
        if(number_of_tracks > -1 && m_tracks >= number_of_tracks) {
            break;
        }

576
        // Print statistics:
577
        auto tracks = m_clipboard->getData<Track>();
578
        m_tracks += (tracks == nullptr ? 0 : static_cast<int>(tracks->size()));
579

580
        if(m_events % 100 == 0) {
581
582
583
584
585
586
587
588
589

            auto kilo_or_mega = [](const double& input) {
                bool mega = (input > 1e6 ? true : false);
                auto value = (mega ? input * 1e-6 : input * 1e-3);
                std::stringstream output;
                output << std::fixed << std::setprecision(mega ? 2 : 1) << value << (mega ? "M" : "k");
                return output.str();
            };

590
            LOG_PROGRESS(STATUS, "event_loop")
591
                << "Ev: " << kilo_or_mega(m_events) << " Tr: " << kilo_or_mega(m_tracks) << " (" << std::setprecision(3)
592
                << (static_cast<double>(m_tracks) / m_events) << "/ev)"
593
594
                << (m_clipboard->isEventDefined()
                        ? " t = " + Units::display(m_clipboard->getEvent()->start(), {"ns", "us", "ms", "s"})
595
                        : "");
596
597
        }

Simon Spannagel's avatar
Simon Spannagel committed
598
599
        // Clear objects from this iteration from the clipboard
        m_clipboard->clear();
600

601
        // Check if any of the modules return a value saying it should stop
602
        if(!run) {
Simon Spannagel's avatar
Simon Spannagel committed
603
            break;
604
605
        }

Simon Spannagel's avatar
Simon Spannagel committed
606
        // Increment event number
607
        m_events++;
608
609
610

        // Check for user termination and stop the event loop:
        if(m_terminate) {
611
            break;
612
        }
613
614
615
    }
}

616
void ModuleManager::terminate() {
617
618
619
    m_terminate = true;
}

620
// Initalise all modules
621
void ModuleManager::initialiseAll() {
622
623
624
    // Loop over all modules and initialise them
    LOG(STATUS) << "=================| Initialising modules |==================";
    for(auto& module : m_modules) {
625
626
627
        // Pass the config manager to this instance
        module->set_config_manager(conf_manager_);

628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
        // Create main ROOT directory for this module class if it does not exists yet
        LOG(TRACE) << "Creating and accessing ROOT directory";
        std::string module_name = module->getConfig().getName();
        auto directory = m_histogramFile->GetDirectory(module_name.c_str());
        if(directory == nullptr) {
            directory = m_histogramFile->mkdir(module_name.c_str());
            if(directory == nullptr) {
                throw RuntimeError("Cannot create or access overall ROOT directory for module " + module_name);
            }
        }
        directory->cd();

        // Create local directory for this instance
        TDirectory* local_directory = nullptr;
        if(module->get_identifier().getIdentifier().empty()) {
            local_directory = directory;
        } else {
            local_directory = directory->mkdir(module->get_identifier().getIdentifier().c_str());
            if(local_directory == nullptr) {
                throw RuntimeError("Cannot create or access local ROOT directory for module " + module->getUniqueName());
            }
        }

        // Change to the directory and save it in the module
        local_directory->cd();
        module->set_ROOT_directory(local_directory);

655
656
657
        // Set init module section header
        std::string old_section_name = Log::getSection();
        std::string section_name = "I:";
Simon Spannagel's avatar
Simon Spannagel committed
658
        section_name += module->getUniqueName();
659
660
        Log::setSection(section_name);
        // Set module specific settings
Simon Spannagel's avatar
Simon Spannagel committed
661
        auto old_settings = set_module_before(module->getUniqueName(), module->getConfig());
662
663
        // Change to our ROOT directory
        module->getROOTDirectory()->cd();
664

665
        LOG_PROGRESS(STATUS, "MOD_INIT_LOOP") << "Initialising \"" << module->getUniqueName() << "\"";
666
667
        // Initialise the module
        module->initialise();
668
669
670

        // Reset logging
        Log::setSection(old_section_name);
671
        set_module_after(old_settings);
Simon Spannagel's avatar
Simon Spannagel committed
672
    }
673
674
}

675
// Finalise all modules
676
void ModuleManager::finaliseAll() {
677
    Configuration& global_config = conf_manager_->getGlobalConfiguration();
Simon Spannagel's avatar
Simon Spannagel committed
678

679
680
681
    // Loop over all modules and finalise them
    LOG(STATUS) << "===================| Finalising modules |===================";
    for(auto& module : m_modules) {
682
683
        // Set init module section header
        std::string old_section_name = Log::getSection();
Simon Spannagel's avatar
Simon Spannagel committed
684
        std::string section_name = "F:";
Simon Spannagel's avatar
Simon Spannagel committed
685
        section_name += module->getUniqueName();
686
687
        Log::setSection(section_name);
        // Set module specific settings
Simon Spannagel's avatar
Simon Spannagel committed
688
        auto old_settings = set_module_before(module->getUniqueName(), module->getConfig());
689
690
        // Change to our ROOT directory
        module->getROOTDirectory()->cd();
691

692
693
        // Finalise the module
        module->finalise();
694

695
696
697
        // Store all ROOT objects:
        module->getROOTDirectory()->Write();

698
699
700
        // Remove the pointer to the ROOT directory after finalizing
        module->set_ROOT_directory(nullptr);

701
702
        // Reset logging
        Log::setSection(old_section_name);
703
        set_module_after(old_settings);
Simon Spannagel's avatar
Simon Spannagel committed
704
    }
Simon Spannagel's avatar
Simon Spannagel committed
705

Simon Spannagel's avatar
Simon Spannagel committed
706
707
    // Write the output histogram file
    m_histogramFile->Close();
708

709
    LOG(STATUS) << "Wrote histogram output file to " << global_config.getPath("histogram_file");
710

711
712
    // Write out update detectors file:
    if(global_config.has("detectors_file_updated")) {
713
        std::string path = global_config.getPath("detectors_file_updated");
714
        // Check if the file exists
715
716
717
718
719
720
721
722
723
        if(corryvreckan::path_is_file(path)) {
            if(global_config.get<bool>("deny_overwrite", false)) {
                throw RuntimeError("Overwriting of existing detectors file " + path + " denied");
            }
            LOG(WARNING) << "Detectors file " << path << " exists and will be overwritten.";
            corryvreckan::remove_file(path);
        }

        std::ofstream file(path);
724
        if(!file) {
725
            throw RuntimeError("Cannot create detectors file " + path);
726
727
728
        }

        ConfigReader final_detectors;
729
        for(auto& detector : m_detectors) {
730
731
732
733
            final_detectors.addConfiguration(detector->getConfiguration());
        }

        final_detectors.write(file);
734
        LOG(STATUS) << "Wrote updated detector configuration to " << path;
735
736
    }

Simon Spannagel's avatar
Simon Spannagel committed
737
738
    // Check the timing for all events
    timing();
739
740
}

741
// Display timing statistics for each module, over all events and per event
742
void ModuleManager::timing() {
743
    LOG(STATUS) << "===============| Wall-clock timing (seconds) |================";
744
    for(auto& module : m_modules) {
Simon Spannagel's avatar
Simon Spannagel committed
745
        auto identifier = module->get_identifier().getIdentifier();
746
        LOG(STATUS) << std::setw(20) << module->getConfig().getName() << (identifier.empty() ? "   " : " : ")
Simon Spannagel's avatar
Simon Spannagel committed
747
                    << std::setw(10) << identifier << "  --  " << std::fixed << std::setprecision(5)
748
749
                    << module_execution_time_[module.get()] << "s = " << std::setprecision(6)
                    << 1000 * module_execution_time_[module.get()] / m_events << "ms/evt";
Simon Spannagel's avatar
Simon Spannagel committed
750
    }
751
    LOG(STATUS) << "==============================================================";
752
}
753
754

// Helper functions to set the module specific log settings if necessary
755
std::tuple<LogLevel, LogFormat> ModuleManager::set_module_before(const std::string&, const Configuration& config) {
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
    // Set new log level if necessary
    LogLevel prev_level = Log::getReportingLevel();
    if(config.has("log_level")) {
        std::string log_level_string = config.get<std::string>("log_level");
        std::transform(log_level_string.begin(), log_level_string.end(), log_level_string.begin(), ::toupper);
        try {
            LogLevel log_level = Log::getLevelFromString(log_level_string);
            if(log_level != prev_level) {
                LOG(TRACE) << "Local log level is set to " << log_level_string;
                Log::setReportingLevel(log_level);
            }
        } catch(std::invalid_argument& e) {
            throw InvalidValueError(config, "log_level", e.what());
        }
    }

    // Set new log format if necessary
    LogFormat prev_format = Log::getFormat();
    if(config.has("log_format")) {
        std::string log_format_string = config.get<std::string>("log_format");
        std::transform(log_format_string.begin(), log_format_string.end(), log_format_string.begin(), ::toupper);
        try {
            LogFormat log_format = Log::getFormatFromString(log_format_string);
            if(log_format != prev_format) {
                LOG(TRACE) << "Local log format is set to " << log_format_string;
                Log::setFormat(log_format);
            }
        } catch(std::invalid_argument& e) {
            throw InvalidValueError(config, "log_format", e.what());
        }
    }

    return std::make_tuple(prev_level, prev_format);
}
790
void ModuleManager::set_module_after(std::tuple<LogLevel, LogFormat> prev) {
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
    // Reset the previous log level
    LogLevel cur_level = Log::getReportingLevel();
    LogLevel old_level = std::get<0>(prev);
    if(cur_level != old_level) {
        Log::setReportingLevel(old_level);
        LOG(TRACE) << "Reset log level to global level of " << Log::getStringFromLevel(old_level);
    }

    // Reset the previous log format
    LogFormat cur_format = Log::getFormat();
    LogFormat old_format = std::get<1>(prev);
    if(cur_format != old_format) {
        Log::setFormat(old_format);
        LOG(TRACE) << "Reset log format to global level of " << Log::getStringFromFormat(old_format);
    }
}