ModuleManager.cpp 32.3 KB
Newer Older
1
2
3
4
5
6
7
8
/** @file
 *  @brief Interface to the core framework
 *  @copyright Copyright (c) 2017 CERN and the Corryvreckan authors.
 * This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
 * In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
 * Intergovernmental Organization or submit itself to any jurisdiction.
 */

9
// ROOT include files
10
11
12
13
#include <Math/DisplacementVector2D.h>
#include <Math/Vector2D.h>
#include <Math/Vector3D.h>
#include <TFile.h>
14
#include <TSystem.h>
15
16

// Local include files
17
18
19
#include "ModuleManager.hpp"
#include "core/utils/log.h"
#include "exceptions.h"
20

21
#include <chrono>
22
23
#include <dlfcn.h>
#include <fstream>
24
#include <iomanip>
25

26
27
#define CORRYVRECKAN_MODULE_PREFIX "libCorryvreckanModule"
#define CORRYVRECKAN_GENERATOR_FUNCTION "corryvreckan_module_generator"
28
29
#define CORRYVRECKAN_GLOBAL_FUNCTION "corryvreckan_module_is_global"
#define CORRYVRECKAN_DUT_FUNCTION "corryvreckan_module_is_dut"
30
#define CORRYVRECKAN_TYPE_FUNCTION "corryvreckan_detector_types"
31

32
33
using namespace corryvreckan;

34
// Default constructor
35
ModuleManager::ModuleManager(std::string config_file_name, std::vector<std::string> options) : m_terminate(false) {
36

Simon Spannagel's avatar
Simon Spannagel committed
37
38
39
40
41
    LOG(TRACE) << "Loading Corryvreckan";

    // Put welcome message
    LOG(STATUS) << "Welcome to Corryvreckan " << CORRYVRECKAN_PROJECT_VERSION;

Simon Spannagel's avatar
Simon Spannagel committed
42
43
    // Load the global configuration
    conf_mgr_ = std::make_unique<corryvreckan::ConfigManager>(std::move(config_file_name));
44

Simon Spannagel's avatar
Simon Spannagel committed
45
46
47
48
    // Configure the standard special sections
    conf_mgr_->setGlobalHeaderName("Corryvreckan");
    conf_mgr_->addGlobalHeaderName("");
    conf_mgr_->addIgnoreHeaderName("Ignore");
49

50
51
52
53
54
    // Parse all command line options
    for(auto& option : options) {
        conf_mgr_->parseOption(option);
    }

Simon Spannagel's avatar
Simon Spannagel committed
55
    // Fetch the global configuration
Simon Spannagel's avatar
Simon Spannagel committed
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
    global_config = conf_mgr_->getGlobalConfiguration();

    // Set the log level from config if not specified earlier
    std::string log_level_string;
    if(Log::getReportingLevel() == LogLevel::NONE) {
        log_level_string = global_config.get<std::string>("log_level", "INFO");
        std::transform(log_level_string.begin(), log_level_string.end(), log_level_string.begin(), ::toupper);
        try {
            LogLevel log_level = Log::getLevelFromString(log_level_string);
            Log::setReportingLevel(log_level);
        } catch(std::invalid_argument& e) {
            LOG(ERROR) << "Log level \"" << log_level_string
                       << "\" specified in the configuration is invalid, defaulting to INFO instead";
            Log::setReportingLevel(LogLevel::INFO);
        }
    } else {
        log_level_string = Log::getStringFromLevel(Log::getReportingLevel());
    }

    // Set the log format from config
    std::string log_format_string = global_config.get<std::string>("log_format", "DEFAULT");
    std::transform(log_format_string.begin(), log_format_string.end(), log_format_string.begin(), ::toupper);
    try {
        LogFormat log_format = Log::getFormatFromString(log_format_string);
        Log::setFormat(log_format);
    } catch(std::invalid_argument& e) {
        LOG(ERROR) << "Log format \"" << log_format_string
                   << "\" specified in the configuration is invalid, using DEFAULT instead";
        Log::setFormat(LogFormat::DEFAULT);
    }

    // Open log file to write output to
    if(global_config.has("log_file")) {
        // NOTE: this stream should be available for the duration of the logging
        log_file_.open(global_config.getPath("log_file"), std::ios_base::out | std::ios_base::trunc);
        Log::addStream(log_file_);
    }

    // Wait for the first detailed messages until level and format are properly set
    LOG(TRACE) << "Global log level is set to " << log_level_string;
    LOG(TRACE) << "Global log format is set to " << log_format_string;
97

Simon Spannagel's avatar
Simon Spannagel committed
98
99
    // New clipboard for storage:
    m_clipboard = new Clipboard();
100
101
}

102
void ModuleManager::load() {
103
104
105

    add_units();

106
    load_detectors();
107
    load_modules();
108
109
}

110
void ModuleManager::load_detectors() {
111

112
113
114
    // Flag for the reference detector
    bool found_reference = false;

115
    std::vector<std::string> detectors_files = global_config.getPathArray("detectors_file");
116

117
118
    for(auto& detectors_file : detectors_files) {
        std::fstream file(detectors_file);
119
120
121
122
        ConfigManager det_mgr(detectors_file);
        det_mgr.addIgnoreHeaderName("Ignore");

        for(auto& detector : det_mgr.getConfigurations()) {
123
124
125
126

            std::string name = detector.getName();

            // Check if we have a duplicate:
127
128
129
            if(std::find_if(detectors.begin(), detectors.end(), [&name](std::shared_ptr<Detector> obj) {
                   return obj->name() == name;
               }) != detectors.end()) {
130
131
132
133
                throw InvalidValueError(
                    global_config, "detectors_file", "Detector " + detector.getName() + " defined twice");
            }

134
            LOG_PROGRESS(STATUS, "DET_LOAD_LOOP") << "Loading detector " << name;
135
            auto det_parm = std::make_shared<Detector>(detector);
136

137
138
139
140
141
142
            // Check if we already found a reference plane:
            if(found_reference && det_parm->role() == DetectorRole::REFERENCE) {
                throw InvalidValueError(global_config, "detectors_file", "Found more than one reference detector");
            }

            // Switch flag if we found the reference plane:
143
144
145
146
            if(det_parm->role() == DetectorRole::REFERENCE) {
                found_reference = true;
                m_reference = det_parm;
            }
147

148
149
150
            // Add the new detector to the global list:
            detectors.push_back(det_parm);
        }
151
    }
152

153
154
155
156
157
    // Check that exactly one detector is marked as reference:
    if(!found_reference) {
        throw InvalidValueError(global_config, "detectors_file", "Found no detector marked as reference");
    }

158
    LOG_PROGRESS(STATUS, "DET_LOAD_LOOP") << "Loaded " << detectors.size() << " detectors";
159
160

    // Finally, sort the list of detectors by z position (from lowest to highest)
161
    std::sort(detectors.begin(), detectors.end(), [](std::shared_ptr<Detector> det1, std::shared_ptr<Detector> det2) {
162
        return det1->displacement().Z() < det2->displacement().Z();
Simon Spannagel's avatar
Simon Spannagel committed
163
    });
164
}
165

166
void ModuleManager::load_modules() {
Simon Spannagel's avatar
Simon Spannagel committed
167
168
169
    std::vector<Configuration> configs = conf_mgr_->getConfigurations();

    // Create histogram output file
170
171
172
    global_config.setAlias("histogram_file", "histogramFile");
    std::string histogramFile = global_config.getPath("histogram_file");

173
    m_histogramFile = std::make_unique<TFile>(histogramFile.c_str(), "RECREATE");
Simon Spannagel's avatar
Simon Spannagel committed
174
    if(m_histogramFile->IsZombie()) {
175
        throw RuntimeError("Cannot create main ROOT file " + histogramFile);
Simon Spannagel's avatar
Simon Spannagel committed
176
    }
177
    m_histogramFile->cd();
Simon Spannagel's avatar
Simon Spannagel committed
178

179
    LOG(DEBUG) << "Start loading modules, have " << configs.size() << " configurations.";
Simon Spannagel's avatar
Simon Spannagel committed
180
181
    // Loop through all non-global configurations
    for(auto& config : configs) {
182
        // Load library for each module. Libraries are named (by convention + CMAKE libCorryvreckanModule Name.suffix
Simon Spannagel's avatar
Simon Spannagel committed
183
        std::string lib_name =
184
            std::string(CORRYVRECKAN_MODULE_PREFIX).append(config.getName()).append(SHARED_LIBRARY_SUFFIX);
185
        LOG_PROGRESS(STATUS, "MOD_LOAD_LOOP") << "Loading module " << config.getName();
Simon Spannagel's avatar
Simon Spannagel committed
186
187
188
189
190
191
192

        void* lib = nullptr;
        bool load_error = false;
        dlerror();
        if(loaded_libraries_.count(lib_name) == 0) {
            // If library is not loaded then try to load it first from the config
            // directories
Simon Spannagel's avatar
Simon Spannagel committed
193
194
            if(global_config.has("library_directories")) {
                std::vector<std::string> lib_paths = global_config.getPathArray("library_directories", true);
Simon Spannagel's avatar
Simon Spannagel committed
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
                for(auto& lib_path : lib_paths) {
                    std::string full_lib_path = lib_path;
                    full_lib_path += "/";
                    full_lib_path += lib_name;

                    // Check if the absolute file exists and try to load if it exists
                    std::ifstream check_file(full_lib_path);
                    if(check_file.good()) {
                        lib = dlopen(full_lib_path.c_str(), RTLD_NOW);
                        if(lib != nullptr) {
                            LOG(DEBUG) << "Found library in configuration specified directory at " << full_lib_path;
                        } else {
                            load_error = true;
                        }
                        break;
                    }
                }
            }

            // Otherwise try to load from the standard paths if not found already
            if(!load_error && lib == nullptr) {
                lib = dlopen(lib_name.c_str(), RTLD_NOW);

                if(lib != nullptr) {
                    LOG(TRACE) << "Opened library";
                    Dl_info dl_info;
                    dl_info.dli_fname = "";

                    // workaround to get the location of the library
                    int ret = dladdr(dlsym(lib, CORRYVRECKAN_GENERATOR_FUNCTION), &dl_info);
                    if(ret != 0) {
                        LOG(DEBUG) << "Found library during global search in runtime paths at " << dl_info.dli_fname;
                    } else {
                        LOG(WARNING) << "Found library during global search but could not "
                                        "deduce location, likely broken library";
                    }
                } else {
                    load_error = true;
                }
234
235
            }
        } else {
Simon Spannagel's avatar
Simon Spannagel committed
236
237
            // Otherwise just fetch it from the cache
            lib = loaded_libraries_[lib_name];
238
239
        }

Simon Spannagel's avatar
Simon Spannagel committed
240
241
242
        // If library did not load then throw exception
        if(load_error) {
            const char* lib_error = dlerror();
243

Simon Spannagel's avatar
Simon Spannagel committed
244
245
246
247
248
249
250
            // Find the name of the loaded library if it exists
            std::string lib_error_str = lib_error;
            size_t end_pos = lib_error_str.find(':');
            std::string problem_lib;
            if(end_pos != std::string::npos) {
                problem_lib = lib_error_str.substr(0, end_pos);
            }
251

Simon Spannagel's avatar
Simon Spannagel committed
252
253
254
255
256
257
258
259
260
261
            // FIXME is checking the error in this way portable?
            if(lib_error != nullptr && std::strstr(lib_error, "cannot allocate memory in static TLS block") != nullptr) {
                LOG(ERROR) << "Library could not be loaded: not enough thread local storage "
                              "available"
                           << std::endl
                           << "Try one of below workarounds:" << std::endl
                           << "- Rerun library with the environmental variable LD_PRELOAD='" << problem_lib << "'"
                           << std::endl
                           << "- Recompile the library " << problem_lib << " with tls-model=global-dynamic";
            } else if(lib_error != nullptr && std::strstr(lib_error, "cannot open shared object file") != nullptr &&
262
                      problem_lib.find(CORRYVRECKAN_MODULE_PREFIX) == std::string::npos) {
Simon Spannagel's avatar
Simon Spannagel committed
263
264
265
266
267
268
269
270
271
272
273
274
                LOG(ERROR) << "Library could not be loaded: one of its dependencies is missing" << std::endl
                           << "The name of the missing library is " << problem_lib << std::endl
                           << "Please make sure the library is properly initialized and try "
                              "again";
            } else {
                LOG(ERROR) << "Library could not be loaded: it is not available" << std::endl
                           << " - Did you enable the library during building? " << std::endl
                           << " - Did you spell the library name correctly (case-sensitive)? ";
                if(lib_error != nullptr) {
                    LOG(DEBUG) << "Detailed error: " << lib_error;
                }
            }
275

276
            throw corryvreckan::DynamicLibraryError("Error loading " + config.getName());
Simon Spannagel's avatar
Simon Spannagel committed
277
278
279
        }
        // Remember that this library was loaded
        loaded_libraries_[lib_name] = lib;
280

281
        // Check if this module is produced once, or once per detector
282
283
        void* globalFunction = dlsym(loaded_libraries_[lib_name], CORRYVRECKAN_GLOBAL_FUNCTION);
        void* dutFunction = dlsym(loaded_libraries_[lib_name], CORRYVRECKAN_DUT_FUNCTION);
284
        void* typeFunction = dlsym(loaded_libraries_[lib_name], CORRYVRECKAN_TYPE_FUNCTION);
285

286
        // If the global function was not found, throw an error
287
        if(globalFunction == nullptr || dutFunction == nullptr || typeFunction == nullptr) {
288
289
            LOG(ERROR) << "Module library is invalid or outdated: required interface function not found!";
            throw corryvreckan::DynamicLibraryError(config.getName());
290
291
        }

292
293
294
295
296
        bool global = reinterpret_cast<bool (*)()>(globalFunction)();      // NOLINT
        bool dut_only = reinterpret_cast<bool (*)()>(dutFunction)();       // NOLINT
        char* type_tokens = reinterpret_cast<char* (*)()>(typeFunction)(); // NOLINT

        std::vector<std::string> types = get_type_vector(type_tokens);
297

298
299
300
        // Apply the module specific options to the module configuration
        conf_mgr_->applyOptions(config.getName(), config);

Simon Spannagel's avatar
Simon Spannagel committed
301
302
303
        // Add the global internal parameters to the configuration
        std::string global_dir = gSystem->pwd();
        config.set<std::string>("_global_dir", global_dir);
304

305
        // Merge the global configuration into the modules config:
306
307
        config.merge(global_config);

308
        // Create the modules from the library
309
        if(global) {
310
            auto module = create_unique_module(loaded_libraries_[lib_name], config, types);
Simon Spannagel's avatar
Simon Spannagel committed
311
312
            m_modules.push_back(module.second);
            m_modules.back()->set_identifier(module.first);
313
        } else {
314
            auto modules = create_detector_modules(loaded_libraries_[lib_name], config, dut_only, types);
315
            for(const auto& mod : modules) {
Simon Spannagel's avatar
Simon Spannagel committed
316
317
                m_modules.push_back(mod.second);
                m_modules.back()->set_identifier(mod.first);
318
319
            }
        }
Simon Spannagel's avatar
Simon Spannagel committed
320
    }
321
322
323
324
325

    // Set the reference detector:
    for(auto& module : m_modules) {
        module->setReference(m_reference);
    }
326
    LOG_PROGRESS(STATUS, "MOD_LOAD_LOOP") << "Loaded " << configs.size() << " modules";
327
328
}

329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
std::vector<std::string> ModuleManager::get_type_vector(char* type_tokens) {
    std::vector<std::string> types;

    std::stringstream tokenstream(type_tokens);
    while(tokenstream.good()) {
        std::string token;
        getline(tokenstream, token, ',');
        if(token.empty()) {
            continue;
        }
        std::transform(token.begin(), token.end(), token.begin(), ::tolower);
        types.push_back(token);
    }
    return types;
}

345
346
std::pair<ModuleIdentifier, Module*>
ModuleManager::create_unique_module(void* library, Configuration config, std::vector<std::string>) {
Simon Spannagel's avatar
Simon Spannagel committed
347
348
349
350
351
    // Create the identifier
    ModuleIdentifier identifier(config.getName(), "", 0);

    LOG(TRACE) << "Creating module " << identifier.getUniqueName() << ", using generator \""
               << CORRYVRECKAN_GENERATOR_FUNCTION << "\"";
Simon Spannagel's avatar
Simon Spannagel committed
352
353
354
355
356

    // Get the generator function for this module
    void* generator = dlsym(library, CORRYVRECKAN_GENERATOR_FUNCTION);
    // If the generator function was not found, throw an error
    if(generator == nullptr) {
357
        LOG(ERROR) << "Module library is invalid or outdated: required "
Simon Spannagel's avatar
Simon Spannagel committed
358
                      "interface function not found!";
359
        throw corryvreckan::RuntimeError("Error instantiating module from " + config.getName());
Simon Spannagel's avatar
Simon Spannagel committed
360
    }
361

Simon Spannagel's avatar
Simon Spannagel committed
362
    // Convert to correct generator function
363
364
    auto module_generator =
        reinterpret_cast<Module* (*)(Configuration, std::vector<std::shared_ptr<Detector>>)>(generator); // NOLINT
365

366
    // Figure out which detectors should run on this module:
367
    std::vector<std::shared_ptr<Detector>> module_det;
368
    if(!config.has("detectors")) {
369
        module_det = detectors;
370
371
372
373
374
    } else {
        std::vector<std::string> det_list = config.getArray<std::string>("detectors");

        for(auto& d : detectors) {
            if(std::find(det_list.begin(), det_list.end(), d->name()) != det_list.end()) {
375
                module_det.push_back(d);
376
377
378
379
            }
        }
    }

380
381
382
383
    // Check for potentially masked detectors:
    if(config.has("masked")) {
        std::vector<std::string> mask_list = config.getArray<std::string>("masked");

384
        for(auto it = module_det.begin(); it != module_det.end();) {
385
386
            // Remove detectors which are masked:
            if(std::find(mask_list.begin(), mask_list.end(), (*it)->name()) != mask_list.end()) {
387
                it = module_det.erase(it);
388
389
390
391
392
393
            } else {
                it++;
            }
        }
    }

394
395
396
397
398
399
    // Set the log section header
    std::string old_section_name = Log::getSection();
    std::string section_name = "C:";
    section_name += config.getName();
    Log::setSection(section_name);
    // Set module specific log settings
400
401
402
    auto old_settings = set_module_before(config.getName(), config);
    // Build module
    Module* module = module_generator(config, module_det);
403
404
    // Reset log
    Log::setSection(old_section_name);
405
    set_module_after(old_settings);
406

407
    // Return the module to the analysis
Simon Spannagel's avatar
Simon Spannagel committed
408
    return std::make_pair(identifier, module);
409
410
}

Simon Spannagel's avatar
Simon Spannagel committed
411
std::vector<std::pair<ModuleIdentifier, Module*>>
412
ModuleManager::create_detector_modules(void* library, Configuration config, bool dut_only, std::vector<std::string> types) {
413
414
415
416
417
418
419
420
421
422
423
424
425
    LOG(TRACE) << "Creating instantiations for module " << config.getName() << ", using generator \""
               << CORRYVRECKAN_GENERATOR_FUNCTION << "\"";

    // Get the generator function for this module
    void* generator = dlsym(library, CORRYVRECKAN_GENERATOR_FUNCTION);
    // If the generator function was not found, throw an error
    if(generator == nullptr) {
        LOG(ERROR) << "Module library is invalid or outdated: required "
                      "interface function not found!";
        throw corryvreckan::RuntimeError("Error instantiating module from " + config.getName());
    }

    // Convert to correct generator function
426
    auto module_generator = reinterpret_cast<Module* (*)(Configuration, std::shared_ptr<Detector>)>(generator); // NOLINT
427
428

    // Figure out which detectors should run on this module:
429
    std::vector<std::shared_ptr<Detector>> module_det;
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
    if(!config.has("detectors")) {
        module_det = detectors;
    } else {
        std::vector<std::string> det_list = config.getArray<std::string>("detectors");

        for(auto& d : detectors) {
            if(std::find(det_list.begin(), det_list.end(), d->name()) != det_list.end()) {
                module_det.push_back(d);
            }
        }
    }

    // Check for potentially masked detectors:
    if(config.has("masked")) {
        std::vector<std::string> mask_list = config.getArray<std::string>("masked");

        for(auto it = module_det.begin(); it != module_det.end();) {
            // Remove detectors which are masked:
            if(std::find(mask_list.begin(), mask_list.end(), (*it)->name()) != mask_list.end()) {
                it = module_det.erase(it);
            } else {
                it++;
            }
        }
    }

Simon Spannagel's avatar
Simon Spannagel committed
456
    std::vector<std::pair<ModuleIdentifier, Module*>> modules;
457
458
    auto module_base_name = config.getName();
    for(const auto& detector : module_det) {
Simon Spannagel's avatar
Simon Spannagel committed
459
        ModuleIdentifier identifier(module_base_name, detector->name(), 0);
460
461
        // If this should only be instantiated for DUTs, skip otherwise:
        if(dut_only && !detector->isDUT()) {
Simon Spannagel's avatar
Simon Spannagel committed
462
            LOG(TRACE) << "Skipping instantiation \"" << identifier.getUniqueName() << "\", detector is no DUT";
463
464
            continue;
        }
465
466
467
468
469
470
471
472

        // Do not instantiate module if detector type is not mentioned as supported:
        auto detectortype = detector->type();
        std::transform(detectortype.begin(), detectortype.end(), detectortype.begin(), ::tolower);
        if(!types.empty() && std::find(types.begin(), types.end(), detectortype) == types.end()) {
            LOG(TRACE) << "Skipping instantiation \"" << identifier.getUniqueName() << "\", detector type mismatch";
            continue;
        }
Simon Spannagel's avatar
Simon Spannagel committed
473
        LOG(TRACE) << "Creating instantiation \"" << identifier.getUniqueName() << "\"";
474
475
476
477

        // Set the log section header
        std::string old_section_name = Log::getSection();
        std::string section_name = "C:";
Simon Spannagel's avatar
Simon Spannagel committed
478
        section_name += identifier.getUniqueName();
479
480
        Log::setSection(section_name);
        // Set module specific log settings
Simon Spannagel's avatar
Simon Spannagel committed
481
        auto old_settings = set_module_before(identifier.getUniqueName(), config);
482
        // Build module
Simon Spannagel's avatar
Simon Spannagel committed
483
        modules.emplace_back(identifier, module_generator(config, detector));
484
485
486
487
488
489
490
491
492
        // Reset log
        Log::setSection(old_section_name);
        set_module_after(old_settings);
    }

    // Return the list of modules to the analysis
    return modules;
}

493
// Run the analysis loop - this initialises, runs and finalises all modules
494
void ModuleManager::run() {
Simon Spannagel's avatar
Simon Spannagel committed
495

496
    // Check if we have an event or track limit:
Simon Spannagel's avatar
Simon Spannagel committed
497
    int number_of_events = global_config.get<int>("number_of_events", -1);
498
    int number_of_tracks = global_config.get<int>("number_of_tracks", -1);
499
    auto run_time = global_config.get<double>("run_time", static_cast<double>(Units::convert(-1.0, "s")));
500

501
    // Loop over all events, running each module on each "event"
Simon Spannagel's avatar
Simon Spannagel committed
502
503
    LOG(STATUS) << "========================| Event loop |========================";
    m_events = 1;
504
    int events_prev = 1;
505
506
    m_tracks = 0;
    int skipped = 0;
507
    int skipped_prev = 0;
508

Simon Spannagel's avatar
Simon Spannagel committed
509
510
511
    while(1) {
        bool run = true;
        bool noData = false;
512

Simon Spannagel's avatar
Simon Spannagel committed
513
        // Check if we have reached the maximum number of events
514

Simon Spannagel's avatar
Simon Spannagel committed
515
516
517
        if(number_of_events > -1 && m_events >= number_of_events)
            break;

518
        if(run_time > 0.0 && (m_clipboard->get_persistent("eventStart")) >= run_time)
519
520
            break;

521
522
523
524
        // Check if we have reached the maximum number of tracks
        if(number_of_tracks > -1 && m_tracks >= number_of_tracks)
            break;

525
526
        // Run all modules
        for(auto& module : m_modules) {
527
528
529
            // Get current time
            auto start = std::chrono::steady_clock::now();

530
531
532
            // Set run module section header
            std::string old_section_name = Log::getSection();
            std::string section_name = "R:";
Simon Spannagel's avatar
Simon Spannagel committed
533
            section_name += module->getUniqueName();
534
535
            Log::setSection(section_name);
            // Set module specific settings
Simon Spannagel's avatar
Simon Spannagel committed
536
            auto old_settings = set_module_before(module->getUniqueName(), module->getConfig());
Simon Spannagel's avatar
Simon Spannagel committed
537
            // Change to the output file directory
538
539
            module->getROOTDirectory()->cd();

540
            StatusCode check = module->run(m_clipboard);
541

542
543
            // Reset logging
            Log::setSection(old_section_name);
544
            set_module_after(old_settings);
545
546
547
548
549

            // Update execution time
            auto end = std::chrono::steady_clock::now();
            module_execution_time_[module] += static_cast<std::chrono::duration<long double>>(end - start).count();

Simon Spannagel's avatar
Simon Spannagel committed
550
551
            if(check == NoData) {
                noData = true;
552
                skipped++;
Simon Spannagel's avatar
Simon Spannagel committed
553
554
                break;
            } // Nothing to be done in this event
555
            if(check == Failure) {
Simon Spannagel's avatar
Simon Spannagel committed
556
                run = false;
557
            }
Simon Spannagel's avatar
Simon Spannagel committed
558
559
        }

560
        // Print statistics:
561
        Tracks* tracks = reinterpret_cast<Tracks*>(m_clipboard->get("tracks"));
562
        m_tracks += (tracks == nullptr ? 0 : static_cast<int>(tracks->size()));
563
564
565
566
567
568
569
570
571
572
573
574

        bool update_progress = false;
        if(m_events % 100 == 0 && m_events != events_prev) {
            update_progress = true;
        }
        if(skipped % 1000 == 0 && skipped != skipped_prev) {
            update_progress = true;
        }

        if(update_progress) {
            skipped_prev = skipped;
            events_prev = m_events;
575
            LOG_PROGRESS(STATUS, "event_loop")
576
                << "Ev: +" << m_events << " \\" << skipped << " Tr: " << m_tracks << " (" << std::setprecision(3)
577
                << (static_cast<double>(m_tracks) / m_events) << "/ev)"
578
579
580
                << (m_clipboard->has_persistent("eventStart")
                        ? " t = " + Units::display(m_clipboard->get_persistent("eventStart"), {"ns", "us", "ms", "s"})
                        : "");
581
582
        }

Simon Spannagel's avatar
Simon Spannagel committed
583
584
        // Clear objects from this iteration from the clipboard
        m_clipboard->clear();
585

586
        // Check if any of the modules return a value saying it should stop
587
        if(!run) {
Simon Spannagel's avatar
Simon Spannagel committed
588
            break;
589
590
        }

Simon Spannagel's avatar
Simon Spannagel committed
591
        // Increment event number
592
        if(!noData) {
Simon Spannagel's avatar
Simon Spannagel committed
593
            m_events++;
594
        }
595
596
597

        // Check for user termination and stop the event loop:
        if(m_terminate) {
598
            break;
599
        }
600
601
602
    }
}

603
void ModuleManager::terminate() {
604
605
606
    m_terminate = true;
}

607
// Initalise all modules
608
void ModuleManager::initialiseAll() {
609
610
611
    // Loop over all modules and initialise them
    LOG(STATUS) << "=================| Initialising modules |==================";
    for(auto& module : m_modules) {
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
        // Create main ROOT directory for this module class if it does not exists yet
        LOG(TRACE) << "Creating and accessing ROOT directory";
        std::string module_name = module->getConfig().getName();
        auto directory = m_histogramFile->GetDirectory(module_name.c_str());
        if(directory == nullptr) {
            directory = m_histogramFile->mkdir(module_name.c_str());
            if(directory == nullptr) {
                throw RuntimeError("Cannot create or access overall ROOT directory for module " + module_name);
            }
        }
        directory->cd();

        // Create local directory for this instance
        TDirectory* local_directory = nullptr;
        if(module->get_identifier().getIdentifier().empty()) {
            local_directory = directory;
        } else {
            local_directory = directory->mkdir(module->get_identifier().getIdentifier().c_str());
            if(local_directory == nullptr) {
                throw RuntimeError("Cannot create or access local ROOT directory for module " + module->getUniqueName());
            }
        }

        // Change to the directory and save it in the module
        local_directory->cd();
        module->set_ROOT_directory(local_directory);

639
640
641
        // Set init module section header
        std::string old_section_name = Log::getSection();
        std::string section_name = "I:";
Simon Spannagel's avatar
Simon Spannagel committed
642
        section_name += module->getUniqueName();
643
644
        Log::setSection(section_name);
        // Set module specific settings
Simon Spannagel's avatar
Simon Spannagel committed
645
        auto old_settings = set_module_before(module->getUniqueName(), module->getConfig());
646
647
        // Change to our ROOT directory
        module->getROOTDirectory()->cd();
648

649
        LOG_PROGRESS(STATUS, "MOD_INIT_LOOP") << "Initialising \"" << module->getUniqueName() << "\"";
650
651
        // Initialise the module
        module->initialise();
652
653
654

        // Reset logging
        Log::setSection(old_section_name);
655
        set_module_after(old_settings);
Simon Spannagel's avatar
Simon Spannagel committed
656
    }
657
658
}

659
// Finalise all modules
660
void ModuleManager::finaliseAll() {
Simon Spannagel's avatar
Simon Spannagel committed
661

662
663
664
    // Loop over all modules and finalise them
    LOG(STATUS) << "===================| Finalising modules |===================";
    for(auto& module : m_modules) {
665
666
        // Set init module section header
        std::string old_section_name = Log::getSection();
Simon Spannagel's avatar
Simon Spannagel committed
667
        std::string section_name = "F:";
Simon Spannagel's avatar
Simon Spannagel committed
668
        section_name += module->getUniqueName();
669
670
        Log::setSection(section_name);
        // Set module specific settings
Simon Spannagel's avatar
Simon Spannagel committed
671
        auto old_settings = set_module_before(module->getUniqueName(), module->getConfig());
672
673
        // Change to our ROOT directory
        module->getROOTDirectory()->cd();
674

675
676
        // Finalise the module
        module->finalise();
677

678
679
680
        // Store all ROOT objects:
        module->getROOTDirectory()->Write();

681
682
683
        // Remove the pointer to the ROOT directory after finalizing
        module->set_ROOT_directory(nullptr);

684
685
        // Reset logging
        Log::setSection(old_section_name);
686
        set_module_after(old_settings);
Simon Spannagel's avatar
Simon Spannagel committed
687
    }
Simon Spannagel's avatar
Simon Spannagel committed
688

Simon Spannagel's avatar
Simon Spannagel committed
689
690
    // Write the output histogram file
    m_histogramFile->Close();
691

692
    LOG(STATUS) << "Wrote histogram output file to " << global_config.getPath("histogramFile");
693

694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
    // Write out update detectors file:
    if(global_config.has("detectors_file_updated")) {
        std::string file_name = global_config.getPath("detectors_file_updated");
        // Check if the file exists
        std::ofstream file(file_name);
        if(!file) {
            throw ConfigFileUnavailableError(file_name);
        }

        ConfigReader final_detectors;
        for(auto& detector : detectors) {
            final_detectors.addConfiguration(detector->getConfiguration());
        }

        final_detectors.write(file);
        LOG(STATUS) << "Wrote updated detector configuration to " << file_name;
    }

Simon Spannagel's avatar
Simon Spannagel committed
712
713
    // Check the timing for all events
    timing();
714
715
}

716
// Display timing statistics for each module, over all events and per event
717
void ModuleManager::timing() {
718
    LOG(STATUS) << "===============| Wall-clock timing (seconds) |================";
719
    for(auto& module : m_modules) {
Simon Spannagel's avatar
Simon Spannagel committed
720
721
722
        auto identifier = module->get_identifier().getIdentifier();
        LOG(STATUS) << std::setw(25) << module->getConfig().getName() << (identifier.empty() ? "   " : " : ")
                    << std::setw(10) << identifier << "  --  " << std::fixed << std::setprecision(5)
723
724
                    << module_execution_time_[module] << " = " << std::setprecision(9)
                    << module_execution_time_[module] / m_events << " s/evt";
Simon Spannagel's avatar
Simon Spannagel committed
725
    }
726
    LOG(STATUS) << "==============================================================";
727
}
728
729

// Helper functions to set the module specific log settings if necessary
730
std::tuple<LogLevel, LogFormat> ModuleManager::set_module_before(const std::string&, const Configuration& config) {
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
    // Set new log level if necessary
    LogLevel prev_level = Log::getReportingLevel();
    if(config.has("log_level")) {
        std::string log_level_string = config.get<std::string>("log_level");
        std::transform(log_level_string.begin(), log_level_string.end(), log_level_string.begin(), ::toupper);
        try {
            LogLevel log_level = Log::getLevelFromString(log_level_string);
            if(log_level != prev_level) {
                LOG(TRACE) << "Local log level is set to " << log_level_string;
                Log::setReportingLevel(log_level);
            }
        } catch(std::invalid_argument& e) {
            throw InvalidValueError(config, "log_level", e.what());
        }
    }

    // Set new log format if necessary
    LogFormat prev_format = Log::getFormat();
    if(config.has("log_format")) {
        std::string log_format_string = config.get<std::string>("log_format");
        std::transform(log_format_string.begin(), log_format_string.end(), log_format_string.begin(), ::toupper);
        try {
            LogFormat log_format = Log::getFormatFromString(log_format_string);
            if(log_format != prev_format) {
                LOG(TRACE) << "Local log format is set to " << log_format_string;
                Log::setFormat(log_format);
            }
        } catch(std::invalid_argument& e) {
            throw InvalidValueError(config, "log_format", e.what());
        }
    }

    return std::make_tuple(prev_level, prev_format);
}
765
void ModuleManager::set_module_after(std::tuple<LogLevel, LogFormat> prev) {
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
    // Reset the previous log level
    LogLevel cur_level = Log::getReportingLevel();
    LogLevel old_level = std::get<0>(prev);
    if(cur_level != old_level) {
        Log::setReportingLevel(old_level);
        LOG(TRACE) << "Reset log level to global level of " << Log::getStringFromLevel(old_level);
    }

    // Reset the previous log format
    LogFormat cur_format = Log::getFormat();
    LogFormat old_format = std::get<1>(prev);
    if(cur_format != old_format) {
        Log::setFormat(old_format);
        LOG(TRACE) << "Reset log format to global level of " << Log::getStringFromFormat(old_format);
    }
}
782

783
void ModuleManager::add_units() {
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827

    LOG(TRACE) << "Adding physical units";

    // LENGTH
    Units::add("nm", 1e-6);
    Units::add("um", 1e-3);
    Units::add("mm", 1);
    Units::add("cm", 1e1);
    Units::add("dm", 1e2);
    Units::add("m", 1e3);
    Units::add("km", 1e6);

    // TIME
    Units::add("ps", 1e-3);
    Units::add("ns", 1);
    Units::add("us", 1e3);
    Units::add("ms", 1e6);
    Units::add("s", 1e9);

    // TEMPERATURE
    Units::add("K", 1);

    // ENERGY
    Units::add("eV", 1e-6);
    Units::add("keV", 1e-3);
    Units::add("MeV", 1);
    Units::add("GeV", 1e3);

    // CHARGE
    Units::add("e", 1);
    Units::add("ke", 1e3);
    Units::add("C", 1.6021766208e-19);

    // VOLTAGE
    // NOTE: fixed by above
    Units::add("V", 1e-6);
    Units::add("kV", 1e-3);

    // ANGLES
    // NOTE: these are fake units
    Units::add("deg", 0.01745329252);
    Units::add("rad", 1);
    Units::add("mrad", 1e-3);
}