Merge branch 'master' into unify_configkeys

doc/usermanual/chapters/framework.tex

@@ -87,6 +87,12 @@ More information can be found in Section~\ref{sec:logging_verbosity}.
 \item \parameter{log_file}: File where the log output should be written to in addition to printing to the standard output (usually the terminal).
 Another (additional) location to write to can be specified on the command line using the \texttt{-l} parameter (see Section~\ref{sec:executable}).
 \item \parameter{library_directories}: Additional directories to search for module libraries, before searching the default paths.
+\item \parameter{output_directory}: Directory to write all output files into.
+Subdirectories are created automatically for all module instantiations.
+This directory will also contain the \parameter{histogram_file} specified via the parameter described above.
+Defaults to the current working directory with the subdirectory \textit{output/} attached.
+\item \parameter{purge_output_directory}: Decides whether the content of an already existing output directory is deleted before a new run starts. Defaults to \texttt{false}, i.e. files are kept but will be overwritten by new files created by the framework.
+\item \parameter{deny_overwrite}: Forces the framework to abort the run and throw an exception when attempting to overwrite an existing file. Defaults to \texttt{false}, i.e. files are overwritten when requested. This setting is inherited by all modules, but can be overwritten in the configuration section of each of the modules.
 \end{itemize}
 
 \section{Modules and the Module Manager}

src/core/CMakeLists.txt

@@ -3,6 +3,7 @@ INCLUDE_DIRECTORIES(SYSTEM ${CORRYVRECKAN_DEPS_INCLUDE_DIRS})
 
 # Create core library
 ADD_LIBRARY(CorryvreckanCore SHARED
+    Corryvreckan.cpp
     detector/Detector.cpp
     utils/log.cpp
     utils/unit.cpp

src/core/Corryvreckan.cpp

+/** @file
+ *  @brief Implementation of interface to the core framework
+ *  @copyright Copyright (c) 2017 CERN and the Corryvreckan authors.
+ * This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
+ * In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
+ * Intergovernmental Organization or submit itself to any jurisdiction.
+ */
+
+#include "Corryvreckan.hpp"
+
+#include <chrono>
+#include <climits>
+#include <fstream>
+#include <memory>
+#include <stdexcept>
+#include <thread>
+#include <utility>
+
+#include <TROOT.h>
+#include <TRandom.h>
+#include <TStyle.h>
+#include <TSystem.h>
+
+#include "core/config/exceptions.h"
+#include "core/utils/file.h"
+#include "core/utils/log.h"
+#include "core/utils/unit.h"
+
+using namespace corryvreckan;
+
+/**
+ * This class will own the managers for the lifetime of the simulation. Will do early initialization:
+ * - Configure the special header sections.
+ * - Set the log level and log format as requested.
+ * - Load the detector configuration and parse it
+ */
+Corryvreckan::Corryvreckan(std::string config_file_name,
+                           const std::vector<std::string>& module_options,
+                           const std::vector<std::string>& detector_options)
+    : terminate_(false), has_run_(false), mod_mgr_(std::make_unique<ModuleManager>()) {
+
+    // Load the global configuration
+    conf_mgr_ = std::make_unique<ConfigManager>(std::move(config_file_name),
+                                                std::initializer_list<std::string>({"Corryvreckan", ""}),
+                                                std::initializer_list<std::string>({"Ignore"}));
+
+    // Load and apply the provided module options
+    conf_mgr_->loadModuleOptions(module_options);
+
+    // Load and apply the provided detector options
+    conf_mgr_->loadDetectorOptions(detector_options);
+
+    // Fetch the global configuration
+    Configuration& global_config = conf_mgr_->getGlobalConfiguration();
+
+    // Set the log level from config if not specified earlier
+    std::string log_level_string;
+    if(Log::getReportingLevel() == LogLevel::NONE) {
+        log_level_string = global_config.get<std::string>("log_level", "INFO");
+        std::transform(log_level_string.begin(), log_level_string.end(), log_level_string.begin(), ::toupper);
+        try {
+            LogLevel log_level = Log::getLevelFromString(log_level_string);
+            Log::setReportingLevel(log_level);
+        } catch(std::invalid_argument& e) {
+            LOG(ERROR) << "Log level \"" << log_level_string
+                       << "\" specified in the configuration is invalid, defaulting to INFO instead";
+            Log::setReportingLevel(LogLevel::INFO);
+        }
+    } else {
+        log_level_string = Log::getStringFromLevel(Log::getReportingLevel());
+    }
+
+    // Set the log format from config
+    std::string log_format_string = global_config.get<std::string>("log_format", "DEFAULT");
+    std::transform(log_format_string.begin(), log_format_string.end(), log_format_string.begin(), ::toupper);
+    try {
+        LogFormat log_format = Log::getFormatFromString(log_format_string);
+        Log::setFormat(log_format);
+    } catch(std::invalid_argument& e) {
+        LOG(ERROR) << "Log format \"" << log_format_string
+                   << "\" specified in the configuration is invalid, using DEFAULT instead";
+        Log::setFormat(LogFormat::DEFAULT);
+    }
+
+    // Open log file to write output to
+    if(global_config.has("log_file")) {
+        // NOTE: this stream should be available for the duration of the logging
+        log_file_.open(global_config.getPath("log_file"), std::ios_base::out | std::ios_base::trunc);
+        LOG(TRACE) << "Added log stream to file " << global_config.getPath("log_file");
+        Log::addStream(log_file_);
+    }
+
+    // Wait for the first detailed messages until level and format are properly set
+    LOG(TRACE) << "Global log level is set to " << log_level_string;
+    LOG(TRACE) << "Global log format is set to " << log_format_string;
+}
+
+/**
+ * Performs the initialization, including:
+ * - Determine and create the output directory
+ * - Include all the defined units
+ * - Load the modules from the configuration
+ */
+void Corryvreckan::load() {
+    LOG(TRACE) << "Loading Corryvreckan";
+
+    // Fetch the global configuration
+    Configuration& global_config = conf_mgr_->getGlobalConfiguration();
+
+    // Put welcome message and set version
+    LOG(STATUS) << "Welcome to Corryvreckan " << CORRYVRECKAN_PROJECT_VERSION;
+    global_config.set<std::string>("version", CORRYVRECKAN_PROJECT_VERSION);
+
+    // Get output directory
+    std::string directory = gSystem->pwd();
+    directory += "/output";
+    if(global_config.has("output_directory")) {
+        // Use config specified one if available
+        directory = global_config.getPath("output_directory");
+    }
+
+    // Use existing output directory if it exists
+    bool create_output_dir = true;
+    if(corryvreckan::path_is_directory(directory)) {
+        if(global_config.get<bool>("purge_output_directory", false)) {
+            LOG(DEBUG) << "Deleting previous output directory " << directory;
+            corryvreckan::remove_path(directory);
+        } else {
+            LOG(DEBUG) << "Output directory " << directory << " already exists";
+            create_output_dir = false;
+        }
+    }
+    // Create the output directory
+    try {
+        if(create_output_dir) {
+            LOG(DEBUG) << "Creating output directory " << directory;
+            corryvreckan::create_directories(directory);
+        }
+        // Change to the new/existing output directory
+        gSystem->ChangeDirectory(directory.c_str());
+    } catch(std::invalid_argument& e) {
+        LOG(ERROR) << "Cannot create output directory " << directory << ": " << e.what()
+                   << ". Using current directory instead.";
+    }
+
+    // Set the default units to use
+    add_units();
+
+    // Load the modules from the configuration
+    if(!terminate_) {
+        mod_mgr_->load(conf_mgr_.get());
+    } else {
+        LOG(INFO) << "Skip loading modules because termination is requested";
+    }
+}
+
+/**
+ * Runs the Module::init() method linearly for every module
+ */
+void Corryvreckan::init() {
+    if(!terminate_) {
+        LOG(TRACE) << "Initializing Corryvreckan";
+        mod_mgr_->initialiseAll();
+    } else {
+        LOG(INFO) << "Skip initializing modules because termination is requested";
+    }
+}
+/**
+ * Runs every modules Module::run() method linearly for the number of events
+ */
+void Corryvreckan::run() {
+    if(!terminate_) {
+        LOG(TRACE) << "Running Corryvreckan";
+        mod_mgr_->run();
+
+        // Set that we have run and want to finalize as well
+        has_run_ = true;
+    } else {
+        LOG(INFO) << "Skip running modules because termination is requested";
+    }
+}
+/**
+ * Runs all modules Module::finalize() method linearly for every module
+ */
+void Corryvreckan::finalize() {
+    if(has_run_) {
+        LOG(TRACE) << "Finalizing Corryvreckan";
+        mod_mgr_->finaliseAll();
+    } else {
+        LOG(INFO) << "Skip finalizing modules because no module did run";
+    }
+}
+
+/*
+ * This function can be called safely from any signal handler. Time between the request to terminate
+ * and the actual termination is not always negigible.
+ */
+void Corryvreckan::terminate() {
+    terminate_ = true;
+    mod_mgr_->terminate();
+}
+
+void Corryvreckan::add_units() {
+    LOG(TRACE) << "Adding physical units";
+
+    // LENGTH
+    Units::add("nm", 1e-6);
+    Units::add("um", 1e-3);
+    Units::add("mm", 1);
+    Units::add("cm", 1e1);
+    Units::add("dm", 1e2);
+    Units::add("m", 1e3);
+    Units::add("km", 1e6);
+
+    // TIME
+    Units::add("ps", 1e-3);
+    Units::add("ns", 1);
+    Units::add("us", 1e3);
+    Units::add("ms", 1e6);
+    Units::add("s", 1e9);
+
+    // TEMPERATURE
+    Units::add("K", 1);
+
+    // ENERGY
+    Units::add("eV", 1e-6);
+    Units::add("keV", 1e-3);
+    Units::add("MeV", 1);
+    Units::add("GeV", 1e3);
+
+    // CHARGE
+    Units::add("e", 1);
+    Units::add("ke", 1e3);
+    Units::add("fC", 1 / 1.6021766208e-4);
+    Units::add("C", 1 / 1.6021766208e-19);
+
+    // VOLTAGE
+    // NOTE: fixed by above
+    Units::add("V", 1e-6);
+    Units::add("kV", 1e-3);
+
+    // MAGNETIC FIELD
+    Units::add("T", 1e-3);
+    Units::add("mT", 1e-6);
+
+    // ANGLES
+    // NOTE: these are fake units
+    Units::add("deg", 0.01745329252);
+    Units::add("rad", 1);
+    Units::add("mrad", 1e-3);
+}

src/core/Corryvreckan.hpp

+/** @file
+ *  @brief Interface to the core framework
+ *  @copyright Copyright (c) 2017 CERN and the Corryvreckan authors.
+ * This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
+ * In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
+ * Intergovernmental Organization or submit itself to any jurisdiction.
+ */
+
+/**
+ * @defgroup Managers Managers
+ * @brief The global set of managers used in framework
+ */
+
+#ifndef CORRYVRECKAN_CORRYVRECKAN_H
+#define CORRYVRECKAN_CORRYVRECKAN_H
+
+#include <atomic>
+#include <fstream>
+#include <memory>
+#include <string>
+
+#include "config/ConfigManager.hpp"
+#include "module/ModuleManager.hpp"
+
+namespace corryvreckan {
+    /**
+     * @brief Provides the link between the core framework and the executable.
+     *
+     * Supply the path location the main configuration which should be provided to the executable. Hereafter this class
+     * should be used to load, initialize, run and finalize all the modules.
+     */
+
+    class Corryvreckan {
+    public:
+        /**
+         * @brief Constructs Corryvreckan and initialize all managers
+         * @param config_file_name Path of the main configuration file
+         * @param options List of extra configuration options
+         */
+        explicit Corryvreckan(std::string config_file_name,
+                              const std::vector<std::string>& module_options = std::vector<std::string>(),
+                              const std::vector<std::string>& detector_options = std::vector<std::string>());
+
+        /**
+         * @brief Load modules from the main configuration and construct them
+         * @warning Should be called after the \ref Corryvreckan() "constructor"
+         */
+        void load();
+
+        /**
+         * @brief Initialize all modules (pre-run)
+         * @warning Should be called after the \ref Corryvreckan::load "load function"
+         */
+        void init();
+
+        /**
+         * @brief Run all modules for the number of events (run)
+         * @warning Should be called after the \ref Corryvreckan::init "init function"
+         */
+        void run();
+
+        /**
+         * @brief Finalize all modules (post-run)
+         * @warning Should be called after the \ref Corryvreckan::run "run function"
+         */
+        void finalize();
+
+        /**
+         * @brief Request termination as early as possible without changing the standard flow
+         */
+        void terminate();
+
+    private:
+        /**
+         * @brief Sets the default unit conventions
+         */
+        void add_units();
+
+        /**
+         * @brief Set the default ROOT plot style
+         */
+        void set_style();
+
+        // Indicate the framework should terminate
+        std::atomic<bool> terminate_;
+        std::atomic<bool> has_run_;
+
+        // Log file if specified
+        std::ofstream log_file_;
+
+        // All managers in the framework
+        std::unique_ptr<ModuleManager> mod_mgr_;
+        std::unique_ptr<ConfigManager> conf_mgr_;
+    };
+} // namespace corryvreckan
+
+#endif /* CORRYVRECKAN_CORRYVRECKAN_H */

src/core/module/Module.cpp

@@ -7,6 +7,7 @@
  */
 
 #include "Module.hpp"
+#include "core/utils/file.h"
 
 using namespace corryvreckan;
 
@@ -38,6 +39,64 @@ ModuleIdentifier Module::get_identifier() const {
     return identifier_;
 }
 
+/**
+ * @throws ModuleError If the file cannot be accessed (or created if it did not yet exist)
+ * @throws InvalidModuleActionException If this method is called from the constructor with the global flag false
+ * @throws ModuleError If the file exists but the "deny_overwrite" flag is set to true (defaults to false)
+ * @warning A local path cannot be fetched from the constructor, because the instantiation logic has not finished yet
+ *
+ * The output path is automatically created if it does not exists. The path is always accessible if this functions returns.
+ * Obeys the "deny_overwrite" parameter of the module.
+ */
+std::string Module::createOutputFile(const std::string& path, bool global) {
+    std::string file;
+    if(global) {
+        file = m_config.get<std::string>("_global_dir", std::string());
+    } else {
+        file = m_config.get<std::string>("_output_dir", std::string());
+    }
+
+    // The file name will only be empty if this method is executed from the constructor
+    if(file.empty()) {
+        throw InvalidModuleActionException("Cannot access local output path in constructor");
+    }
+
+    try {
+        // Create all the required main directories
+        corryvreckan::create_directories(file);
+
+        // Add the file itself
+        file += "/";
+        file += path;
+
+        if(path_is_file(file)) {
+            auto global_overwrite = getConfigManager()->getGlobalConfiguration().get<bool>("deny_overwrite", false);
+            if(m_config.get<bool>("deny_overwrite", global_overwrite)) {
+                throw ModuleError("Overwriting of existing file " + file + " denied.");
+            }
+            LOG(WARNING) << "File " << file << " exists and will be overwritten.";
+            try {
+                corryvreckan::remove_file(file);
+            } catch(std::invalid_argument& e) {
+                throw ModuleError("Deleting file " + file + " failed: " + e.what());
+            }
+        }
+
+        // Open the file to check if it can be accessed
+        std::fstream file_stream(file, std::ios_base::out | std::ios_base::app);
+        if(!file_stream.good()) {
+            throw std::invalid_argument("file not accessible");
+        }
+
+        // Convert the file to an absolute path
+        file = get_canonical_path(file);
+    } catch(std::invalid_argument& e) {
+        throw ModuleError("Path " + file + " cannot be created");
+    }
+
+    return file;
+}
+
 /**
  * @throws InvalidModuleActionException If this method is called from the constructor or destructor
  * @warning Cannot be used from the constructor, because the instantiation logic has not finished yet
@@ -78,3 +137,21 @@ bool Module::has_detector(std::string name) {
     }
     return true;
 }
+
+Configuration& Module::get_configuration() {
+    return m_config;
+}
+
+/**
+ * @throws InvalidModuleActionException If this method is called from the constructor or destructor
+ * @warning This function technically allows to write to the configurations of other modules, but this should never be done
+ */
+ConfigManager* Module::getConfigManager() {
+    if(conf_manager_ == nullptr) {
+        throw InvalidModuleActionException("Cannot access the config manager in constructor or destructor.");
+    };
+    return conf_manager_;
+}
+void Module::set_config_manager(ConfigManager* conf_manager) {
+    conf_manager_ = conf_manager;
+}

src/core/module/Module.hpp

@@ -13,7 +13,7 @@
 
 #include "ModuleIdentifier.hpp"
 #include "core/clipboard/Clipboard.hpp"
-#include "core/config/Configuration.hpp"
+#include "core/config/ConfigManager.hpp"
 #include "core/detector/Detector.hpp"
 #include "exceptions.h"
 
@@ -92,6 +92,14 @@ namespace corryvreckan {
          */
         Configuration& getConfig() { return m_config; }
 
+        /**
+         * @brief Create and return an absolute path to be used for output from a relative path
+         * @param path Relative path to add after the main output directory
+         * @param global True if the global output directory should be used instead of the module-specific version
+         * @return Canonical path to an output file
+         */
+        std::string createOutputFile(const std::string& path, bool global = false);
+
         /**
          * @brief Initialise the module before the event sequence
          *
@@ -116,6 +124,12 @@ namespace corryvreckan {
          */
         virtual void finalise(){};
 
+        /**
+         * @brief Get the config manager object to allow to read the global and other module configurations
+         * @return Pointer to the config manager
+         */
+        ConfigManager* getConfigManager();
+
         /**
          * @brief Get ROOT directory which should be used to output histograms et cetera
          * @return ROOT directory for storage
@@ -123,7 +137,11 @@ namespace corryvreckan {
         TDirectory* getROOTDirectory() const;
 
     protected:
-        // Configuration of this module
+        /**
+         * @brief Get the module configuration for internal use
+         * @return Configuration of the module
+         */
+        Configuration& get_configuration();
         Configuration m_config;
 
         /**
@@ -172,6 +190,13 @@ namespace corryvreckan {
         ModuleIdentifier get_identifier() const;
         ModuleIdentifier identifier_;
 
+        /**
+         * @brief Set the link to the config manager
+         * @param conf_manager ConfigManager holding all relevant configurations
+         */
+        void set_config_manager(ConfigManager* config);
+        ConfigManager* conf_manager_{nullptr};
+
         /**
          * @brief Set the output ROOT directory for this module
          * @param directory ROOT directory for storage

src/core/module/ModuleManager.cpp

@@ -15,6 +15,7 @@
 
 // Local include files
 #include "ModuleManager.hpp"
+#include "core/utils/file.h"
 #include "core/utils/log.h"
 #include "exceptions.h"
 
@@ -32,77 +33,13 @@
 using namespace corryvreckan;
 
 // Default constructor
-ModuleManager::ModuleManager(std::string config_file_name,
-                             const std::vector<std::string>& module_options,
-                             const std::vector<std::string>& detector_options)
-    : m_reference(nullptr), m_terminate(false) {
-
-    LOG(TRACE) << "Loading Corryvreckan";
-
-    // Put welcome message
-    LOG(STATUS) << "Welcome to Corryvreckan " << CORRYVRECKAN_PROJECT_VERSION;
-
-    // Load the global configuration
-    conf_manager_ = std::make_unique<corryvreckan::ConfigManager>(std::move(config_file_name),
-                                                                  std::initializer_list<std::string>({"Corryvreckan", ""}),
-                                                                  std::initializer_list<std::string>({"Ignore"}));
-
-    // Load and apply the provided module options
-    conf_manager_->loadModuleOptions(module_options);
-
-    // Load and apply the provided detector options
-    conf_manager_->loadDetectorOptions(detector_options);
-
-    // Fetch the global configuration
-    Configuration& global_config = conf_manager_->getGlobalConfiguration();
-
-    // Set the log level from config if not specified earlier
-    std::string log_level_string;
-    if(Log::getReportingLevel() == LogLevel::NONE) {
-        log_level_string = global_config.get<std::string>("log_level", "INFO");
-        std::transform(log_level_string.begin(), log_level_string.end(), log_level_string.begin(), ::toupper);
-        try {
-            LogLevel log_level = Log::getLevelFromString(log_level_string);
-            Log::setReportingLevel(log_level);
-        } catch(std::invalid_argument& e) {
-            LOG(ERROR) << "Log level \"" << log_level_string
-                       << "\" specified in the configuration is invalid, defaulting to INFO instead";
-            Log::setReportingLevel(LogLevel::INFO);
-        }
-    } else {
-        log_level_string = Log::getStringFromLevel(Log::getReportingLevel());
-    }
-
-    // Set the log format from config
-    std::string log_format_string = global_config.get<std::string>("log_format", "DEFAULT");
-    std::transform(log_format_string.begin(), log_format_string.end(), log_format_string.begin(), ::toupper);
-    try {
-        LogFormat log_format = Log::getFormatFromString(log_format_string);
-        Log::setFormat(log_format);
-    } catch(std::invalid_argument& e) {
-        LOG(ERROR) << "Log format \"" << log_format_string
-                   << "\" specified in the configuration is invalid, using DEFAULT instead";
-        Log::setFormat(LogFormat::DEFAULT);
-    }
-
-    // Open log file to write output to
-    if(global_config.has("log_file")) {
-        // NOTE: this stream should be available for the duration of the logging
-        log_file_.open(global_config.getPath("log_file"), std::ios_base::out | std::ios_base::trunc);
-        Log::addStream(log_file_);
-    }
-
-    // Wait for the first detailed messages until level and format are properly set
-    LOG(TRACE) << "Global log level is set to " << log_level_string;
-    LOG(TRACE) << "Global log format is set to " << log_format_string;
-
+ModuleManager::ModuleManager() : m_reference(nullptr), m_terminate(false) {
     // New clipboard for storage:
     m_clipboard = std::make_shared<Clipboard>();
 }
 
-void ModuleManager::load() {
-
-    add_units();
+void ModuleManager::load(ConfigManager* conf_mgr) {
+    conf_manager_ = conf_mgr;