Skip to content

GitLab

Commit a7e16a3c authored by Simon Spannagel's avatar Simon Spannagel
Browse files

Update config parser with upstream Allpix Squared version

parent 17d58264
Hide whitespace changes
Inline Side-by-side
src/core/config/ConfigManager.cpp
View file @ a7e16a3c
......@@ -2,10 +2,8 @@
* @file
* @brief Implementation of config manager
* @copyright Copyright (c) 2017 CERN and the Allpix Squared authors.
* This software is distributed under the terms of the MIT License, copied
* verbatim in the file "LICENSE.md".
* In applying this license, CERN does not waive the privileges and immunities
* granted to it by virtue of its status as an
* This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
* In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
* Intergovernmental Organization or submit itself to any jurisdiction.
*/
......@@ -24,8 +22,7 @@
using namespace corryvreckan;
/**
* @throws ConfigFileUnavailableError If the main configuration file cannot be
* accessed
* @throws ConfigFileUnavailableError If the main configuration file cannot be accessed
*/
ConfigManager::ConfigManager(std::string file_name) : file_name_(std::move(file_name)) {
LOG(TRACE) << "Using " << file_name_ << " as main configuration file";
......@@ -39,6 +36,9 @@ ConfigManager::ConfigManager(std::string file_name) : file_name_(std::move(file_
// Convert main file to absolute path
file_name_ = corryvreckan::get_absolute_path(file_name_);
// Initialize global base configuration with absolute file name
global_base_config_ = Configuration("", file_name_);
// Read the file
reader_.add(file, file_name_);
}
......@@ -57,11 +57,14 @@ void ConfigManager::addGlobalHeaderName(std::string name) {
}
/**
* The global configuration is the combination of all sections with a global
* header.
* The global configuration is the combination of all sections with a global header.
*/
Configuration ConfigManager::getGlobalConfiguration() {
Configuration global_config(global_default_name_, file_name_);
// Copy base config and set name
Configuration global_config = global_base_config_;
global_config.setName(global_default_name_);
// Add all other global configuration
for(auto& global_name : global_names_) {
auto configs = reader_.getConfigurations(global_name);
for(auto& config : configs) {
......@@ -75,15 +78,48 @@ void ConfigManager::addIgnoreHeaderName(std::string name) {
ignore_names_.emplace(std::move(name));
}
/**
* Option is split in a key / value pair, an error is thrown if that is not possible. When the key contains at least one dot
* it is interpreted as a relative configuration with the module identified by the first dot. In that case the option is
* applied during module loading when either the unique or the configuration name match. Otherwise the key is interpreted as
* global key and is added to the global header.
*/
void ConfigManager::parseOption(std::string line) {
line = corryvreckan::trim(line);
auto key_value = ConfigReader::parseKeyValue(line);
auto key = key_value.first;
auto value = key_value.second;
auto dot_pos = key.find('.');
if(dot_pos == std::string::npos) {
// Global option, add to the global base config
global_base_config_.setText(key, value);
} else {
// Other identifier bound option is passed
auto identifier = key.substr(0, dot_pos);
key = key.substr(dot_pos + 1);
identifier_options_[identifier].push_back(std::make_pair(key, value));
}
}
bool ConfigManager::applyOptions(const std::string& identifier, Configuration& config) {
if(identifier_options_.find(identifier) == identifier_options_.end()) {
return false;
}
for(auto& key_value : identifier_options_[identifier]) {
config.setText(key_value.first, key_value.second);
}
return true;
}
bool ConfigManager::hasConfiguration(const std::string& name) {
return reader_.hasConfiguration(name);
}
/**
* All special global and ignored sections are removed before returning the rest
* of the configurations. The list of normal
* sections is used by the ModuleManager to instantiate all the required
* modules.
* All special global and ignored sections are removed before returning the rest of the configurations. The list of normal
* sections is used by the ModuleManager to instantiate all the required modules.
*/
std::vector<Configuration> ConfigManager::getConfigurations() const {
std::vector<Configuration> result;
......
src/core/config/ConfigManager.hpp
View file @ a7e16a3c
/**
* @file
* @brief Interface to the main configuration and its normal and special
* sections
* @brief Interface to the main configuration and its normal and special sections
* @copyright Copyright (c) 2017 CERN and the Allpix Squared authors.
* This software is distributed under the terms of the MIT License, copied
* verbatim in the file "LICENSE.md".
* In applying this license, CERN does not waive the privileges and immunities
* granted to it by virtue of its status as an
* This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
* In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
* Intergovernmental Organization or submit itself to any jurisdiction.
*/
......@@ -24,16 +21,12 @@ namespace corryvreckan {
/**
* @ingroup Managers
* @brief Manager responsible for loading and providing access to the main
* configuration
* @brief Manager responsible for loading and providing access to the main configuration
*
* The main configuration is the single most important source of configuration.
* It is split up in:
* - Global headers that are combined into a single global (not module specific)
* configuration
* The main configuration is the single most important source of configuration. It is split up in:
* - Global headers that are combined into a single global (not module specific) configuration
* - Ignored headers that are not used at all (mainly useful for debugging)
* - All other headers representing all modules that have to be instantiated by
* the ModuleManager
* - All other headers representing all modules that have to be instantiated by the ModuleManager
*
* Configuration sections are always case-sensitive.
*/
......@@ -91,6 +84,20 @@ namespace corryvreckan {
// TODO [doc] Rename to ignoreHeader
void addIgnoreHeaderName(std::string name);
/**
* @brief Parse an extra configuration option
* @param line Line with the option
*/
void parseOption(std::string line);
/**
* @brief Apply all relevant options to the passed configuration
* @param identifier Identifier to select the options to apply
* @param config Configuration option where the options should be applied to
* @return True if the configuration was changed because of applied options
*/
bool applyOptions(const std::string& identifier, Configuration& config);
/**
* @brief Return if section with given name exists
* @param name Name of the section
......@@ -109,9 +116,13 @@ namespace corryvreckan {
ConfigReader reader_;
std::string global_default_name_;
std::set<std::string> global_names_;
std::set<std::string> ignore_names_;
Configuration global_base_config_;
std::string global_default_name_{};
std::set<std::string> global_names_{};
std::set<std::string> ignore_names_{};
std::map<std::string, std::vector<std::pair<std::string, std::string>>> identifier_options_{};
};
} // namespace corryvreckan
......
src/core/config/ConfigReader.cpp
View file @ a7e16a3c
......@@ -2,10 +2,8 @@
* @file
* @brief Implementation of config reader
* @copyright Copyright (c) 2017 CERN and the Allpix Squared authors.
* This software is distributed under the terms of the MIT License, copied
* verbatim in the file "LICENSE.md".
* In applying this license, CERN does not waive the privileges and immunities
* granted to it by virtue of its status as an
* This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
* In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
* Intergovernmental Organization or submit itself to any jurisdiction.
*/
......@@ -45,11 +43,58 @@ void ConfigReader::copy_init_map() {
}
/**
* @throws ConfigParseError If an error occurred during the parsing of the
* stream
* @throws KeyValueParseError If the key / value pair could not be parsed
*
* The configuration is immediately parsed and all of its configurations are
* available after the functions returns.
* The key / value pair is split according to the format specifications
*/
std::pair<std::string, std::string> ConfigReader::parseKeyValue(std::string line) {
line = corryvreckan::trim(line);
size_t equals_pos = line.find('=');
if(equals_pos != std::string::npos) {
std::string key = trim(std::string(line, 0, equals_pos));
std::string value = trim(std::string(line, equals_pos + 1));
char last_quote = 0;
for(size_t i = 0; i < value.size(); ++i) {
if(value[i] == '\'' || value[i] == '\"') {
if(last_quote == 0) {
last_quote = value[i];
} else if(last_quote == value[i]) {
last_quote = 0;
}
}
if(last_quote == 0 && value[i] == '#') {
value = std::string(value, 0, i);
break;
}
}
// Check if key contains only alphanumeric or underscores
bool valid_key = true;
for(auto& ch : key) {
if(isalnum(ch) == 0 && ch != '_' && ch != '.' && ch != ':') {
valid_key = false;
break;
}
}
// Check if value is not empty and key is valid
if(!valid_key) {
throw KeyValueParseError(line, "key is not valid");
}
if(value.empty()) {
throw KeyValueParseError(line, "value is empty");
}
return std::make_pair(key, corryvreckan::trim(value));
}
// Key / value pair does not contain equal sign
throw KeyValueParseError(line, "missing equality sign to split key and value");
}
/**
* @throws ConfigParseError If an error occurred during the parsing of the stream
*
* The configuration is immediately parsed and all of its configurations are available after the functions returns.
*/
void ConfigReader::add(std::istream& stream, std::string file_name) {
LOG(TRACE) << "Parsing configuration file " << file_name;
......@@ -73,67 +118,54 @@ void ConfigReader::add(std::istream& stream, std::string file_name) {
std::getline(stream, line);
++line_num;
// Find equal sign
size_t equals_pos = line.find('=');
if(equals_pos == std::string::npos) {
line = corryvreckan::trim(line);
// Ignore empty lines or comments
if(line.empty() || line.front() == '#') {
continue;
}
// Ignore empty lines or comments
if(line == "" || line[0] == '#') {
continue;
// Check if section header or key-value pair
if(line.front() == '[') {
// Line should be a section header with an alphanumeric name
size_t idx = 1;
for(; idx < line.length() - 1; ++idx) {
if(isalnum(line[idx]) == 0) {
break;
}
}
// Parse new section
if(line[0] == '[' && line[line.length() - 1] == ']') {
std::string remain = corryvreckan::trim(line.substr(idx + 1));
if(line[idx] == ']' && (remain.empty() || remain.front() == '#')) {
// Ignore empty sections if they contain no configurations
if(!conf.getName().empty() || conf.countSettings() > 0) {
// Add previous section
addConfiguration(conf);
addConfiguration(std::move(conf));
}
// Begin new section
section_name = std::string(line, 1, line.length() - 2);
section_name = std::string(line, 1, idx - 1);
conf = Configuration(section_name, file_name);
} else {
// FIXME: should be a bit more helpful...
// Section header is not valid
throw ConfigParseError(file_name, line_num);
}
} else {
std::string key = trim(std::string(line, 0, equals_pos));
std::string value = trim(std::string(line, equals_pos + 1));
char ins = 0;
for(size_t i = 0; i < value.size(); ++i) {
if(value[i] == '\'' || value[i] == '\"') {
if(ins == 0) {
ins = value[i];
} else if(ins == value[i]) {
ins = 0;
}
}
if(ins == 0 && value[i] == '#') {
value = std::string(value, 0, i);
break;
}
} else if(isalpha(line.front()) != 0) {
// Line should be a key / value pair with an equal sign
try {
// Parse the key value pair
auto key_value = parseKeyValue(line);
// Add the config key
conf.setText(key_value.first, key_value.second);
} catch(KeyValueParseError& e) {
// Rethrow key / value parse error as a configuration parse error
throw ConfigParseError(file_name, line_num);
}
// Add the config key
conf.setText(key, trim(value));
} else {
// Line is not a comment, key/value pair or section header
throw ConfigParseError(file_name, line_num);
}
}
// Add last section
addConfiguration(conf);
}
void ConfigReader::write(std::ostream& stream) {
for(auto& conf : conf_array_) {
stream << "[" << conf.getName() << "]" << std::endl;
for(auto& key : conf.getAll()) {
stream << key.first << " = " << key.second << std::endl;
}
stream << std::endl;
}
addConfiguration(std::move(conf));
}
void ConfigReader::addConfiguration(Configuration config) {
......@@ -204,3 +236,13 @@ std::vector<Configuration> ConfigReader::getConfigurations(std::string name) con
std::vector<Configuration> ConfigReader::getConfigurations() const {
return std::vector<Configuration>(conf_array_.begin(), conf_array_.end());
}
void ConfigReader::write(std::ostream& stream) {
for(auto& conf : conf_array_) {
stream << "[" << conf.getName() << "]" << std::endl;
for(auto& key : conf.getAll()) {
stream << key.first << " = " << key.second << std::endl;
}
stream << std::endl;
}
}
src/core/config/ConfigReader.hpp
View file @ a7e16a3c
......@@ -2,10 +2,8 @@
* @file
* @brief Provides a reader for configuration files
* @copyright Copyright (c) 2017 CERN and the Allpix Squared authors.
* This software is distributed under the terms of the MIT License, copied
* verbatim in the file "LICENSE.md".
* In applying this license, CERN does not waive the privileges and immunities
* granted to it by virtue of its status as an
* This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
* In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
* Intergovernmental Organization or submit itself to any jurisdiction.
*/
......@@ -25,11 +23,9 @@ namespace corryvreckan {
/**
* @brief Reader of configuration files
*
* Read the internal configuration file format used in the framework. The format
* contains
* Read the internal configuration file format used in the framework. The format contains
* - A set of section header between [ and ] brackets
* - Key/value pairs linked to the last defined section (or the empty section if
* none has been defined yet)
* - Key/value pairs linked to the last defined section (or the empty section if none has been defined yet)
*/
class ConfigReader {
public:
......@@ -40,24 +36,23 @@ namespace corryvreckan {
/**
* @brief Constructs a config reader with a single attached stream
* @param stream Stream to read configuration from
* @param file_name Name of the file related to the stream or empty if not
* linked to a file
* @param file_name Name of the file related to the stream or empty if not linked to a file
*/
explicit ConfigReader(std::istream& stream, std::string file_name = "");
/**
* @brief Adds a configuration stream to read
* @param stream Stream to read configuration from
* @param file_name Name of the file related to the stream or empty if not
* linked to a file
* @brief Parse a line as key-value pair
* @param line Line to interpret
* @return Pair of the key and the value
*/
void add(std::istream&, std::string file_name = "");
static std::pair<std::string, std::string> parseKeyValue(std::string line);
/**
* @brief Writes all configurations held to the stream provided
* @param stream Stream to write configuration to
* @brief Adds a configuration stream to read
* @param stream Stream to read configuration from
* @param file_name Name of the file related to the stream or empty if not linked to a file
*/
void write(std::ostream& stream);
void add(std::istream&, std::string file_name = "");
/**
* @brief Directly add a configuration object to the reader
......@@ -89,8 +84,7 @@ namespace corryvreckan {
/**
* @brief Check if a configuration exists
* @param name Name of a configuration header to search for
* @return True if at least a single configuration with this name exists,
* false otherwise
* @return True if at least a single configuration with this name exists, false otherwise
*/
bool hasConfiguration(std::string name) const;
/**
......@@ -101,8 +95,7 @@ namespace corryvreckan {
unsigned int countConfigurations(std::string name) const;
/**
* @brief Get cmobined configuration of all empty sections (usually the
* header)
* @brief Get cmobined configuration of all empty sections (usually the header)
* @note Typically this is only the section at the top of the file
* @return Configuration object for the empty section
*/
......@@ -121,6 +114,12 @@ namespace corryvreckan {
*/
std::vector<Configuration> getConfigurations() const;
/**
* @brief Writes all configurations held to the stream provided
* @param stream Stream to write configuration to
*/
void write(std::ostream& stream);
private:
/**
* @brief Initialize the configuration map after copy of the class
......
src/core/config/Configuration.cpp
View file @ a7e16a3c
......@@ -2,10 +2,8 @@
* @file
* @brief Implementation of configuration
* @copyright Copyright (c) 2017 CERN and the Allpix Squared authors.
* This software is distributed under the terms of the MIT License, copied
* verbatim in the file "LICENSE.md".
* In applying this license, CERN does not waive the privileges and immunities
* granted to it by virtue of its status as an
* This software is distributed under the terms of the MIT License, copied verbatim in the file "LICENSE.md".
* In applying this license, CERN does not waive the privileges and immunities granted to it by virtue of its status as an
* Intergovernmental Organization or submit itself to any jurisdiction.
*/
......@@ -19,8 +17,6 @@
#include "core/utils/file.h"
#include "exceptions.h"
#include <iostream>
using namespace corryvreckan;
Configuration::Configuration(std::string name, std::string path) : name_(std::move(name)), path_(std::move(path)) {}
......@@ -32,6 +28,9 @@ bool Configuration::has(const std::string& key) const {
std::string Configuration::getName() const {
return name_;
}
void Configuration::setName(const std::string& name) {
name_ = name;
}
std::string Configuration::getFilePath() const {
return path_;
}
......@@ -52,11 +51,9 @@ std::string Configuration::getText(const std::string& key, const std::string& de
}
/**
* @throws InvalidValueError If the path did not exists while the check_exists
* parameter is given
* @throws InvalidValueError If the path did not exists while the check_exists parameter is given
*
* For a relative path the absolute path of the configuration file is
* preprended. Absolute paths are not changed.
* For a relative path the absolute path of the configuration file is preprended. Absolute paths are not changed.
*/
// TODO [doc] Document canonicalizing behaviour
std::string Configuration::getPath(const std::string& key, bool check_exists) const {
......@@ -67,11 +64,9 @@ std::string Configuration::getPath(const std::string& key, bool check_exists) co
}
}
/**
* @throws InvalidValueError If the path did not exists while the check_exists
* parameter is given
* @throws InvalidValueError If the path did not exists while the check_exists parameter is given
*
* For all relative paths the absolute path of the configuration file is
* preprended. Absolute paths are not changed.
* For all relative paths the absolute path of the configuration file is preprended. Absolute paths are not changed.
*/
// TODO [doc] Document canonicalizing behaviour
std::vector<std::string> Configuration::getPathArray(const std::string& key, bool check_exists) const {
......@@ -92,7 +87,7 @@ std::vector<std::string> Configuration::getPathArray(const std::string& key, boo
*/
std::string Configuration::path_to_absolute(std::string path, bool canonicalize_path) const {
// If not a absolute path, make it an absolute path
if(path[0] != '/') {
if(path.front() != '/') {
// Get base directory of config file
std::string directory = path_.substr(0, path_.find_last_of('/'));
......@@ -131,8 +126,7 @@ unsigned int Configuration::countSettings() const {
}
/**
* All keys that are already defined earlier in this configuration are not
* changed.
* All keys that are already defined earlier in this configuration are not changed.
*/
void Configuration::merge(const Configuration& other) {
for(auto config_pair : other.config_) {
......@@ -149,7 +143,7 @@ std::vector<std::pair<std::string, std::string>> Configuration::getAll() {
// Loop over all configuration keys
for(auto& key_value : config_) {
// Skip internal keys starting with an underscore
if(!key_value.first.empty() && key_value.first[0] == '_') {
if(!key_value.first.empty() && key_value.first.front() == '_') {
continue;
}
......@@ -158,3 +152,88 @@ std::vector<std::pair<std::string, std::string>> Configuration::getAll() {
return result;
}
/**
* String is recursively parsed for all pair of [ and ] brackets. All parts between single or double quotation marks are
* skipped.
*/
std::unique_ptr<Configuration::parse_node> Configuration::parse_value(std::string str, int depth) {
using parse_node = Configuration::parse_node;
auto node = std::make_unique<parse_node>();
str = corryvreckan::trim(str);
if(str.empty()) {
throw std::invalid_argument("element is empty");
}
// Initialize variables for non-zero levels
size_t beg = 1, lst = 1;
int in_dpt = 0;
bool in_dpt_chg = false;
// Implicitly add pair of brackets on zero level
if(depth == 0) {
beg = lst = 0;
in_dpt = 1;
}
for(size_t i = 0; i < str.size(); ++i) {
// Skip over quotation marks
if(str[i] == '\'' || str[i] == '\"') {
i = str.find(str[i], i + 1);
if(i == std::string::npos) {
throw std::invalid_argument("quotes are not balanced");
}
continue;
}
// Handle brackets
if(str[i] == '[') {
++in_dpt;
if(!in_dpt_chg && i != 0) {
throw std::invalid_argument("invalid start bracket");
}
in_dpt_chg = true;
} else if(str[i] == ']') {
if(in_dpt == 0) {
throw std::invalid_argument("brackets are not matched");
}
--in_dpt;
in_dpt_chg = true;
}
// Make subitems at the zero level
if(in_dpt == 1 && (str[i] == ',' || (isspace(str[i]) != 0 && (isspace(str[i - 1]) == 0 && str[i - 1] != ',')))) {
node->children.push_back(parse_value(str.substr(lst, i - lst), depth + 1));
lst = i + 1;
}
}
if((depth > 0 && in_dpt != 0) || (depth == 0 && in_dpt != 1)) {
throw std::invalid_argument("brackets are not balanced");
}
// Determine if array or value
if(in_dpt_chg || depth == 0) {
// Handle last array item
size_t end = str.size();
if(depth != 0) {
if(str.back() !=