Commit a4613660 authored by Simon Spannagel's avatar Simon Spannagel
Browse files

Merge branch 'master' into docs

parents f6a14764 283449ca
Pipeline #1109829 passed with stages
in 17 minutes and 24 seconds
...@@ -76,6 +76,28 @@ cmp:cc7-gcc: ...@@ -76,6 +76,28 @@ cmp:cc7-gcc:
- lib - lib
expire_in: 3 hour expire_in: 3 hour
cmp:cc7-docker:
stage: compilation
tags:
- docker
image:
name: gitlab-registry.cern.ch/corryvreckan/corryvreckan/corryvreckan-deps
entrypoint: [""]
before_script:
- source scl_source enable devtoolset-7 || echo " "
script:
- mkdir build
- cd build
- cmake3 -DBUILD_EventLoaderEUDAQ2=ON -DCMAKE_BUILD_TYPE=RELEASE -DROOT_DIR=$ROOTSYS ..
- make
- make install
artifacts:
paths:
- build
- bin
- lib
expire_in: 3 hour
cmp:cc7-llvm: cmp:cc7-llvm:
stage: compilation stage: compilation
tags: tags:
...@@ -124,10 +146,12 @@ cmp:mac1014-clang: ...@@ -124,10 +146,12 @@ cmp:mac1014-clang:
tags: tags:
- docker - docker
dependencies: dependencies:
- cmp:cc7-gcc - cmp:cc7-docker
image: clicdp/cc7-base image:
name: gitlab-registry.cern.ch/corryvreckan/corryvreckan/corryvreckan-deps
entrypoint: [""]
before_script: before_script:
- source .gitlab-ci.d/init_x86_64.sh - source scl_source enable devtoolset-7 || echo " "
tst:telescope: tst:telescope:
extends: .test extends: .test
......
...@@ -145,13 +145,14 @@ INCLUDE("cmake/clang-cpp-checks.cmake") ...@@ -145,13 +145,14 @@ INCLUDE("cmake/clang-cpp-checks.cmake")
# Always include sources from top directory # Always include sources from top directory
INCLUDE_DIRECTORIES(src) INCLUDE_DIRECTORIES(src)
# Build objects library
ADD_SUBDIRECTORY(src/objects)
SET(CORRYVRECKAN_LIBRARIES ${CORRYVRECKAN_LIBRARIES} CorryvreckanObjects)
# Build core Corryvreckan library # Build core Corryvreckan library
ADD_SUBDIRECTORY(src/core) ADD_SUBDIRECTORY(src/core)
SET(CORRYVRECKAN_LIBRARIES ${CORRYVRECKAN_LIBRARIES} CorryvreckanCore) SET(CORRYVRECKAN_LIBRARIES ${CORRYVRECKAN_LIBRARIES} CorryvreckanCore)
# Build objects library
ADD_SUBDIRECTORY(src/objects)
SET(CORRYVRECKAN_LIBRARIES ${CORRYVRECKAN_LIBRARIES} CorryvreckanObjects)
# Build required modules # Build required modules
ADD_SUBDIRECTORY(src/modules) ADD_SUBDIRECTORY(src/modules)
......
...@@ -70,4 +70,6 @@ This software is distributed under the terms of the MIT license. A copy of this ...@@ -70,4 +70,6 @@ This software is distributed under the terms of the MIT license. A copy of this
The documentation is distributed under the terms of the CC-BY-4.0 license. This license can be found in [doc/COPYING.md](doc/COPYING.md). The documentation is distributed under the terms of the CC-BY-4.0 license. This license can be found in [doc/COPYING.md](doc/COPYING.md).
This project strongly profits from the developments done for the [Allpix Squared project](https://cern.ch/allpix-squared) which is released under the MIT license. Especially the configuration class, the module instantiation logic and the file reader and writer modules have profited heavily by their corresponding framework components in Allpix Squared.
The LaTeX and Pandoc CMake modules used by Corryvreckan are licensed under the BSD 3-Clause License. The LaTeX and Pandoc CMake modules used by Corryvreckan are licensed under the BSD 3-Clause License.
...@@ -119,6 +119,91 @@ my_switch = true ...@@ -119,6 +119,91 @@ my_switch = true
my_other_switch = 0 my_other_switch = 0
\end{minted} \end{minted}
\subsection{File format}
\label{sec:config_file_format}
Throughout the framework, a simplified version of TOML~\cite{tomlgit} is used as standard format for configuration files.
The format is defined as follows:
\begin{enumerate}
\item All whitespace at the beginning or end of a line are stripped by the parser.
In the rest of this format specification the \textit{line} refers to the line with this whitespace stripped.
\item Empty lines are ignored.
\item Every non-empty line should start with either \texttt{\#}, \texttt{[} or an alphanumeric character.
Every other character should lead to an immediate parse error.
\item If the line starts with a hash character (\texttt{\#}), it is interpreted as comment and all other content on the same line is ignored.
\item If the line starts with an open square bracket (\texttt{[}), it indicates a section header (also known as configuration header).
The line should contain a string with alphanumeric characters and underscores, indicating the header name, followed by a closing square bracket (\texttt{]}), to end the header.
After any number of ignored whitespace characters there could be a \texttt{\#} character.
If this is the case, the rest of the line is handled as specified in point~3.
Otherwise there should not be any other character (except the whitespace) on the line.
Any line that does not comply to these specifications should lead to an immediate parse error.
Multiple section headers with the same name are allowed.
All key-value pairs following this section header are part of this section until a new section header is started.
\item If the line starts with an alphanumeric character, the line should indicate a key-value pair.
The beginning of the line should contain a string of alphabetic characters, numbers, dots (\texttt{.}), colons (\texttt{\:}) and underscores (\texttt{\_}), but it may only start with an alphanumeric character.
This string indicates the 'key'.
After an optional number of ignored whitespace, the key should be followed by an equality sign (\texttt{$=$}).
Any text between the \texttt{$=$} and the first \texttt{\#} character not enclosed within a pair of single or double quotes (\texttt{'} or \texttt{"}) is known as the non-stripped string.
Any character after the \texttt{\#} is handled as specified in point 3.
If the line does not contain any non-enclosed \texttt{\#} character, the value ends at the end of the line instead.
The 'value' of the key-value pair is the non-stripped string with all whitespace in front and at the end stripped.
The value may not be empty.
Any line that does not comply to these specifications should lead to an immediate parse error.
\item The value may consist of multiple nested dimensions which are grouped by pairs of square brackets (\texttt{[} and \texttt{]}).
The number of square brackets should be properly balanced, otherwise an error is raised.
Square brackets which should not be used for grouping should be enclosed in quotation marks.
Every dimension is split at every whitespace sequence and comma character (\texttt{,}) not enclosed in quotation marks.
Implicit square brackets are added to the begin and end of the value, if these are not explicitly added.
A few situations require explicit addition of outer brackets such as matrices with only one column element, i.e. with dimension 1xN.
\item The sections of the value which are interpreted as separate entities are named elements.
For a single value the element is on the zeroth dimension, for arrays on the first dimension and for matrices on the second dimension.
Elements can be forced by using quotation marks, either single or double quotes (\texttt{'} or \texttt{"}).
The number of both types of quotation marks should be properly balanced, otherwise an error is raised.
The conversion to the elements to the actual type is performed when accessing the value.
\item All key-value pairs defined before the first section header are part of a zero-length empty section header.
\end{enumerate}
\subsection{Accessing parameters}
\label{sec:accessing_parameters}
Values are accessed via the configuration object.
In the following example, the key is a string called \parameter{key}, the object is named \parameter{config} and the type \parameter{TYPE} is a valid C++ type the value should represent.
The values can be accessed via the following methods:
\begin{minted}[frame=single,framesep=3pt,breaklines=true,tabsize=2,linenos]{c++}
// Returns true if the key exists and false otherwise
config.has("key")
// Returns the number of keys found from the provided initializer list:
config.count({"key1", "key2", "key3"});
// Returns the value in the given type, throws an exception if not existing or a conversion to TYPE is not possible
config.get<TYPE>("key")
// Returns the value in the given type or the provided default value if it does not exist
config.get<TYPE>("key", default_value)
// Returns an array of elements of the given type
config.getArray<TYPE>("key")
// Returns a matrix: an array of arrays of elements of the given type
config.getMatrix<TYPE>("key")
// Returns an absolute (canonical if it should exist) path to a file
config.getPath("key", true /* check if path exists */)
// Return an array of absolute paths
config.getPathArray("key", false /* do not check if paths exists */)
// Returns the value as literal text including possible quotation marks
config.getText("key")
// Set the value of key to the default value if the key is not defined
config.setDefault("key", default_value)
// Set the value of the key to the default array if key is not defined
config.setDefaultArray<TYPE>("key", vector_of_default_values)
// Create an alias named new_key for the already existing old_key or throws an exception if the old_key does not exist
config.setAlias("new_key", "old_key")
\end{minted}
Conversions to the requested type are using the \parameter{from_string} and \parameter{to_string} methods provided by the framework string utility library.
These conversions largely follow standard C++ parsing, with one important exception.
If (and only if) the value is retrieved as a C/C++ string and the string is fully enclosed by a pair of \texttt{"} characters, these are stripped before returning the value.
Strings can thus also be provided with or without quotation marks.
\begin{warning}
It should be noted that a conversion from string to the requested type is a comparatively heavy operation.
For performance-critical sections of the code, one should consider fetching the configuration value once and caching it in a local variable.
\end{warning}
\section{Main configuration} \section{Main configuration}
\label{sec:main_config} \label{sec:main_config}
The main configuration consists of a set of sections specifying the modules used. The main configuration consists of a set of sections specifying the modules used.
......
...@@ -147,3 +147,12 @@ keywords = "Simulation, Silicon detectors, Geant4, TCAD, Drift–diffusion" ...@@ -147,3 +147,12 @@ keywords = "Simulation, Silicon detectors, Geant4, TCAD, Drift–diffusion"
institution = {CERN}, institution = {CERN},
url = "https://cern.ch/allpix-squared/usermanual/allpix-manual.pdf", url = "https://cern.ch/allpix-squared/usermanual/allpix-manual.pdf",
} }
@online{tomlgit,
title = {TOML},
subtitle = {Tom's Obvious, Minimal Language},
author = {Tom Preston-Werner},
url = {https://github.com/toml-lang/toml},
publisher = {Github},
journal = {Github repository},
commit = {e5d623ecdc26327699157381bde3ccd6ed8c67de}
}
...@@ -11,7 +11,8 @@ WORKDIR "/data" ...@@ -11,7 +11,8 @@ WORKDIR "/data"
RUN source scl_source enable devtoolset-7 && \ RUN source scl_source enable devtoolset-7 && \
mkdir -p /opt/corryvreckan/build && \ mkdir -p /opt/corryvreckan/build && \
cd /opt/corryvreckan/build && \ cd /opt/corryvreckan/build && \
cmake3 -DCMAKE_INSTALL_PREFIX=../ \ cmake3 -DBUILD_EventLoaderEUDAQ2=ON \
-DCMAKE_INSTALL_PREFIX=../ \
-DCMAKE_MODULE_PATH="/usr/share/cmake3/Modules/;/usr/share/cmake/Modules/" \ -DCMAKE_MODULE_PATH="/usr/share/cmake3/Modules/;/usr/share/cmake/Modules/" \
.. && \ .. && \
make -j`grep -c processor /proc/cpuinfo` && \ make -j`grep -c processor /proc/cpuinfo` && \
......
...@@ -31,3 +31,52 @@ RUN source scl_source enable devtoolset-7 && \ ...@@ -31,3 +31,52 @@ RUN source scl_source enable devtoolset-7 && \
make -j`grep -c processor /proc/cpuinfo` && \ make -j`grep -c processor /proc/cpuinfo` && \
make install && \ make install && \
rm -rf ${ROOTSYS}/{src,build} rm -rf ${ROOTSYS}/{src,build}
ENV PEARY_VERSION v0.9.7
# Add layer for Caribou Peary
ENV PEARYPATH="/opt/peary"
RUN source scl_source enable devtoolset-7 && \
yum install -y readline-devel && \
mkdir -p ${PEARYPATH}/{src,build} && \
curl -L -o ${PEARYPATH}/peary-${PEARY_VERSION}.zip \
https://gitlab.cern.ch/Caribou/peary/-/archive/${PEARY_VERSION}/peary-${PEARY_VERSION}.zip && \
unzip ${PEARYPATH}/peary-${PEARY_VERSION}.zip -d ${PEARYPATH}/src && \
rm -f ${PEARYPATH}/peary-${PEARY_VERSION}.zip && \
cd ${PEARYPATH}/build && \
cmake3 -DINTERFACE_I2C=OFF \
-DINTERFACE_SPI=OFF \
-DINTERFACE_SPI_CLICpix2=OFF \
-DINTERFACE_MEM=OFF \
-DINSTALL_HEADERS=ON \
-DINSTALL_PREFIX=../ \
../src/peary-${PEARY_VERSION} && \
make -j`grep -c processor /proc/cpuinfo` && \
make install && \
rm -rf ${PEARYPATH}/{src,build}
ENV EUDAQ2_VERSION 2.3.0
# Add layer for EUDAQ2
ENV EUDAQ2PATH="/opt/eudaq2"
RUN source scl_source enable devtoolset-7 && \
mkdir -p ${EUDAQ2PATH}/{src,build} && \
curl -L -o ${EUDAQ2PATH}/eudaq2.${EUDAQ2_VERSION}.zip \
https://github.com/eudaq/eudaq/archive/v${EUDAQ2_VERSION}.zip && \
unzip ${EUDAQ2PATH}/eudaq2.${EUDAQ2_VERSION}.zip -d ${EUDAQ2PATH}/src && \
rm -f ${EUDAQ2PATH}/eudaq2.${EUDAQ2_VERSION}.zip && \
cd ${EUDAQ2PATH}/build && \
cmake3 -DEUDAQ_BUILD_EXECUTABLE=OFF \
-DEUDAQ_BUILD_GUI=OFF \
-DUSER_CARIBOU_BUILD=ON \
-DPEARYLIBS=${PEARYPATH}/lib \
-DPEARYINCLUDE=${PEARYPATH}/include/peary \
-DUSER_EUDET_BUILD=ON \
-DUSER_TLU_BUILD=ON \
-DEUDAQ_INSTALL_PREFIX=../ \
../src/eudaq-${EUDAQ2_VERSION} && \
make -j`grep -c processor /proc/cpuinfo` && \
make install && \
rm -rf ${EUDAQ2PATH}/{src,build}
#include "Clipboard.hpp" #include "Clipboard.hpp"
#include "core/utils/log.h"
#include "exceptions.h" #include "exceptions.h"
#include "objects/Object.hpp" #include "objects/Object.hpp"
using namespace corryvreckan; using namespace corryvreckan;
void Clipboard::put(std::string name, Objects* objects) { void Clipboard::putPersistentData(std::string name, double value) {
m_data.insert(ClipboardData::value_type(name, objects));
}
void Clipboard::put(std::string name, std::string type, Objects* objects) {
m_data.insert(ClipboardData::value_type(name + type, objects));
}
void Clipboard::put_persistent(std::string name, double value) {
m_persistent_data[name] = value; m_persistent_data[name] = value;
} }
Objects* Clipboard::get(std::string name, std::string type) { double Clipboard::getPersistentData(std::string name) const {
if(m_data.count(name + type) == 0) {
return nullptr;
}
return m_data[name + type];
}
double Clipboard::get_persistent(std::string name) const {
try { try {
return m_persistent_data.at(name); return m_persistent_data.at(name);
} catch(std::out_of_range&) { } catch(std::out_of_range&) {
...@@ -32,11 +16,15 @@ double Clipboard::get_persistent(std::string name) const { ...@@ -32,11 +16,15 @@ double Clipboard::get_persistent(std::string name) const {
} }
} }
bool Clipboard::event_defined() const { bool Clipboard::hasPersistentData(std::string name) const {
return m_persistent_data.find(name) != m_persistent_data.end();
}
bool Clipboard::isEventDefined() const {
return (m_event != nullptr); return (m_event != nullptr);
} }
void Clipboard::put_event(std::shared_ptr<Event> event) { void Clipboard::putEvent(std::shared_ptr<Event> event) {
// Already defined: // Already defined:
if(m_event) { if(m_event) {
throw InvalidDataError("Event already defined. Only one module can place an event definition"); throw InvalidDataError("Event already defined. Only one module can place an event definition");
...@@ -45,25 +33,33 @@ void Clipboard::put_event(std::shared_ptr<Event> event) { ...@@ -45,25 +33,33 @@ void Clipboard::put_event(std::shared_ptr<Event> event) {
} }
} }
std::shared_ptr<Event> Clipboard::get_event() const { std::shared_ptr<Event> Clipboard::getEvent() const {
if(!m_event) { if(!m_event) {
throw InvalidDataError("Event not defined. Add Metronome module or Event reader defining the event"); throw InvalidDataError("Event not defined. Add Metronome module or Event reader defining the event");
} }
return m_event; return m_event;
} }
bool Clipboard::has_persistent(std::string name) const {
return m_persistent_data.find(name) != m_persistent_data.end();
}
void Clipboard::clear() { void Clipboard::clear() {
for(auto set = m_data.cbegin(); set != m_data.cend();) { // Loop over all data types
Objects* collection = set->second; for(auto block = m_data.cbegin(); block != m_data.cend();) {
for(Objects::iterator it = collection->begin(); it != collection->end(); ++it) { auto collections = block->second;
delete(*it);
// Loop over all stored collections of this type
for(auto set = collections.cbegin(); set != collections.cend();) {
std::shared_ptr<ObjectVector> collection = std::static_pointer_cast<ObjectVector>(set->second);
// Loop over all objects and delete them
for(ObjectVector::iterator it = collection->begin(); it != collection->end(); ++it) {
// All objects are destroyed together in this clear function at the end of the event. To avoid costly
// reverse-iterations through the TRef dependency hash lists, we just tell ROOT not to care about possible
// TRef-dependants and to just destroy the object directly by resetting the `kMustCleanup` bit.
(*it)->ResetBit(kMustCleanup);
// Delete the object itself:
delete(*it);
}
set = collections.erase(set);
} }
delete collection; block = m_data.erase(block);
set = m_data.erase(set);
} }
// Resetting the event definition: // Resetting the event definition:
...@@ -72,8 +68,20 @@ void Clipboard::clear() { ...@@ -72,8 +68,20 @@ void Clipboard::clear() {
std::vector<std::string> Clipboard::listCollections() const { std::vector<std::string> Clipboard::listCollections() const {
std::vector<std::string> collections; std::vector<std::string> collections;
for(auto& dataset : m_data) {
collections.push_back(dataset.first); for(const auto& block : m_data) {
std::string line(corryvreckan::demangle(block.first.name()));
line += ": ";
for(const auto& set : block.second) {
std::shared_ptr<ObjectVector> collection = std::static_pointer_cast<ObjectVector>(set.second);
line += set.first + " (" + collection->size() + ") ";
}
line += "\n";
collections.push_back(line);
} }
return collections; return collections;
} }
const ClipboardData& Clipboard::getAll() const {
return m_data;
}
...@@ -13,6 +13,7 @@ ...@@ -13,6 +13,7 @@
#include <iostream> #include <iostream>
#include <memory> #include <memory>
#include <string> #include <string>
#include <typeindex>
#include <unordered_map> #include <unordered_map>
#include "core/utils/log.h" #include "core/utils/log.h"
...@@ -20,6 +21,7 @@ ...@@ -20,6 +21,7 @@
#include "objects/Object.hpp" #include "objects/Object.hpp"
namespace corryvreckan { namespace corryvreckan {
typedef std::map<std::type_index, std::map<std::string, std::shared_ptr<void>>> ClipboardData;
/** /**
* @brief Class for temporary data storage for exachange between modules * @brief Class for temporary data storage for exachange between modules
...@@ -32,6 +34,7 @@ namespace corryvreckan { ...@@ -32,6 +34,7 @@ namespace corryvreckan {
* information which should outlast a single event. This is dubbed the "persistent storage" * information which should outlast a single event. This is dubbed the "persistent storage"
*/ */
class Clipboard { class Clipboard {
friend class ModuleManager;
public: public:
/** /**
...@@ -44,54 +47,44 @@ namespace corryvreckan { ...@@ -44,54 +47,44 @@ namespace corryvreckan {
virtual ~Clipboard() {} virtual ~Clipboard() {}
/** /**
* @brief Add object to the clipboard * @brief Method to add a vector of objects to the clipboard
* @param name Name of the collection to be stored * @param objects Shared pointer to vector of objects to be stored
* @param objects vector of Objects to store * @param key Identifying key for this set of objects. Defaults to empty key
*/ */
void put(std::string name, Objects* objects); template <typename T> void putData(std::shared_ptr<std::vector<T*>> objects, const std::string& key = "");
/** /**
* @brief Add object to the clipboard * @brief Method to retrieve objects from the clipboard
* @param name Name of the collection to be stored * @param key Identifying key of objects to be fetched. Defaults to empty key
* @param type Type of the object collection to be stored
* @param Objects vector of Objects to store
*/ */
void put(std::string name, std::string type, Objects* objects); template <typename T> std::shared_ptr<std::vector<T*>> getData(const std::string& key = "") const;
/**
* @brief Retrieve objects from the clipboard
* @param name Name of the object collection to fetch
* @param type Type of objects to be retrieved
* @return Vector of Object pointers
*/
Objects* get(std::string name, std::string type = "");
/** /**
* @brief Check whether an event has been defined * @brief Check whether an event has been defined
* @return true if an event has been defined, false otherwise * @return true if an event has been defined, false otherwise
*/ */
bool event_defined() const; bool isEventDefined() const;
/** /**
* @brief Store the event object * @brief Store the event object
* @param event The event object to be stored * @param event The event object to be stored
* @thorws InvalidDataError in case an event exist already * @thorws InvalidDataError in case an event exist already
*/ */
void put_event(std::shared_ptr<Event> event); void putEvent(std::shared_ptr<Event> event);
/** /**
* @brief Retrieve the event object * @brief Retrieve the event object
* @returnShared pointer to the event * @returnShared pointer to the event
* @throws MissingDataError in case no event is available. * @throws MissingDataError in case no event is available.
*/ */
std::shared_ptr<Event> get_event() const; std::shared_ptr<Event> getEvent() const;
/** /**
* @brief Store or update variable on the persistent clipboard storage * @brief Store or update variable on the persistent clipboard storage
* @param name Name of the variable * @param name Name of the variable
* @param value Value to be stored * @param value Value to be stored
*/ */
void put_persistent(std::string name, double value); void putPersistentData(std::string name, double value);
/** /**
* @brief Retrieve variable from the persistent clipboard storage * @brief Retrieve variable from the persistent clipboard storage
...@@ -99,19 +92,14 @@ namespace corryvreckan { ...@@ -99,19 +92,14 @@ namespace corryvreckan {
* @return Stored value from the persistent clipboard storage * @return Stored value from the persistent clipboard storage
* @throws MissingDataError in case the key is not found. * @throws MissingDataError in case the key is not found.
*/ */
double get_persistent(std::string name) const; double getPersistentData(std::string name) const;
/** /**
* @brief Check if variable exists on the persistent clipboard storage * @brief Check if variable exists on the persistent clipboard storage
* @param name Name of the variable * @param name Name of the variable
* @return True if value exists, false if it does not exist. * @return True if value exists, false if it does not exist.
*/ */
bool has_persistent(std::string name) const; bool hasPersistentData(std::string name) const;
/**
* @brief Clear the event storage of the clipboard
*/
void clear();
/** /**
* @brief Get a list of currently held collections on the clipboard event storage * @brief Get a list of currently held collections on the clipboard event storage
...@@ -119,8 +107,17 @@ namespace corryvreckan { ...@@ -119,8 +107,17 @@ namespace corryvreckan {
*/ */
std::vector<std::string> listCollections() const; std::vector<std::string> listCollections() const;
/**
* @brief Retrieve all currently stored clipboard data
* @return All clipboard data
*/
const ClipboardData& getAll() const;
private: private:
typedef std::map<std::string, Objects*> ClipboardData; /**
* @brief Clear the event storage of the clipboard
*/
void clear();
// Container for data, list of all data held // Container for data, list of all data held
ClipboardData m_data; ClipboardData m_data;
...@@ -133,4 +130,7 @@ namespace corryvreckan { ...@@ -133,4 +130,7 @@ namespace corryvreckan {
}; };
} // namespace corryvreckan } // namespace corryvreckan
// Include template members
#include "Clipboard.tpp"
#endif // CORRYVRECKAN_CLIPBOARD_H