Commit d9f74619 authored by Alexander Ferk's avatar Alexander Ferk
Browse files

reset changes to FileWriter

parent 66e53e99
FROM gitlab-registry.cern.ch/corryvreckan/corryvreckan/corryvreckan-base:latest
MAINTAINER Simon Spannagel <simon.spannagel@cern.ch>
ENV ROOT6_VERSION 6.20.04
ENV ROOT6_VERSION 6.10.08
# Add layer for ROOT6
ENV ROOTSYS="/opt/root6"
......
......@@ -6,7 +6,7 @@ CORRYVRECKAN_MODULE_SOURCES(${MODULE_NAME}
FileWriter.cpp
)
TARGET_LINK_LIBRARIES(${MODULE_NAME} ROOT::Tree ROOT::RIO)
TARGET_LINK_LIBRARIES(${MODULE_NAME} ROOT::Tree)
# Provide standard install target
CORRYVRECKAN_MODULE_INSTALL(${MODULE_NAME})
......@@ -38,41 +38,11 @@ FileWriter::~FileWriter() {
}
void FileWriter::initialise() {
// Output format
auto fileFormats = {
"root",
"txt",
"json",
}; // known output formats - place here or in header?
output_format_ = m_config.get<std::string>("format", "root");
if((std::find(std::begin(fileFormats), std::end(fileFormats), output_format_) == std::end(fileFormats))) {
throw InvalidValueError(m_config, "format", "unknown file format specified");
}
// Create output file
output_file_name_ = createOutputFile(
corryvreckan::add_file_extension(m_config.get<std::string>("file_name", "data"), output_format_), true);
if(output_format_ == "root") {
output_TFile_ = std::make_unique<TFile>(output_file_name_.c_str(), "RECREATE");
if(output_TFile_->IsZombie()) {
ModuleError("Creating output file failed, cannot continue");
}
output_TFile_->cd();
// Create event tree:
event_tree_ = std::make_unique<TTree>("Event", (std::string("Tree of Events").c_str()));
event_tree_->Bronch("global", "corryvreckan::Event", &event_);
} else if(output_format_ == "txt") {
output_file_ = std::make_unique<std::ofstream>(output_file_name_);
*output_file_ << "# Corryvreckan ASCII data" << std::endl << std::endl;
} else if(output_format_ == "json") {
output_file_ = std::make_unique<std::ofstream>(output_file_name_);
// print first line to create JSON-Array of objects
*output_file_ << "[" << std::endl;
} else {
throw InvalidValueError(m_config, "format", "specified format is not implemented");
}
output_file_name_ =
createOutputFile(corryvreckan::add_file_extension(m_config.get<std::string>("file_name", "data"), "root"), true);
output_file_ = std::make_unique<TFile>(output_file_name_.c_str(), "RECREATE");
output_file_->cd();
// Read include and exclude list
if(m_config.has("include") && m_config.has("exclude")) {
......@@ -84,7 +54,10 @@ void FileWriter::initialise() {
auto exc_arr = m_config.getArray<std::string>("exclude");
exclude_.insert(exc_arr.begin(), exc_arr.end());
}
m_eventNumber = 0;
// Create event tree:
event_tree_ = std::make_unique<TTree>("Event", (std::string("Tree of Events").c_str()));
event_tree_->Bronch("global", "corryvreckan::Event", &event_);
}
StatusCode FileWriter::run(std::shared_ptr<Clipboard> clipboard) {
......@@ -93,16 +66,10 @@ StatusCode FileWriter::run(std::shared_ptr<Clipboard> clipboard) {
ModuleError("No Clipboard event defined, cannot continue");
}
if(output_format_ == "root") {
// Read event from clipboard and write to tree:
event_ = clipboard->getEvent().get();
event_tree_->Fill();
write_cnt_++;
} else if(output_format_ == "txt") {
// Print the current event:
*output_file_ << "=== " << m_eventNumber << " ===" << std::endl;
*output_file_ << *clipboard->getEvent() << std::endl;
}
// Read event from clipboard and write to tree:
event_ = clipboard->getEvent().get();
event_tree_->Fill();
write_cnt_++;
auto data = clipboard->getAll();
LOG(DEBUG) << "Clipboard has " << data.size() << " different object types.";
......@@ -129,65 +96,51 @@ StatusCode FileWriter::run(std::shared_ptr<Clipboard> clipboard) {
detector_name = detector_block.first;
}
if(output_format_ == "root") {
auto objects = std::static_pointer_cast<ObjectVector>(detector_block.second);
LOG(TRACE) << " - " << detector_name << ": " << objects->size();
// Create a new branch of the correct type if this object has not been received before
auto index_tuple = std::make_tuple(type_idx, detector_name);
if(write_list_.find(index_tuple) == write_list_.end()) {
// Add vector of objects to write to the write list
write_list_[index_tuple] = new std::vector<Object*>();
auto addr = &write_list_[index_tuple];
auto new_tree = (trees_.find(class_name) == trees_.end());
if(new_tree) {
// Create new tree
output_TFile_->cd();
trees_.emplace(
class_name,
std::make_unique<TTree>(class_name.c_str(), (std::string("Tree of ") + class_name).c_str()));
}
auto objects = std::static_pointer_cast<ObjectVector>(detector_block.second);
LOG(TRACE) << " - " << detector_name << ": " << objects->size();
std::string branch_name = detector_name.empty() ? "global" : detector_name;
trees_[class_name]->Bronch(
branch_name.c_str(), (std::string("std::vector<") + class_name_full + "*>").c_str(), addr);
if(new_tree) {
LOG(DEBUG) << "Pre-filling new tree of " << class_name << " with " << m_eventNumber
<< " empty events";
for(unsigned int i = 0; i < m_eventNumber; ++i) {
trees_[class_name]->Fill();
}
} else {
LOG(DEBUG) << "Pre-filling new branch " << branch_name << " of " << class_name << " with "
<< m_eventNumber << " empty events";
auto* branch = trees_[class_name]->GetBranch(branch_name.c_str());
for(unsigned int i = 0; i < m_eventNumber; ++i) {
branch->Fill();
}
}
}
// Create a new branch of the correct type if this object has not been received before
auto index_tuple = std::make_tuple(type_idx, detector_name);
if(write_list_.find(index_tuple) == write_list_.end()) {
// Fill the branch vector
for(auto& object : *objects) {
++write_cnt_;
write_list_[index_tuple]->push_back(object);
}
} else if(output_format_ == "txt") {
*output_file_ << "--- " << (detector_name.empty() ? "<global>" : detector_name) << " ---" << std::endl;
auto objects = std::static_pointer_cast<ObjectVector>(detector_block.second);
for(auto& object : *objects) {
*output_file_ << *object << std::endl;
// Add vector of objects to write to the write list
write_list_[index_tuple] = new std::vector<Object*>();
auto addr = &write_list_[index_tuple];
auto new_tree = (trees_.find(class_name) == trees_.end());
if(new_tree) {
// Create new tree
output_file_->cd();
trees_.emplace(
class_name,
std::make_unique<TTree>(class_name.c_str(), (std::string("Tree of ") + class_name).c_str()));
}
} else if(output_format_ == "json") {
auto objects = std::static_pointer_cast<ObjectVector>(detector_block.second);
for(auto& object : *objects) {
*output_file_ << TBufferJSON::ToJSON(object) << "," << std::endl;
std::string branch_name = detector_name.empty() ? "global" : detector_name;
trees_[class_name]->Bronch(
branch_name.c_str(), (std::string("std::vector<") + class_name_full + "*>").c_str(), addr);
if(new_tree) {
LOG(DEBUG) << "Pre-filling new tree of " << class_name << " with " << last_event_ << " empty events";
for(unsigned int i = 0; i < last_event_; ++i) {
trees_[class_name]->Fill();
}
} else {
LOG(DEBUG) << "Pre-filling new branch " << branch_name << " of " << class_name << " with "
<< last_event_ << " empty events";
auto* branch = trees_[class_name]->GetBranch(branch_name.c_str());
for(unsigned int i = 0; i < last_event_; ++i) {
branch->Fill();
}
}
}
// Fill the branch vector
for(auto& object : *objects) {
++write_cnt_;
write_list_[index_tuple]->push_back(object);
}
}
} catch(...) {
LOG(WARNING) << "Cannot process object of type" << corryvreckan::demangle(block.first.name());
......@@ -195,50 +148,39 @@ StatusCode FileWriter::run(std::shared_ptr<Clipboard> clipboard) {
}
}
m_eventNumber++;
LOG(TRACE) << "Writing new objects to tree";
output_file_->cd();
if(output_format_ == "root") {
LOG(TRACE) << "Writing new objects to tree";
output_TFile_->cd();
last_event_++;
// Fill the tree with the current received messages
for(auto& tree : trees_) {
tree.second->Fill();
}
// Fill the tree with the current received messages
for(auto& tree : trees_) {
tree.second->Fill();
}
// Clear the current message list
for(auto& index_data : write_list_) {
index_data.second->clear();
}
// Clear the current message list
for(auto& index_data : write_list_) {
index_data.second->clear();
}
return StatusCode::Success;
}
void FileWriter::finalise() {
if(output_format_ == "root") {
LOG(TRACE) << "Writing objects to file";
output_TFile_->cd();
int branch_count = 0;
for(auto& tree : trees_) {
// Update statistics
branch_count += tree.second->GetListOfBranches()->GetEntries();
}
branch_count += event_tree_->GetListOfBranches()->GetEntries();
LOG(TRACE) << "Writing objects to file";
output_file_->cd();
// Finish writing to output file
output_TFile_->Write();
int branch_count = 0;
for(auto& tree : trees_) {
// Update statistics
branch_count += tree.second->GetListOfBranches()->GetEntries();
}
branch_count += event_tree_->GetListOfBranches()->GetEntries();
// Print statistics
LOG(STATUS) << "Wrote " << write_cnt_ << " objects to " << branch_count << " branches in file:" << std::endl
<< output_file_name_;
// Finish writing to output file
output_file_->Write();
} else if(output_format_ == "txt") {
LOG(STATUS) << "Wrote " << m_eventNumber - 1 << " events in file " << output_file_name_;
} else if(output_format_ == "json") {
// finalize the JSON Array, add one empty Object to satisfy JSON Rules
*output_file_ << "{" << std::endl << "}" << std::endl << "]";
LOG(STATUS) << "Wrote " << m_eventNumber - 1 << " events in file " << output_file_name_;
}
// Print statistics
LOG(STATUS) << "Wrote " << write_cnt_ << " objects to " << branch_count << " branches in file:" << std::endl
<< output_file_name_;
}
......@@ -10,10 +10,8 @@
#include <map>
#include <string>
#include <TBufferJSON.h>
#include <TFile.h>
#include <TTree.h>
#include <iostream>
#include "core/module/Module.hpp"
......@@ -62,9 +60,7 @@ namespace corryvreckan {
std::set<std::string> exclude_;
// Output data file to write
std::string output_format_;
std::unique_ptr<TFile> output_TFile_;
std::unique_ptr<std::ofstream> output_file_;
std::unique_ptr<TFile> output_file_;
std::string output_file_name_{};
// List of trees that are stored in data file
......@@ -73,7 +69,7 @@ namespace corryvreckan {
Event* event_{};
// Last event processed
unsigned int m_eventNumber{0};
unsigned int last_event_{0};
// List of objects of a particular type, bound to a specific detector and having a particular name
std::map<std::tuple<std::type_index, std::string>, std::vector<Object*>*> write_list_;
......
......@@ -4,16 +4,12 @@
**Status**: Functional
### Description
Creates an output file with the specified clipboard data depending on the format:
* `root`: Reads all objects from the clipboard into a vector of base class object pointers. The first time a new type of object is received, a new tree is created bearing the class name of this object. For every detector, a new branch is created within this tree. A leaf is automatically created for every member of the object. The vector of objects is then written to the file for every event it is dispatched, saving an empty vector if an event does not include the specific object.
* `txt`: The output is created using `corryvreckan::object::print()` and is equivalent to the output of `TextWriter`.
* `json`: The output file contains a JSON-Array of all objects using `TBufferJSON::ToJASON(object)`. Beware that this results in a flat structure unlike the root file.
Reads all objects from the clipboard into a vector of base class object pointers. The first time a new type of object is received, a new tree is created bearing the class name of this object. For every detector, a new branch is created within this tree. A leaf is automatically created for every member of the object. The vector of objects is then written to the file for every event it is dispatched, saving an empty vector if an event does not include the specific object.
### Parameters
* `file_name` : Name of the data file to create, relative to the output directory of the framework. The file extension will be appended if not present.
* `format`: Format of the data file to create. Currently supported are `root`, `txt` and `json`. Defaults to `root`.
* `include` : Array of object names (without `corryvreckan::` prefix) to write to the file, all other object names are ignored (cannot be used together simultaneously with the *exclude* parameter).
* `exclude`: Array of object names (without `corryvreckan::` prefix) that are not written to the file (cannot be used together simultaneously with the *include* parameter).
* `file_name` : Name of the data file to create, relative to the output directory of the framework. The file extension `.root` will be appended if not present.
* `include` : Array of object names (without `corryvreckan::` prefix) to write to the ROOT trees, all other object names are ignored (cannot be used together simultaneously with the *exclude* parameter).
* `exclude`: Array of object names (without `corryvreckan::` prefix) that are not written to the ROOT trees (cannot be used together simultaneously with the *include* parameter).
### Usage
To create the default file (with the name *data.root*) containing trees for all objects except for Cluster, the following configuration can be placed at the end of the main configuration:
......@@ -22,5 +18,4 @@ To create the default file (with the name *data.root*) containing trees for all
[FileWriter]
file_name = "data.root"
exclude = "Cluster"
format = "root"
```
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment