diff --git a/src/OutputFile.cc b/src/OutputFile.cc index 0136227a22640f2391cacc02effe9e3cb52bbadf..4f683d858cf3d3cf6c13150fdcac5b8f0ace0136 100644 --- a/src/OutputFile.cc +++ b/src/OutputFile.cc @@ -31,10 +31,6 @@ void Detail::LumisectionMetadata::AddFileMetadata(FileMetadata&& fmd) { num_files++; } -bool Detail::LumisectionMetadata::IsLastIndex(int index) const { - return index == static_cast<int>(max_index); -} - std::string Detail::LumisectionMetadata::MakeFooter(uint32_t run_number) const { std::stringstream footer; footer << "{\n \"data\":[\""; diff --git a/src/OutputFile.h b/src/OutputFile.h index d92fc0250e959c4cb9eb6474ea48c4faaf9ec6b9..143bba3458f6587f266928cf8769f5814d8a0cff 100644 --- a/src/OutputFile.h +++ b/src/OutputFile.h @@ -40,15 +40,29 @@ struct FileMetadata { }; struct LumisectionMetadata { + uint32_t lumisection{}; + uint32_t index_in_lumisection{}; size_t file_size{}; uint32_t num_orbits{}; uint32_t num_files{}; uint32_t max_index; - explicit LumisectionMetadata(uint32_t max_index_per_ls) : max_index(max_index_per_ls) {} + // Max index per lumisection should only be enforced if CMSSW headers are enabled + explicit LumisectionMetadata(uint32_t max_index_per_ls = UINT32_MAX) + : max_index(max_index_per_ls) {} + + void UpdateIndex(uint32_t global_index) { + auto new_ls = (global_index / (max_index + 1) + 1); + auto new_idx = global_index % (max_index + 1); + LOG(TRACE) << "Lumisection = " << lumisection << " ==> " << new_ls; + LOG(TRACE) << "Index in LS = " << index_in_lumisection << " ==> " << new_idx; + // Lumisection should only ever change after structure has been reset + assert(lumisection == 0 || lumisection == new_ls); + lumisection = new_ls; + index_in_lumisection = new_idx; + } - bool IsLastIndex(int index) const; - void Reset() { file_size = num_orbits = num_files = 0; } + auto IsLastIndex() const { return index_in_lumisection == max_index; } void AddFileMetadata(FileMetadata&& fmd); std::string MakeFooter(uint32_t run_number) const; }; @@ -91,6 +105,10 @@ class OutputFile { OutputFile() = default; + ~OutputFile() { + if (bool(ls_footer_)) free(ls_footer_); + } + void ReserveHeader() { assert(md_.size == 0); md_.size = sizeof(HeaderType); @@ -119,9 +137,9 @@ class OutputFile { header_ = HeaderType(md_.source_id, md_.num_orbits, md_.run_number, md_.lumisection, md_.size); } - void SetLumisectionFooter(Detail::LumisectionMetadata &&md) { + void SetLumisectionFooter(std::unique_ptr<Detail::LumisectionMetadata>&& ls) { // The last file in a lumisection carries an LS metadata footer - ls_footer_.emplace(md); + ls_footer_ = ls.release(); } auto GetFilename() const { return filename_; } @@ -130,8 +148,8 @@ class OutputFile { auto GetDestinationDir() const { return destination_path_; } auto GetLumisection() const { return md_.lumisection; } auto GetMetadata() const { return md_; } - auto GetLumisectionFooter() const { return ls_footer_.value(); } - auto HasLumisectionFooter() const { return ls_footer_.has_value(); } + auto GetLumisectionFooter() const { return *ls_footer_; } + auto HasLumisectionFooter() const { return bool(ls_footer_); } auto HasPayload() const { return md_.size > sizeof(HeaderType); } auto HasHeader() const { return header_.has_value(); }; auto GetHeader() const -> std::optional<HeaderType> { return header_; } @@ -146,7 +164,7 @@ class OutputFile { private: std::string working_path_; std::string destination_path_; - std::optional<Detail::LumisectionMetadata> ls_footer_; + Detail::LumisectionMetadata* ls_footer_{nullptr}; auto Open() -> bool { tools::CreateDirectory(working_path_); diff --git a/src/OutputFileHandler.cc b/src/OutputFileHandler.cc index 9b3cbef09afee48e941f07941d685a3f3ef44da9..afe0abafded15e75cf3152bf8d611be4bbdfd297 100644 --- a/src/OutputFileHandler.cc +++ b/src/OutputFileHandler.cc @@ -26,11 +26,21 @@ void OutputFileHandler::UpdateRunInfo(uint32_t run, uint32_t index) { LOG(TRACE) << "Previous index: " << current_index_ << " | New index: " << index; current_index_ = static_cast<int>(index); } + + if (!ls_) { + if (HasCmsswHeaders()) + ls_ = std::make_unique<Detail::LumisectionMetadata>( + static_cast<uint32_t>(constants::N_orbits_per_lumisection / nOrbitsPerFile_) - 1); + else + ls_ = std::make_unique<Detail::LumisectionMetadata>(); + } + + ls_->UpdateIndex(current_index_); } void OutputFileHandler::CommitFile(uint32_t run, uint32_t index) { - ls_.AddFileMetadata(outputFile_.GetMetadata()); - if (ls_.IsLastIndex(current_index_) && IsMainPipeline() && HasCmsswHeaders()) { + ls_->AddFileMetadata(outputFile_.GetMetadata()); + if (ls_->IsLastIndex() && IsMainPipeline() && HasCmsswHeaders()) { // If last in lumisection and using CMSSW header and is the main pipeline LOG(TRACE) << "Last file in lumisection; handing over LS metadata footer"; outputFile_.SetLumisectionFooter(std::move(ls_)); @@ -56,14 +66,17 @@ int OutputFileHandler::StageSlice(const char *buffer, size_t size_bytes, uint32_ void OutputFileHandler::NewFile() { // Create a new file - uint32_t ls = 1 + static_cast<uint32_t>(current_index_ / (ls_.max_index + 1)); - uint32_t index = current_index_ % (ls_.max_index + 1); - LOG(TRACE) << "opening file with index " << current_index_ << ", in lumisection " << ls; + auto lumi = ls_->lumisection; + + // uint32_t ls = 1 + static_cast<uint32_t>(current_index_ / (ls_.max_index + 1)); + // uint32_t index = current_index_ % (ls_.max_index + 1); + LOG(TRACE) << "opening file with index " << current_index_ << ", in lumisection " << lumi; auto working_path = sink_.GetRootPath() + GetWorkingDir(); - outputFile_ = OutputFile(FormatFilename(run_.number, current_index_, ls), - {static_cast<uint32_t>(run_.number), ls, index, sourceID_}, &run_, - std::move(working_path), GetSinkDataDir()); + outputFile_ = + OutputFile(FormatFilename(run_.number, ls_->index_in_lumisection, lumi), + {static_cast<uint32_t>(run_.number), lumi, ls_->index_in_lumisection, sourceID_}, + &run_, std::move(working_path), GetSinkDataDir()); // reserve space for CMSSW header if required if (HasCmsswHeaders()) { outputFile_.ReserveHeader(); @@ -74,26 +87,30 @@ void OutputFileHandler::NewFile() { std::string OutputFileHandler::FormatFilename(uint32_t run_number, uint32_t index, uint32_t ls) { if (HasCmsswHeaders()) return Detail::FormatRun(run_number) + "_" + Detail::FormatLumisection(ls) + "_" + - Detail::FormatFileIndex(index % (ls_.max_index + 1)) + loc_.filename_suffix; + Detail::FormatFileIndex(index) + loc_.filename_suffix; else return loc_.filename_prefix + "_" + Detail::FormatRun(run_number) + "_" + Detail::FormatFileIndex(index) + loc_.filename_suffix; } -void OutputFileHandler::CommitLumisection(uint32_t ls_index) { - sink_.WriteLumisectionFooter(run_.number, ls_index, ls_.MakeFooter(run_.number)); +void OutputFileHandler::CommitLumisection() { + sink_.WriteLumisectionFooter(run_.number, ls_->lumisection, ls_->MakeFooter(run_.number)); // Update run counters - run_.AddLumisectionMetadata(ls_); - ls_.Reset(); + run_.AddLumisectionMetadata(*ls_); + + // Delete from heap + ls_.reset(nullptr); } void OutputFileHandler::CommitRun() { assert(current_index_ > 0 && nOrbitsPerFile_ > 0); int ls_index = 1 + (current_index_ * nOrbitsPerFile_) / constants::N_orbits_per_lumisection; + assert(ls_->lumisection == static_cast<uint32_t>(ls_index)); + if (IsMainPipeline()) { - CommitLumisection(ls_index); + CommitLumisection(); } - sink_.WriteRunFooter(run_.number, run_.MakeFooter(ls_index)); + sink_.WriteRunFooter(run_.number, run_.MakeFooter(ls_->lumisection)); } \ No newline at end of file diff --git a/src/OutputFileHandler.h b/src/OutputFileHandler.h index 139abd0395f5134a26dfa1fdc26a397d01d3b108..9831f528faf75e014927c5572ba4152a448c812b 100644 --- a/src/OutputFileHandler.h +++ b/src/OutputFileHandler.h @@ -32,8 +32,10 @@ class OutputFileHandler { has_cmssw_headers_(has_cmssw_headers), is_main_pipeline_(loc_.filename_suffix == ".raw"), sourceID_(source_id), - ls_(static_cast<uint32_t>(constants::N_orbits_per_lumisection / num_orbits_per_file) - 1), - sink_(root_path_) {} + sink_(root_path_) { + ls_ = std::make_unique<Detail::LumisectionMetadata>( + static_cast<uint32_t>(constants::N_orbits_per_lumisection / num_orbits_per_file) - 1); + } ~OutputFileHandler() { CommitFile(run_.number, current_index_); } @@ -46,7 +48,7 @@ class OutputFileHandler { auto IsMainPipeline() const { return is_main_pipeline_; } void CommitFile(uint32_t run, uint32_t index); - void CommitLumisection(uint32_t ls_index); + void CommitLumisection(); void CommitRun(); std::string GetWorkingDir() const; @@ -68,7 +70,7 @@ class OutputFileHandler { public: Detail::RunMetadata run_{}; - Detail::LumisectionMetadata ls_; + std::unique_ptr<Detail::LumisectionMetadata> ls_; FileSink sink_; };