diff --git a/src/OutputFile.cc b/src/OutputFile.cc index 1787ab324429a5983ee21007df4dd552a9f3c4e8..0136227a22640f2391cacc02effe9e3cb52bbadf 100644 --- a/src/OutputFile.cc +++ b/src/OutputFile.cc @@ -31,7 +31,11 @@ void Detail::LumisectionMetadata::AddFileMetadata(FileMetadata&& fmd) { num_files++; } -std::string Detail::LumisectionMetadata::MakeFooter(uint32_t run_number) { +bool Detail::LumisectionMetadata::IsLastIndex(int index) const { + return index == static_cast<int>(max_index); +} + +std::string Detail::LumisectionMetadata::MakeFooter(uint32_t run_number) const { std::stringstream footer; footer << "{\n \"data\":[\""; footer << num_orbits << "\",\""; // NEvents @@ -47,11 +51,12 @@ std::string Detail::LumisectionMetadata::MakeFooter(uint32_t run_number) { } void Detail::RunMetadata::AddLumisectionMetadata(LumisectionMetadata& ls) { + std::lock_guard<std::mutex> l(mutex_); num_orbits += ls.num_orbits; num_files += ls.num_files; } -std::string Detail::RunMetadata::MakeFooter(uint32_t ls_index) { +std::string Detail::RunMetadata::MakeFooter(uint32_t ls_index) const { std::stringstream footer; footer << "{\n \"data\":[\""; footer << num_orbits << "\",\""; // NEvents diff --git a/src/OutputFile.h b/src/OutputFile.h index 1d3d8edc54d7b649d412f13c4a9f191ed71d1b08..dd8a9733d999b37bbecbd01a7cea6d431c66eb78 100644 --- a/src/OutputFile.h +++ b/src/OutputFile.h @@ -40,7 +40,6 @@ struct FileMetadata { }; struct LumisectionMetadata { - uint32_t index{}; size_t file_size{}; uint32_t num_orbits{}; uint32_t num_files{}; @@ -48,9 +47,10 @@ struct LumisectionMetadata { LumisectionMetadata(uint32_t max_index_per_ls) : max_index(max_index_per_ls) {} + bool IsLastIndex(int index) const; void Reset() { file_size = num_orbits = num_files = 0; } void AddFileMetadata(FileMetadata&& fmd); - std::string MakeFooter(uint32_t run_number); + std::string MakeFooter(uint32_t run_number) const; }; struct RunMetadata { @@ -58,12 +58,15 @@ struct RunMetadata { uint32_t num_orbits; uint32_t num_files; + std::mutex mutex_; + void Reset(int run_number) { + std::lock_guard<std::mutex> l(mutex_); num_orbits = num_files = 0; number = run_number; } void AddLumisectionMetadata(LumisectionMetadata& ls); - std::string MakeFooter(uint32_t ls_index); + std::string MakeFooter(uint32_t ls_index) const; }; } // namespace Detail @@ -116,22 +119,22 @@ class OutputFile { header_ = HeaderType(md_.source_id, md_.num_orbits, md_.run_number, md_.lumisection, md_.size); } - void SetLumisectionFooter(Detail::LumisectionMetadata &md) { + void SetLumisectionFooter(Detail::LumisectionMetadata &&md) { // The last file in a lumisection carries an LS metadata footer ls_footer_.emplace(md); } - auto GetFilename() -> std::string { return filename_; } - auto GetFilePointer() -> FILE* { return file_ptr_; } - auto GetWorkingPath() -> std::string { return working_path_; } - auto GetDestinationDir() -> std::string { return destination_path_; } - auto GetLumisection() -> uint32_t { return md_.lumisection; } - auto GetMetadata() -> Detail::FileMetadata { return md_; } - auto GetLumisectionFooter() -> Detail::LumisectionMetadata { return ls_footer_.value(); } - auto HasLumisectionFooter() -> bool { return ls_footer_.has_value(); } - auto HasPayload() -> bool { return md_.size > sizeof(HeaderType); } - auto HasHeader() -> bool { return header_.has_value(); }; - auto GetHeader() -> std::optional<HeaderType> { return header_; } + auto GetFilename() const { return filename_; } + auto GetFilePointer() const { return file_ptr_; } + auto GetWorkingPath() const { return working_path_; } + auto GetDestinationDir() const { return destination_path_; } + auto GetLumisection() const { return md_.lumisection; } + auto GetMetadata() const { return md_; } + auto GetLumisectionFooter() const { return ls_footer_.value(); } + auto HasLumisectionFooter() const { return ls_footer_.has_value(); } + auto HasPayload() const { return md_.size > sizeof(HeaderType); } + auto HasHeader() const { return header_.has_value(); }; + auto GetHeader() const -> std::optional<HeaderType> { return header_; } private: FILE* file_ptr_{}; diff --git a/src/OutputFileHandler.cc b/src/OutputFileHandler.cc index 1205d3b6d94406abc4179c2d2d14a001991e5db6..d32474e2b0ecc1d6527357923a12df7b49f6e75d 100644 --- a/src/OutputFileHandler.cc +++ b/src/OutputFileHandler.cc @@ -30,10 +30,10 @@ void OutputFileHandler::UpdateRunInfo(uint32_t run, uint32_t index) { void OutputFileHandler::CommitFile(uint32_t run, uint32_t index) { ls_.AddFileMetadata(outputFile_.GetMetadata()); - if (current_index_ == static_cast<int>(ls_.max_index) && IsMainPipeline() && HasCmsswHeaders()) { + if (ls_.IsLastIndex(current_index_) && IsMainPipeline() && HasCmsswHeaders()) { // If last in lumisection and using CMSSW header and is the main pipeline LOG(TRACE) << "Last file in lumisection; handing over LS metadata footer"; - outputFile_.SetLumisectionFooter(ls_); + outputFile_.SetLumisectionFooter(std::move(ls_)); } sink_.Submit(std::move(outputFile_)); @@ -81,7 +81,7 @@ std::string OutputFileHandler::FormatFilename(uint32_t run_number, uint32_t inde } void OutputFileHandler::CommitLumisection(uint32_t ls_index) { - sink_.WriteLumisectionFooter(run_.number, ls_index, ls_); + sink_.WriteLumisectionFooter(run_.number, ls_index, ls_.MakeFooter(run_.number)); // Update run counters run_.AddLumisectionMetadata(ls_); ls_.Reset(); @@ -95,5 +95,5 @@ void OutputFileHandler::CommitRun() { CommitLumisection(ls_index); } - sink_.WriteRunFooter(ls_index, run_); + sink_.WriteRunFooter(run_.number, run_.MakeFooter(ls_index)); } \ No newline at end of file diff --git a/src/sink.cc b/src/sink.cc index c6b19256ba9369c9e973020ada2472b9bf09b0ff..625c6a68b51117cf7ff7697a5d76e7c9f988a4e5 100644 --- a/src/sink.cc +++ b/src/sink.cc @@ -41,7 +41,8 @@ void FileSink::ProcessQueue() { if (file.HasLumisectionFooter()) { auto ls = file.GetLumisectionFooter(); - WriteLumisectionFooter(file.GetMetadata().run_number, file.GetLumisection(), ls); + auto run_number = file.GetMetadata().run_number; + WriteLumisectionFooter(run_number, file.GetLumisection(), ls.MakeFooter(run_number)); file.run_metrics_->AddLumisectionMetadata(ls); } } @@ -77,20 +78,19 @@ void FileSink::Submit(OutputFile &&file) { } } -void FileSink::WriteLumisectionFooter(uint32_t run_number, uint32_t ls_id, - Detail::LumisectionMetadata ls) { +void FileSink::WriteLumisectionFooter(uint32_t run_number, uint32_t ls_id, std::string &&footer) { auto filename = Detail::FormatRun(run_number) + "_" + Detail::FormatLumisection(ls_id) + "_EoLS.jsn"; auto sub_path = Detail::FormatRun(run_number) + "/" + filename; LOG(TRACE) << "Writing EoLS footer file " << sub_path; - BlockingWrite(std::move(sub_path), ls.MakeFooter(run_number)); + BlockingWrite(std::move(sub_path), std::move(footer)); } -void FileSink::WriteRunFooter(uint32_t ls_id, Detail::RunMetadata run) { - auto filename = Detail::FormatRun(run.number) + "_" + Detail::FormatLumisection(0) + "_EoR.jsn"; - auto sub_path = Detail::FormatRun(run.number) + "/" + filename; +void FileSink::WriteRunFooter(uint32_t run_number, std::string &&footer) { + auto filename = Detail::FormatRun(run_number) + "_" + Detail::FormatLumisection(0) + "_EoR.jsn"; + auto sub_path = Detail::FormatRun(run_number) + "/" + filename; LOG(TRACE) << "Writing EoR file " << sub_path; - BlockingWrite(std::move(sub_path), run.MakeFooter(ls_id)); + BlockingWrite(std::move(sub_path), std::move(footer)); } diff --git a/src/sink.h b/src/sink.h index 65daaa2284e7b3d9d924557f9aaf509586ba882b..2a72bbf1872c86d3435e4072c0fe1af9c03a33cd 100644 --- a/src/sink.h +++ b/src/sink.h @@ -31,9 +31,9 @@ class FileSink { void BlockingWrite(std::string &&loc, std::string &&content); void CommitFileHeader(OutputFile &file); - void WriteLumisectionFooter(uint32_t run_number, uint32_t ls_id, Detail::LumisectionMetadata md); - void WriteRunFooter(uint32_t ls_id, Detail::RunMetadata run); - std::string GetRootPath() { return root_path_; } + void WriteLumisectionFooter(uint32_t run_number, uint32_t ls_id, std::string &&footer); + void WriteRunFooter(uint32_t run_number, std::string &&footer); + std::string GetRootPath() const { return root_path_; } private: std::thread commit_thread_;