From 1976f44f6670511003f1dfbc3d01e97162aeeb02 Mon Sep 17 00:00:00 2001 From: Default Online user <Markus.Frank@cern.ch> Date: Thu, 7 Oct 2021 10:22:51 +0200 Subject: [PATCH 1/5] Fixes to dim, mbm, controller --- Online/Dataflow/src/Utils/RawFileMerger.cpp | 153 +++++++++++++----- Online/Dataflow/src/Utils/RawFileMerger.h | 24 ++- Online/Dataflow/src/framework/DiskReader.cpp | 128 +++++++-------- Online/FarmConfig/job/Controller.sh | 25 ++- Online/FarmConfig/job/runTask.sh | 7 +- Online/FarmConfig/options/EBReader.opts | 6 +- Online/FarmConfig/options/HLT2Reader.opts | 31 ++-- Online/FarmConfig/options/HLT2Writer.opts | 5 +- Online/GauchoAppl/src/SaveTimer.cpp | 24 +-- .../OnlineBase/OnlineBase/LOG/FifoLog.inl.h | 13 +- Online/OnlineBase/OnlineBase/MBM/bmdef.h | 3 + Online/OnlineBase/src/MBM/mbmlib_client.cpp | 11 ++ Online/SmiController/src/SmiController.cpp | 7 +- Online/dim/src/open_dns.c | 4 + 14 files changed, 285 insertions(+), 156 deletions(-) diff --git a/Online/Dataflow/src/Utils/RawFileMerger.cpp b/Online/Dataflow/src/Utils/RawFileMerger.cpp index 597cabb42..68285c339 100644 --- a/Online/Dataflow/src/Utils/RawFileMerger.cpp +++ b/Online/Dataflow/src/Utils/RawFileMerger.cpp @@ -23,6 +23,8 @@ #include "RZip.h" // C/C++ include files +#include <system_error> +#include <filesystem> #include <stdexcept> #include <iostream> #include <random> @@ -38,6 +40,7 @@ namespace { class file_writer : public pcie40::encoder_t::output_t { Online::RawFile* file = 0; + public: file_writer(Online::RawFile* f) : file(f) {} virtual size_t output(const void* buffer, size_t len) override @@ -81,6 +84,10 @@ namespace { } } +class RawFileMerger::event_output : public RawFile { + using RawFile::RawFile; +}; + class RawFileMerger::event_mixer { public: std::default_random_engine generator; @@ -129,13 +136,31 @@ public: RawFileMerger::RawFileMerger(const char* output_name, OutputType output_type) : outputName(output_name), outputType(output_type) { - m_mixer = make_unique<event_mixer>(); + this->mixer = make_unique<event_mixer>(); } /// Default destructor RawFileMerger::~RawFileMerger() { } +/// Open new output file +int RawFileMerger::open_output() { + string fname = this->outputName; + if ( write_many ) { + stringstream str; + str << this->outputName << "." << this->out_seq_no; + ++this->out_seq_no; + fname = str.str(); + } + this->output = make_unique<event_output>(fname); + this->output->openWrite(false); + if ( this->output->isOpen() ) { + return 1; + } + cout << "Output file: " << this->output->name() << " cannot be opened [" << RTL::errorString() << "]" << endl; + return 0; +} + /// Start merge cycle: open output file int RawFileMerger::begin() { return this->begin(this->outputName.c_str(), this->outputType); @@ -143,31 +168,74 @@ int RawFileMerger::begin() { /// Start merge cycle: open output file int RawFileMerger::begin(const char* output_name, OutputType output_type) { - m_mixer = make_unique<event_mixer>(); - if ( !this->output.isOpen() ) { + this->mixer = make_unique<event_mixer>(); + this->out_seq_no = 0; + if ( !this->output->isOpen() ) { this->outputName = output_name; this->outputType = output_type; - this->output = RawFile(this->outputName); - this->output.openWrite(false); - if ( this->output.isOpen() ) { - return 1; - } - cout << "Output file: " << this->output.name() << " cannot be opened [" << RTL::errorString() << "]" << endl; - return 0; + return this->open_output(); + } + cout << "Output file: " << this->output->name() << " already open. Instruction forbidden." << endl; + return 0; +} + +/// End the file merging cycle and close the output file +int RawFileMerger::end() { + if ( this->output->isOpen() ) { + size_t bytes = this->output->data_size(); + this->output->reset(); + cout << "Output file " << this->output->name() << " closed after " << bytes << " bytes." << endl; + return 1; } - cout << "Output file: " << this->output.name() << " already open. Instruction forbidden." << endl; + cout << "Output file NOT open. Instruction forbidden." << endl; return 0; } /// Add new input source to the merger object size_t RawFileMerger::add(const char* file_name) { - m_mixer->inputs.emplace_back(file_name); - return this->m_mixer->inputs.size(); + namespace fs = std::filesystem; + fs::path path(file_name); + error_code ec; + + if ( !fs::exists(path, ec) ) { + cout << "Input " << file_name << " is not accessible: " << ec.message() << endl; + return 0; + } + else if ( fs::is_directory(path, ec) ) { + for(auto const& dir : std::filesystem::directory_iterator{path}) { + fs::path dir_path(dir); + if ( dir_path.string() == "." ) + continue; + if ( dir_path.string() == ".." ) + continue; + + fs::path p = fs::absolute(dir_path, ec); + /// Resolve symbolic links if necessary + if ( fs::is_symlink(path) ) + path = fs::absolute(fs::read_symlink(path, ec), ec); + + /// Check for directories: if required recurse + if ( fs::is_directory(path, ec) ) { + if ( fs::is_empty(path, ec) ) + continue; + // If we would like to add whole trees, enable this... + // this->add(path.c_str()); + continue; + } + this->mixer->inputs.emplace_back(path); + } + cout << "Scanned directory " << file_name + << ": " << this->mixer->inputs.size() << " files." << endl; + return this->mixer->inputs.size(); + } + cout << " --> Add file entry: " << file_name << endl; + this->mixer->inputs.emplace_back(file_name); + return this->mixer->inputs.size(); } /// Merge a maximum of 'max_events' events to the output file int RawFileMerger::merge(size_t max_events) { - if ( this->output.isOpen() ) { + if ( this->output->isOpen() ) { if ( this->outputType == PCIE40 ) { return merge_pcie40(max_events); } @@ -187,7 +255,7 @@ int RawFileMerger::merge_tell1(size_t max_events) { unique_ptr<uint8_t []> compressed_buffer(new uint8_t[buff_len]); for (size_t events = max_events; events; --events ) { - long data_len = m_mixer->read(compressed_buffer.get(), buff_len); + long data_len = this->mixer->read(compressed_buffer.get(), buff_len); if ( data_len > 0 ) { EventHeader* hdr = (EventHeader*)compressed_buffer.get(); size_t hdr_len = hdr->sizeOf(hdr->headerVersion()); @@ -218,17 +286,26 @@ int RawFileMerger::merge_tell1(size_t max_events) { continue; } } - long len = this->output.write(data_ptr, data_len); + long len = this->output->write(data_ptr, data_len); if ( len != data_len ) { - cout << "++ Failed to write output record to file: " << this->output.name() << endl; + cout << "++ Failed to write output record to file: " << this->output->name() << endl; break; } num_bytes += len; - if ( num_bytes >= this->max_file_size ) break; + if ( num_bytes >= this->max_file_size ) { + if ( write_many ) { + this->output->reset(); + if ( !this->open_output() ) break; + num_bytes = 0; + } + else { + break; + } + } } } cout << "++ Wrote" << setw(5) << int(num_bytes/1024/1024) << " MB " - << " to " << this->output.name() + << " to " << this->output->name() << endl; return 1; } @@ -237,7 +314,7 @@ int RawFileMerger::merge_tell1(size_t max_events) { int RawFileMerger::merge_pcie40(size_t max_events) { pcie40::encoder_t encoder; pcie40::encoder_t::tae_options tae_opts {0, 0e0}; - file_writer writer {&this->output}; + file_writer writer {this->output.get()}; size_t eid = 0; size_t num_bytes = 0; size_t decomp_len = 0; @@ -247,7 +324,7 @@ int RawFileMerger::merge_pcie40(size_t max_events) { unique_ptr<uint8_t []> compressed_buffer(new uint8_t[buff_len]); for (size_t events = max_events; events; --events ) { - long ret = m_mixer->read(compressed_buffer.get(), buff_len); + long ret = this->mixer->read(compressed_buffer.get(), buff_len); if ( ret > 0 ) { EventHeader* hdr = (EventHeader*)compressed_buffer.get(); size_t hdr_len = hdr->sizeOf(hdr->headerVersion()); @@ -285,31 +362,29 @@ int RawFileMerger::merge_pcie40(size_t max_events) { auto ret = encoder.write_record(writer, tae_opts); encoder.reset(); num_bytes += ret.second; - cout << "++ Wrote" - << " Sources: " << ret.first + cout << "++ Wrote" << this->output->name() << endl + << " Sources: " << ret.first << " Filesize:" << setw(5) << int(num_bytes/1024/1024) << " MB " << " Packing: " << this->packing - << " Background:" << setw(7) << m_mixer->num_evt - << " Signal:" << setw(7) << m_mixer->num_sig + << " Background:" << setw(7) << this->mixer->num_evt + << " Signal:" << setw(7) << this->mixer->num_sig << endl; - if ( num_bytes >= this->max_file_size ) break; + if ( num_bytes >= this->max_file_size ) { + if ( write_many ) { + this->output->reset(); + if ( !this->open_output() ) break; + writer = file_writer{this->output.get()}; + num_bytes = 0; + } + else { + break; + } + } } continue; } - if ( m_mixer->done() ) break; + if ( this->mixer->done() ) break; } return 1; } -/// End the file merging cycle and close the output file -int RawFileMerger::end() { - if ( this->output.isOpen() ) { - size_t bytes = this->output.data_size(); - this->output.reset(); - cout << "Output file " << this->output.name() << " closed after " << bytes << " bytes." << endl; - return 1; - } - cout << "Output file NOT open. Instruction forbidden." << endl; - return 0; -} - diff --git a/Online/Dataflow/src/Utils/RawFileMerger.h b/Online/Dataflow/src/Utils/RawFileMerger.h index 01e67cb6c..f6eb3b03b 100644 --- a/Online/Dataflow/src/Utils/RawFileMerger.h +++ b/Online/Dataflow/src/Utils/RawFileMerger.h @@ -18,11 +18,11 @@ #define ONLINE_DATAFLOW_RAWFILEMERGER_H // Framework include files -#include <Tell1Data/RawFile.h> // C/C++ include files #include <string> #include <memory> +#include <limits> /// Online namespace declaration namespace Online { @@ -35,24 +35,34 @@ namespace Online { */ class RawFileMerger { public: + enum OutputType { PCIE40, TELL1 }; - class event_mixer; - - protected: - RawFile output { }; std::string outputName { }; OutputType outputType { TELL1 }; size_t packing { 1 }; - size_t max_file_size { ~0x0UL }; - std::unique_ptr<event_mixer> m_mixer; + size_t max_file_size { std::numeric_limits<size_t>::max() }; + size_t max_events { std::numeric_limits<size_t>::max() }; + bool write_many { false }; + + protected: + class event_mixer; + class event_output; + + typedef std::unique_ptr<event_mixer> mixer_t; + typedef std::unique_ptr<event_output> output_t; + mixer_t mixer { }; + output_t output { }; + int out_seq_no { 0 }; /// Merge a maximum of 'max_events' events to the output file in TELL1 format int merge_tell1 (size_t max_events = ~0x0UL); /// Merge a maximum of 'max_events' events to the output file in PCIE40 format int merge_pcie40(size_t max_events = ~0x0UL); + /// Open new output file + int open_output(); public: //// Default constructor diff --git a/Online/Dataflow/src/framework/DiskReader.cpp b/Online/Dataflow/src/framework/DiskReader.cpp index 5802b6148..6201b74e1 100644 --- a/Online/Dataflow/src/framework/DiskReader.cpp +++ b/Online/Dataflow/src/framework/DiskReader.cpp @@ -15,7 +15,7 @@ // Author : Markus Frank //========================================================================== -// Framework includes +/// Framework includes #include <Dataflow/DiskReader.h> #include <Dataflow/DataflowTask.h> #include <Dataflow/ControlPlug.h> @@ -29,26 +29,16 @@ #include <RTL/rtl.h> #include <RZip.h> -/// Boost include files -#include <boost/filesystem.hpp> - +/// C/C++ include files #include <fcntl.h> #include <cerrno> #include <cstring> #include <sstream> +#include <filesystem> #include <dim/dic.h> #include <dim/dis.h> -#ifdef _WIN32 -#include <io.h> -#else -#include <unistd.h> -#include <libgen.h> -#define O_BINARY 0 -#endif - -using namespace Online; -using namespace std; +namespace fs = std::filesystem; using namespace Online; using namespace std; @@ -577,6 +567,10 @@ DiskReader::load_event_data(RawFile::Allocator& allocator, RawFile& file) { if ( mep_hdr->is_valid() ) { int data_size = mep_hdr->size*sizeof(uint32_t); data_ptr = allocator(data_size); + if ( !data_ptr ) { + file.position(position); + return { 0, -1, RawFile::NO_INPUT_TYPE }; + } ::memcpy(data_ptr, size, peek_size); status = file.read(data_ptr+peek_size, data_size-peek_size); if ( status < int(data_size-peek_size) ) { @@ -584,6 +578,11 @@ DiskReader::load_event_data(RawFile::Allocator& allocator, RawFile& file) { return { 0, -1, RawFile::NO_INPUT_TYPE }; } ++this->m_burstsIN; + mep_hdr = (pcie40::mep_header_t*)data_ptr; + if ( mep_hdr->num_source > 0 ) { + int packing = mep_hdr->multi_fragment(0)->packingFactor(); + return { size_t(data_size), packing, RawFile::MEP_INPUT_TYPE }; + } return { size_t(data_size), 1, RawFile::MEP_INPUT_TYPE }; } // @@ -953,65 +952,53 @@ DiskReader::DiskIO::DiskIO(DiskReader& r) /// Scan directory for matching files size_t DiskReader::DiskIO::scanDirectory(const string& dir_name) { - namespace fs = boost::filesystem; - if ( !dir_name.empty() ) { - DIR* dir = opendir(dir_name.c_str()); - if (dir) { - struct dirent *entry; - const string& pref = this->filePrefix; - boost::system::error_code ec; - fs::path dir_path(dir_name.c_str()); - bool take_all = (this->allowedRuns.size() > 0 && this->allowedRuns[0]=="*"); + fs::path dir_path(dir_name); + error_code ec; + if ( fs::exists(dir_path, ec) && fs::is_directory(dir_path, ec) ) { + auto pref = fs::path(dir_path).append(this->filePrefix).string(); + bool take_all = (this->allowedRuns.size() > 0 && this->allowedRuns[0]=="*"); - while ((entry = ::readdir(dir)) != 0) { - if ( 0 == ::strcmp(entry->d_name,".") ) { - continue; - } - else if ( 0 == ::strcmp(entry->d_name,"..") ) { - continue; - } - - fs::path path = fs::absolute({entry->d_name}, dir_path, ec); - /// Resolve symbolic links if necessary - if ( fs::is_symlink(path) ) { - path = fs::absolute(path, fs::read_symlink(path, ec), ec); - } - /// Check for directories: if required recurse - if ( fs::is_directory(path, ec) ) { - if ( !this->scan_recursive ) - continue; - if ( fs::is_empty(path, ec) ) - continue; - this->scanDirectory(path.string()); + for(auto const& entry : fs::directory_iterator{dir_path,ec}) { + fs::path path(entry); + if ( path.string() == "." ) + continue; + if ( path.string() == ".." ) + continue; + + /// Check for directories: if required recurse + if ( fs::is_directory(path, ec) ) { + if ( !this->scan_recursive ) continue; - } - /// Got a link or a real file - if ( !pref.empty() && 0 != ::strncmp(entry->d_name,pref.c_str(),pref.length()) ) { + if ( fs::is_empty(path, ec) ) continue; - } - else if ( !take_all ) { - bool take_run = false; - for(auto i=this->allowedRuns.begin(); i!=this->allowedRuns.end(); ++i) { - if ( ::strstr(entry->d_name,(*i).c_str()) == entry->d_name ) { - take_run = true; - break; - } + this->scanDirectory(path.string()); + continue; + } + /// Got a link or a real file + if ( !pref.empty() && 0 != ::strncmp(path.c_str(),pref.c_str(),pref.length()) ) { + continue; + } + else if ( !take_all ) { + bool take_run = false; + for(const auto& r : this->allowedRuns ) { + if ( path.string().find(r) != string::npos ) { + take_run = true; + break; } - if ( !take_run ) continue; } - RawFile file(path.c_str()); - // Remove the identified bad ones from the list... - if ( this->badFiles.find(file.name()) != this->badFiles.end() ) - continue; - this->files.insert(make_pair(entry->d_name,file)); + if ( !take_run ) continue; } - ::closedir(dir); - this->reader.info("Scanned directory %s: %ld files.", dir_name.c_str(), this->files.size()); - return this->files.size(); + RawFile file(path.string()); + // Remove the identified bad ones from the list... + if ( this->badFiles.find(file.name()) != this->badFiles.end() ) + continue; + this->files.insert(make_pair(path.string(),file)); } - auto err = RTL::errorString(); - this->reader.info("Failed to open directory: %s", !err.empty() ? err.c_str() : "????????"); + this->reader.info("Scanned directory %s: %ld files.", dir_name.c_str(), this->files.size()); + return this->files.size(); } + this->reader.info("Failed to open directory: %s [%s]", + dir_path.c_str(), ec.message().c_str()); return 0; } @@ -1063,16 +1050,13 @@ RawFile DiskReader::DiskIO::openFile() { } else if ( sc != 0 ) { // Suspicion of bad disk: - char dir_buff[PATH_MAX]; + auto dir = fs::path(file.name()).parent_path(); ::close(fd); - ::strncpy(dir_buff,file.cname(),PATH_MAX-1); - dir_buff[PATH_MAX-1] = 0; - char* dir = ::dirname(dir_buff); - if ( dir && strlen(dir) > 2 ) { + if ( fs::exists(dir) ) { auto ff = files; - string dir_name = dir; + string dir_name = dir.string(); reader.info("File: %s SKIPPED for HLT processing. Remove input of device:%s", - file.cname(), dir); + file.cname(), dir_name.c_str()); for( const auto& i : files ) { if ( i.second.name().find(dir_name) == 0 ) { auto j = ff.find(i.first); diff --git a/Online/FarmConfig/job/Controller.sh b/Online/FarmConfig/job/Controller.sh index 67d1b4fc4..0a93d9c24 100755 --- a/Online/FarmConfig/job/Controller.sh +++ b/Online/FarmConfig/job/Controller.sh @@ -36,6 +36,27 @@ if test -z "${DIM_DNS_NODE}"; then export DIM_DNS_NODE=${HOST_LONG}; fi; # +# +# +DEBUG_ARGS=; +# +# DEBUG_ARGS="-debug"; +# echo "${UTGID} [ERROR] ${HOST}"; +SMI_DEBUG=0; +if test "${HOST}" = "HLT20101"; then + SMI_DEBUG=1; +fi; +if test "${HOST}" = "HLT20102"; then +# cd /group/online/dataflow/cmtuser/OnlineDev_v7r11; +# . setup.x86_64_v2-centos7-gcc10-do0.vars; + SMI_DEBUG=1; +fi; +if test "${HOST}" = "TDEB03"; then +# cd /group/online/dataflow/cmtuser/OnlineDev_v7r11; +# . setup.x86_64_v2-centos7-gcc10-do0.vars; + SMI_DEBUG=0; +fi; +# make_gdb_input() { echo "run" > /tmp/gdb_commands.txt; echo "thread apply all bt" >> /tmp/gdb_commands.txt; @@ -53,7 +74,7 @@ exec -a ${UTGID} `which genRunner.exe` libSmiController.so smi_controller \ -tmsdns=${TMS_DNS} \ -smidns=${SMI_DNS} \ -smidomain=${SMI_DOMAIN} \ - -smidebug=0 \ + -smidebug=${SMI_DEBUG} \ -smifile=${SMI_FILE} \ -count=${NBOFSLAVES} \ -service=none \ @@ -61,7 +82,7 @@ exec -a ${UTGID} `which genRunner.exe` libSmiController.so smi_controller \ -taskconfig=${ARCH_FILE} \ "${CONTROLLER_REPLACEMENTS}" \ -standalone=1 \ - -bindcpus=0 + -bindcpus=0 ${DEBUG_ARGS} ## ## ## < /tmp/gdb_commands.txt 2>&1 > /dev/shm/logs.dev; diff --git a/Online/FarmConfig/job/runTask.sh b/Online/FarmConfig/job/runTask.sh index dfc58741d..8fd1d760f 100755 --- a/Online/FarmConfig/job/runTask.sh +++ b/Online/FarmConfig/job/runTask.sh @@ -51,9 +51,9 @@ PRINT_COMMAND_LINE=; ####YES; execute() { if test -n "${PRINT_COMMAND_LINE}"; then - echo "[INFO] INFO_OPTIONS: ${INFO_OPTIONS}"; - echo "[INFO] MBM_SETUP_OPTIONS: ${MBM_SETUP_OPTIONS}"; - echo "[INFO] $*"; + echo "${UTGID} [INFO] INFO_OPTIONS: ${INFO_OPTIONS}"; + echo "${UTGID} [INFO] MBM_SETUP_OPTIONS: ${MBM_SETUP_OPTIONS}"; + echo "${UTGID} [INFO] $*"; fi; $*; } @@ -82,6 +82,7 @@ elif test -f ./${PARTITION_NAME}DefaultTask.sh; then execute . ./${PARTITION_NAME}DefaultTask.sh; # else + echo "ERROR Running default dummy task "; cd ${SMICONTROLLERROOT}/scripts; execute exec -a ${UTGID} genPython.exe ${SMICONTROLLERROOT}/scripts/DummyTask.py -utgid ${UTGID} -partition 103 -print ${OUTPUT_LEVEL}; fi; diff --git a/Online/FarmConfig/options/EBReader.opts b/Online/FarmConfig/options/EBReader.opts index 561bcc394..9bd4eca75 100644 --- a/Online/FarmConfig/options/EBReader.opts +++ b/Online/FarmConfig/options/EBReader.opts @@ -17,9 +17,11 @@ Task.HavePause = true; Reader.Buffer = "$MBM_OUTPUT_BUFFER"; Reader.BrokenHosts = ""; Reader.Directories = {"/group/online/dataflow/cmtuser/data/mdf", + "/group/online/dataflow/cmtuser/data/mep", "/group/online/dataflow/cmtuser/data/tae" }; -Reader.FilePrefix = "file_"; +Reader.Directories = { "/daqarea1/fest/mep"}; +Reader.FilePrefix = "00"; // Reader.AllowedRuns = {"*"}; Reader.MuDelay = 0; @@ -58,4 +60,4 @@ Reader.FDBVersion = 1; Reader.PartitionName = "FEST"; Reader.FilePrefix = "${PARTITION}/${RUN}/"; Reader.AllowedRuns = { "219677","219679","219680","219681" }; -*/ \ No newline at end of file +*/ diff --git a/Online/FarmConfig/options/HLT2Reader.opts b/Online/FarmConfig/options/HLT2Reader.opts index aba764084..b078116f1 100644 --- a/Online/FarmConfig/options/HLT2Reader.opts +++ b/Online/FarmConfig/options/HLT2Reader.opts @@ -5,8 +5,8 @@ #include "$FARMCONFIGROOT/options/Monitoring.opts" // Manager.Services = {"Dataflow_MBMClient/MEPManager", -// "Dataflow_BurstReader/Reader", - "Dataflow_StorageReader/Reader", + "Dataflow_BurstReader/Reader", +// "Dataflow_StorageReader/Reader", "Dataflow_RunableWrapper/Wrap", "Dataflow_UI/UI" }; @@ -14,21 +14,30 @@ Manager.Runable = "Wrap"; Wrap.Callable = "Reader"; Task.HavePause = true; // +MEPManager.PartitionBuffers = true; +MEPManager.PartitionName = @OnlineEnv.PartitionName; +MEPManager.PartitionID = @OnlineEnv.PartitionID; +MEPManager.Buffers = {"Events"};//@OnlineEnv.HLT2Reader_Buffers; +// Reader.Buffer = "Events";//@OnlineEnv.Hlt2Reader_Output; Reader.BrokenHosts = ""; +/* Reader.Directories = {"/home/frankm/data"}; Reader.FilePrefix = "fakeMDF_"; Reader.Directories = {"/daqarea1/fest/mdf"}; // Take the first 100 files: 00135282_000000XX_X.mdf Reader.FilePrefix = "00135282_0000000"; // -Reader.Directories = {"/daqarea1/fest/mdf"}; -Reader.FilePrefix = "00135"; // Reader.Directories = {"/group/online/dataflow/cmtuser/data/mdf", "/group/online/dataflow/cmtuser/data/tae" }; +*/ +Reader.Directories = {"/group/online/dataflow/cmtuser/data/mdf"}; Reader.FilePrefix = "file_"; +Reader.Directories = {"/daqarea1/fest/202110/mdf/30000000"}; +Reader.FilePrefix = "00146082"; + // Reader.AllowedRuns = {"*"}; Reader.MuDelay = 0; @@ -45,24 +54,22 @@ Reader.ReuseFile = 0; Reader.PackingFactor = 100; Reader.AllocationSizekB = 20000; Reader.PatchOdin = 0; // 0 = no patching -// -MEPManager.PartitionBuffers = true; -MEPManager.PartitionName = @OnlineEnv.PartitionName; -MEPManager.PartitionID = @OnlineEnv.PartitionID; -MEPManager.Buffers = {"Events"};//@OnlineEnv.HLT2Reader_Buffers; - +/* Reader.PartitionName = "FEST"; Reader.Directories = {""}; Reader.FilePrefix = "/daqarea1/objects/nfs_data/${PARTITION}/${RUN}/Run_${RUN}"; Reader.DataType = "posix"; Reader.Server = "XXEB09.lbdaq.cern.ch:8000"; +*/ // +/* Reader.DataType = "network"; Reader.Server = "devbbdb01.lbdaq.cern.ch:4242"; Reader.FDBVersion = 1; Reader.PartitionName = "FEST"; Reader.FilePrefix = "${PARTITION}/${RUN}/"; -Reader.AllowedRuns = { "219681","219687","219688","219689","219690","219691","219692","219693","219694","219695","219696","219697" }; +Reader.AllowedRuns = { "219722", "219723", "219724", "219725", "299999" }; +Reader.AllowedRuns = { "219722", "299999" }; Reader.NumThreads = 1; Reader.NumBuffers = 3; - +*/ diff --git a/Online/FarmConfig/options/HLT2Writer.opts b/Online/FarmConfig/options/HLT2Writer.opts index 53c975ddc..2732f2843 100644 --- a/Online/FarmConfig/options/HLT2Writer.opts +++ b/Online/FarmConfig/options/HLT2Writer.opts @@ -14,8 +14,7 @@ Writer.Server = "XXEB09.lbdaq.cern.ch:8000"; Writer.FileName = "/objects/data/HLT2/${PARTITION}/${STREAM}/${RUN1000}/${RUN}/Run_${RUN}_${NODE}_${TIME}_${PID}_${SEQ}.mdf"; // EventProc.REQ2 = "EvType=2;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=VIP;Frequency=PERC;Perc=100.0"; - - +// // NFS writing Writer.PartitionName = "FEST"; Writer.ThreadFileQueues = false; @@ -27,5 +26,5 @@ Writer.MaxFileSizeMB = 4000; Writer.BufferSizeMB = 32; Writer.Server = "XXEB09.lbdaq.cern.ch:8100"; Writer.FileName = "file:/daqarea1/objects/nfs_data/${PARTITION}/${RUN}/Run_${RUN}_${NODE}_${TIME}_${SEQ}.mdf"; -Writer.FileName = "/hlt2/objects/${PARTITION}/${RUN}/Run_${RUN}_${NODE}_${TIME}_${SEQ}.mdf"; +Writer.FileName = "/hlt2/HLT2/${PARTITION}/${RUN}/Run_${RUN}_${NODE}_${TIME}_${SEQ}.mdf"; Writer.OutputType = "POSIX"; diff --git a/Online/GauchoAppl/src/SaveTimer.cpp b/Online/GauchoAppl/src/SaveTimer.cpp index 884234f16..0950e76e8 100644 --- a/Online/GauchoAppl/src/SaveTimer.cpp +++ b/Online/GauchoAppl/src/SaveTimer.cpp @@ -62,17 +62,23 @@ void SaveTimer::timerHandler(void) m_adder->UnLock(); for (auto i =m_adder->m_ClassMap->begin(); i!= m_adder->m_ClassMap->end();i++) { size_t siz = i->second->m_buffersize; - if ( m_buffer.first < siz ) { - m_buffer.second.reset(new unsigned char[m_buffer.first=siz]); + if ( siz > 0 ) { + if ( m_buffer.first < siz ) { + m_buffer.second.reset(new unsigned char[m_buffer.first=siz]); + } + ::memcpy(m_buffer.second.get(),i->second->m_buffer,siz); + SerialHeader *hd = (SerialHeader *)m_buffer.second.get(); + unsigned int runo = (unsigned int)hd->run_number; + if ( m_currFile == 0 ) { + makeDirs(runo); + openFile(); + } + savetoFile(m_buffer.second.get()); } - ::memcpy(m_buffer.second.get(),i->second->m_buffer,siz); - SerialHeader *hd = (SerialHeader *)m_buffer.second.get(); - unsigned int runo = (unsigned int)hd->run_number; - if ( m_currFile == 0 ) { - makeDirs(runo); - openFile(); + else { + ::printf("Not writing histograms for %s [No data present]", + i->second->m_outservname.c_str()); } - savetoFile(m_buffer.second.get()); } closeFile(); } diff --git a/Online/OnlineBase/OnlineBase/LOG/FifoLog.inl.h b/Online/OnlineBase/OnlineBase/LOG/FifoLog.inl.h index 836b5b03b..57238fb5e 100644 --- a/Online/OnlineBase/OnlineBase/LOG/FifoLog.inl.h +++ b/Online/OnlineBase/OnlineBase/LOG/FifoLog.inl.h @@ -265,6 +265,7 @@ namespace fifolog { #include <fcntl.h> #include <unistd.h> #include <sys/stat.h> +#include <sys/time.h> #include <sys/types.h> #include <sys/epoll.h> @@ -536,12 +537,16 @@ void logger_imp::flush_line(int stream_id) { if ( this->fifo_fd >= 0 ) { if ( data.first > 0 ) { struct tm tim; + struct timeval now; size_t len, mx; - time_t now = ::time(0); char output_buffer[2*OUTPUT_MAX_LEN]; - ::localtime_r(&now, &tim); - len = ::strftime(output_buffer,sizeof(output_buffer), - "{\"@timestamp\":\"%Y-%m-%dT%H:%M:%S.000000%z\",\"message\":\"",&tim); + char format[256]; + ::gettimeofday(&now, nullptr); + ::localtime_r(&now.tv_sec, &tim); + ::snprintf(format, sizeof(format), + "{\"@timestamp\":\"%%Y-%%m-%%dT%%H:%%M:%%S.%03ld%%z\",\"message\":\"", + long(now.tv_usec/1000)); + len = ::strftime(output_buffer,sizeof(output_buffer), format, &tim); if ( len < sizeof(output_buffer) ) { mx = std::min(sizeof(output_buffer)-len, data.first); ::memcpy(output_buffer+len, data.second, mx); diff --git a/Online/OnlineBase/OnlineBase/MBM/bmdef.h b/Online/OnlineBase/OnlineBase/MBM/bmdef.h index ad88fbdf4..27ba5f6ce 100755 --- a/Online/OnlineBase/OnlineBase/MBM/bmdef.h +++ b/Online/OnlineBase/OnlineBase/MBM/bmdef.h @@ -173,6 +173,9 @@ extern "C" { int mbm_wait_space_a(BMID bm); int mbm_cancel_request (BMID bm); + int mbm_set_cancelled (BMID bm); + int mbm_is_cancelled (BMID bm); + int mbm_stop_consumer (BMID bm); int mbm_stop_producer (BMID bm); int mbm_grant_update (BMID bm); diff --git a/Online/OnlineBase/src/MBM/mbmlib_client.cpp b/Online/OnlineBase/src/MBM/mbmlib_client.cpp index 3462dc39a..7f6a3b352 100755 --- a/Online/OnlineBase/src/MBM/mbmlib_client.cpp +++ b/Online/OnlineBase/src/MBM/mbmlib_client.cpp @@ -485,6 +485,17 @@ int mbm_cancel_request (BMID bm) { return bm->communication.send_request_server(msg,false); } +int mbm_set_cancelled (BMID bm) { + MBM_CHECK_BMID(bm); + bm->cancelled = true; + return MBM_NORMAL; +} + +int mbm_is_cancelled (BMID bm) { + MBM_CHECK_BMID(bm); + return bm->cancelled; +} + // Consumer routines int mbm_add_req (BMID bm, int evtype, const unsigned int* trg_mask, diff --git a/Online/SmiController/src/SmiController.cpp b/Online/SmiController/src/SmiController.cpp index 3e8f791d0..5583639af 100644 --- a/Online/SmiController/src/SmiController.cpp +++ b/Online/SmiController/src/SmiController.cpp @@ -411,7 +411,7 @@ SmiController::TaskProxy::TaskProxy(SmiController* ctrl, const std::string& nam, { string proc = RTL::processName(); const SmiController::config_t& config = ctrl->config; -#if 0 +#if 1 m_tms_dic_dns_ID = ctrl->m_tms_dic_dns_ID; m_local_dic_dns_ID = ctrl->m_local_dic_dns_ID; m_local_dis_dns_ID = ctrl->m_local_dis_dns_ID; @@ -883,7 +883,7 @@ void SmiController::initialize() { m_fsm_tags_ID = ::dis_add_service_dns(m_local_dis_dns_ID, (nam+"/instances").c_str(), "C",0,0,dim_handlers::feed_std_string,(long)&m_instance_info); - m_log->info("DNS dic id: %ld TMS dic id: %ld SMI dic ID: %ld SMI dis id: %ld", + m_log->warning("DNS dic id: %lx TMS dic id: %lx SMI dic ID: %lx SMI dis id: %lx", m_local_dic_dns_ID,m_tms_dic_dns_ID,m_smi_dic_dns_ID,m_smi_dis_dns_ID); if ( config.num_workers < 1 ) { m_log->info("Invalid initial instance count:%d -> forced to 1",config.num_workers); @@ -918,6 +918,7 @@ void SmiController::initialize() { m_defTask->setState(State::OFFLINE); m_vipTask->setState(State::OFFLINE); } + //config.name = RTL::str_upper(string(m_self->getDomain()) + "::Controller"); ::dis_start_serving_dns(m_local_dis_dns_ID, config.name.c_str()); this_thread::sleep_for(chrono::milliseconds(200)); m_inited = true; @@ -1109,7 +1110,7 @@ bool SmiController::attach_tasks(std::map<std::string, SmiTask*>&& tasks) { proxy->task->add_fmc_args("-DDIM_DNS_NODE="+RTL::str_upper(config.dns)); proxy->task->add_fmc_args("-DDIM_DNS_HOST="+RTL::str_upper(RTL::nodeNameShort())); } - ::dis_start_serving_dns(m_local_dis_dns_ID, config.name.c_str()); + //::dis_start_serving_dns(m_local_dis_dns_ID, config.name.c_str()); this_thread::sleep_for(chrono::milliseconds(200)); return true; } diff --git a/Online/dim/src/open_dns.c b/Online/dim/src/open_dns.c index 0b6420331..f976cc169 100755 --- a/Online/dim/src/open_dns.c +++ b/Online/dim/src/open_dns.c @@ -257,6 +257,8 @@ dim_long dis_add_dns(char *node_name, int port_number) connp->connecting = 0; dll_insert_queue( (DLL *) DNS_conn_head, (DLL *) connp ); } + if (connp == DNS_ids[SRC_DIS]) + return (dim_long)0; return (dim_long)connp; } @@ -278,6 +280,8 @@ dim_long dic_add_dns(char *node_name, int port_number) connp->connecting = 0; dll_insert_queue( (DLL *) DNS_conn_head, (DLL *) connp ); } + if (connp == DNS_ids[SRC_DIS]) + return (dim_long)0; return (dim_long)connp; } -- GitLab From 4c0a5d9c15b78b93c4c4b4b986418c531c12a3bc Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Thu, 7 Oct 2021 10:38:21 +0200 Subject: [PATCH 2/5] Fixes to dim, mbm, controller --- Online/Dataflow/Dataflow/MBMClient.h | 2 ++ Online/Dataflow/src/framework/DiskReader.cpp | 1 + Online/Dataflow/src/framework/MBMClient.cpp | 11 +++++++++++ 3 files changed, 14 insertions(+) diff --git a/Online/Dataflow/Dataflow/MBMClient.h b/Online/Dataflow/Dataflow/MBMClient.h index 48c3624ae..303a66448 100755 --- a/Online/Dataflow/Dataflow/MBMClient.h +++ b/Online/Dataflow/Dataflow/MBMClient.h @@ -120,6 +120,8 @@ namespace Online { int connectBuffers(); /// Cancel MBM connections int cancelBuffers(); + /// Queue cancel MBM connections + int requestCancelBuffers(); /// Access an existing buffer manager inclusion BMID access(const std::string& buffer); /// Include into the buffer manager. To use you should know what you are doing. diff --git a/Online/Dataflow/src/framework/DiskReader.cpp b/Online/Dataflow/src/framework/DiskReader.cpp index 6201b74e1..ed448460e 100644 --- a/Online/Dataflow/src/framework/DiskReader.cpp +++ b/Online/Dataflow/src/framework/DiskReader.cpp @@ -381,6 +381,7 @@ int DiskReader::cancel() { m_goValue = GO_DONT_PROCESS; m_receiveEvts = false; if (context.mbm) { + context.mbm->requestCancelBuffers(); //context.mbm->cancelBuffers(); } info("Executed cancellation of pending I/O requests."); diff --git a/Online/Dataflow/src/framework/MBMClient.cpp b/Online/Dataflow/src/framework/MBMClient.cpp index 56afe1a4e..42b54e174 100755 --- a/Online/Dataflow/src/framework/MBMClient.cpp +++ b/Online/Dataflow/src/framework/MBMClient.cpp @@ -168,6 +168,17 @@ int MBMClient::cancelBuffers() { return DF_SUCCESS; } +/// Cancel MBM connections +int MBMClient::requestCancelBuffers() { + if ( !m_inhibitCancel ) { + for(auto bmid : m_bmIDs) { + if ( bmid != MBM_INV_DESC ) ::mbm_set_cancelled(bmid); + } + info("Cancelled pending MBM I/O requests."); + } + return DF_SUCCESS; +} + /// Include into buffer manager as a client BMID MBMClient::access(const std::string& buffer) { auto i = m_buffMap.find(buffer); -- GitLab From 3d17adcbb164f224d0f95ae97d29218fc84ba74f Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Fri, 8 Oct 2021 14:44:53 +0200 Subject: [PATCH 3/5] Fix to MBM --- Online/Dataflow/src/Storage/StorageReader.cpp | 66 ++++++++++++------- Online/Dataflow/src/framework/DiskReader.cpp | 10 ++- Online/Dataflow/src/framework/MBMClient.cpp | 8 ++- Online/FarmConfig/job/BU.sh | 4 +- Online/FarmConfig/job/Controller.sh | 8 --- Online/FarmConfig/job/EBPartPublisher.sh | 12 ++++ Online/FarmConfig/job/EBStorage.sh | 12 ++++ Online/FarmConfig/job/Passthrough.py | 8 ++- Online/FarmConfig/job/runTask.sh | 2 +- Online/FarmConfig/options/EBStorageNFS.opts | 19 ++++++ Online/FarmConfig/options/HLT2Reader.opts | 12 ++-- Online/FarmConfig/options/HLT2Writer.opts | 6 +- .../GaudiOnline/components/OnlineEventApp.cpp | 3 +- Online/OnlineBase/OnlineBase/MBM/bmdef.h | 2 +- Online/OnlineBase/src/MBM/mbmlib_client.cpp | 20 +++--- Online/OnlineKernel/src/MBM/Producer.cpp | 15 ++++- Online/RawBankSizes/options/EventSizeMon.py | 19 +++--- Online/SmiController/src/SmiController.cpp | 4 +- Online/SmiController/src/TasklistHandlers.cpp | 3 +- Online/Storage/Storage/fdb_client.h | 4 +- Online/Storage/src/client/fdb_client.cpp | 6 +- Online/Storage/src/server/fdb_db_server.cpp | 22 +++---- Online/Storage/src/server/fdb_fs_server.cpp | 32 ++++----- 23 files changed, 190 insertions(+), 107 deletions(-) create mode 100755 Online/FarmConfig/job/EBPartPublisher.sh create mode 100755 Online/FarmConfig/job/EBStorage.sh create mode 100644 Online/FarmConfig/options/EBStorageNFS.opts diff --git a/Online/Dataflow/src/Storage/StorageReader.cpp b/Online/Dataflow/src/Storage/StorageReader.cpp index fc9cc3928..30625c065 100644 --- a/Online/Dataflow/src/Storage/StorageReader.cpp +++ b/Online/Dataflow/src/Storage/StorageReader.cpp @@ -118,10 +118,14 @@ namespace Online { StorageReader(const std::string& nam, Context& ctxt); /// Default destructor virtual ~StorageReader() = default; - /// IService implementation: start the service + /// DiskReader implementation: initialize the service + virtual int initialize() override; + /// DiskReader implementation: start the service virtual int start() override; - /// IService implementation: stop the service + /// DiskReader implementation: stop the service virtual int stop() override; + /// DiskReader implementation: finalize the service + virtual int finalize() override; /// Cancel I/O operations of the dataflow component virtual int cancel() override; /// Stop gracefully execution of the dataflow component @@ -187,24 +191,46 @@ StorageReader::StorageReader(const string& nam, Context& ctxt) ROOT::EnableThreadSafety(); } -/// IService implementation: start the service +/// DiskReader implementation: initialize the service +int StorageReader::initialize() { + if ( this->DiskReader::initialize() != DF_SUCCESS ) + return this->error("Failed to initialize DiskReader base class."); + if ( this->m_num_buffers > 0 ) { + for(size_t i = 0; i < this->m_num_buffers; ++i) + this->m_free.emplace_back(Buffer()); + } + return DF_SUCCESS; +} + +/// DiskReader implementation: finalize the service +int StorageReader::finalize() { + if ( this->DiskReader::finalize() != DF_SUCCESS ) + return this->error("Failed to finalize DiskReader base class."); + this->m_free.clear(); + return DF_SUCCESS; +} + +/// DiskReader implementation: start the service int StorageReader::start() { if ( this->DiskReader::start() != DF_SUCCESS ) - return error("Failed to start service base class."); + return this->error("Failed to start DiskReader base class."); this->m_todo.clear(); - this->m_is_reading = 0; + this->m_is_reading = 0; + this->m_eventsIN = 0; this->m_loadEnabled = true; - if ( m_num_buffers > 0 ) { - for(size_t i = 0; i < m_num_buffers; ++i) - m_free.emplace_back(Buffer()); - for(size_t i = 0; i < m_num_threads; ++i) - m_threads.emplace_back(make_unique<thread>([this]() { this->async_load_buffers(); })); + this->m_receiveEvts = true; + this->m_goto_paused = false; + if ( this->m_num_buffers > 0 ) { + for(size_t i = 0; i < this->m_num_threads; ++i) + this->m_threads.emplace_back(make_unique<thread>([this]() { this->async_load_buffers(); })); } return DF_SUCCESS; } -/// IService implementation: stop the service +/// DiskReader implementation: stop the service int StorageReader::stop() { + this->m_loadEnabled = false; + this->m_receiveEvts = false; while ( !this->m_todo.empty() ) { } for(auto& t : this->m_threads) { @@ -212,15 +238,11 @@ int StorageReader::stop() { t.reset(); } this->m_threads.clear(); - this->m_free.clear(); return DiskReader::stop(); } /// Stop gracefully execution of the dataflow component int StorageReader::pause() { - this->m_goto_paused = true; - this->m_goValue = GO_DONT_PROCESS; - this->m_receiveEvts = false; this->m_loadEnabled = false; while ( this->m_free.size() != this->m_num_buffers ) { ::lib_rtl_sleep(100); @@ -413,9 +435,7 @@ void StorageReader::async_load_buffers() { } if ( !got_buffer ) { - if ( !this->m_receiveEvts ) - return; - else if ( this->m_goto_paused ) + if ( !this->m_receiveEvts || this->m_goto_paused || !this->m_loadEnabled ) return; ::lib_rtl_sleep(m_poll_tmo); continue; @@ -434,6 +454,7 @@ void StorageReader::async_load_buffers() { } else { this->m_goto_paused = true; + return; } --this->m_is_reading ; ::lib_rtl_sleep(m_poll_tmo); @@ -471,7 +492,7 @@ int StorageReader::open_file(Buffer& buffer) { while( 1 ) {{ lock_guard<mutex> lock(this->m_bufferLock); if ( m_todo.empty() ) { - if ( !this->m_receiveEvts && 0 == this->m_is_reading ) { + if ( 0 == this->m_is_reading && !this->m_receiveEvts ) { return DF_ERROR; } } @@ -498,10 +519,6 @@ int StorageReader::i_run() { RawFile current_input; auto* mbmInfo = m_mbmInfo[m_buffer]; - this->m_eventsIN = 0; - this->m_goto_paused = false; - this->m_receiveEvts = true; - status = this->waitForGo(); if ( status != DF_CONTINUE ) { return status; @@ -523,8 +540,7 @@ int StorageReader::i_run() { ::usleep(this->m_muDelay); } if ( !(current_input.isMapped() || current_input.isOpen()) ) { - int runno = 0; - this->updateRunNumber(runno); + this->updateRunNumber(0); status = this->open_file(current_buffer); if ( status == DF_ERROR && this->m_goto_paused ) { // Sleep a bit before goung to pause diff --git a/Online/Dataflow/src/framework/DiskReader.cpp b/Online/Dataflow/src/framework/DiskReader.cpp index ed448460e..2c406299c 100644 --- a/Online/Dataflow/src/framework/DiskReader.cpp +++ b/Online/Dataflow/src/framework/DiskReader.cpp @@ -339,8 +339,8 @@ int DiskReader::stop() { /// Stop gracefully execution of the dataflow component int DiskReader::pause() { m_goto_paused = true; - m_goValue = GO_DONT_PROCESS; m_receiveEvts = false; + m_goValue = GO_DONT_PROCESS; if ( context.mbm ) { // This is not terribly elegant, but we have to stop the DIM callback // and wait until the pipeline is empty.... @@ -773,7 +773,10 @@ int DiskReader::declare_mbm_data(MBM::BufferInfo* mbm_info, const char* fname, c info("UNKNOWN Exception while delareEvent: File:%s.", fname); status = MBM_ERROR; } - if (status != MBM_NORMAL) { + if (status == MBM_REQ_CANCEL ) { + return DF_SUCCESS; + } + else if (status != MBM_NORMAL) { return DF_CONTINUE; } // @@ -790,6 +793,9 @@ int DiskReader::declare_mbm_data(MBM::BufferInfo* mbm_info, const char* fname, c info("UNKNOWN Exception while sendSpace: File:%s.", fname); status = MBM_ERROR; } + if (status == MBM_REQ_CANCEL ) { + return DF_SUCCESS; + } } if (status != MBM_NORMAL) { return DF_CONTINUE; diff --git a/Online/Dataflow/src/framework/MBMClient.cpp b/Online/Dataflow/src/framework/MBMClient.cpp index 42b54e174..40921d7aa 100755 --- a/Online/Dataflow/src/framework/MBMClient.cpp +++ b/Online/Dataflow/src/framework/MBMClient.cpp @@ -66,7 +66,11 @@ int MBMClient::start() { return error("Failed to initialize base class Service."); if ( !m_enabled ) context.mbm = nullptr; - return (context.mbm && m_connectWhen == "start") ? i_init() : DF_SUCCESS; + int sc = (context.mbm && m_connectWhen == "start") ? i_init() : DF_SUCCESS; + for(auto bmid : m_bmIDs) { + if ( bmid != MBM_INV_DESC ) ::mbm_set_cancelled(bmid, 0); + } + return sc; } /// Finalize the MBM client @@ -172,7 +176,7 @@ int MBMClient::cancelBuffers() { int MBMClient::requestCancelBuffers() { if ( !m_inhibitCancel ) { for(auto bmid : m_bmIDs) { - if ( bmid != MBM_INV_DESC ) ::mbm_set_cancelled(bmid); + if ( bmid != MBM_INV_DESC ) ::mbm_set_cancelled(bmid, 1); } info("Cancelled pending MBM I/O requests."); } diff --git a/Online/FarmConfig/job/BU.sh b/Online/FarmConfig/job/BU.sh index 9e72a27d4..fcb73e7df 100755 --- a/Online/FarmConfig/job/BU.sh +++ b/Online/FarmConfig/job/BU.sh @@ -13,8 +13,8 @@ unset PYTHONPATH; unset PYTHONHOME; eval `/usr/bin/python2 -c "import os;s=os.environ['UTGID'];print 'export BU_OPTIONS='+s[s.find('BU'):]+'.opts'"`; #. /group/online/dataflow/EventBuilder/EventBuilderRelease/setup.x86_64-centos7-gcc9-opt.vars; -#. /group/online/dataflow/EventBuilder/EventBuilderRelease/setup.x86_64_v2-centos7-gcc10-do0.vars; -. /group/online/dataflow/EventBuilder/EventBuilderRelease/setup.x86_64_v2-centos7-gcc10-opt.vars; +. /group/online/dataflow/EventBuilder/EventBuilderRelease/setup.x86_64_v2-centos7-gcc10-do0.vars; +#. /group/online/dataflow/EventBuilder/EventBuilderRelease/setup.x86_64_v2-centos7-gcc10-opt.vars; # . /group/online/dataflow/EventBuilder/EventBuilderRelease/setup.${CMTCONFIG}.vars; cd ${FARMCONFIGROOT}/job; # diff --git a/Online/FarmConfig/job/Controller.sh b/Online/FarmConfig/job/Controller.sh index 0a93d9c24..165936e78 100755 --- a/Online/FarmConfig/job/Controller.sh +++ b/Online/FarmConfig/job/Controller.sh @@ -43,14 +43,6 @@ DEBUG_ARGS=; # DEBUG_ARGS="-debug"; # echo "${UTGID} [ERROR] ${HOST}"; SMI_DEBUG=0; -if test "${HOST}" = "HLT20101"; then - SMI_DEBUG=1; -fi; -if test "${HOST}" = "HLT20102"; then -# cd /group/online/dataflow/cmtuser/OnlineDev_v7r11; -# . setup.x86_64_v2-centos7-gcc10-do0.vars; - SMI_DEBUG=1; -fi; if test "${HOST}" = "TDEB03"; then # cd /group/online/dataflow/cmtuser/OnlineDev_v7r11; # . setup.x86_64_v2-centos7-gcc10-do0.vars; diff --git a/Online/FarmConfig/job/EBPartPublisher.sh b/Online/FarmConfig/job/EBPartPublisher.sh new file mode 100755 index 000000000..a40a1f02c --- /dev/null +++ b/Online/FarmConfig/job/EBPartPublisher.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# ========================================================================= +# +# Default script to start the event reader task on the HLT farm +# +# Author M.Frank +# Version: 1.0 +# Date: 20/05/2013 +# +# ========================================================================= +export ADDER_TYPE=${TASK_TYPE}; +exec -a ${UTGID} genPython.exe `which gaudirun.py` ./AddersFromArchitecture.py --application=Online::OnlineApplication; diff --git a/Online/FarmConfig/job/EBStorage.sh b/Online/FarmConfig/job/EBStorage.sh new file mode 100755 index 000000000..bdf20c369 --- /dev/null +++ b/Online/FarmConfig/job/EBStorage.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# ========================================================================= +# +# Default script to start the buffer manager on the HLT farm worker node +# +# Author M.Frank +# Version: 1.0 +# Date: 05/03/2021 +# +# ========================================================================= +# +execute `dataflow_task Class1` -opts=${STATIC_OPTS}/${TASK_TYPE}.opts ${AUTO_STARTUP} ${DEBUG_STARTUP}; diff --git a/Online/FarmConfig/job/Passthrough.py b/Online/FarmConfig/job/Passthrough.py index d5a68281c..dd444772b 100644 --- a/Online/FarmConfig/job/Passthrough.py +++ b/Online/FarmConfig/job/Passthrough.py @@ -33,10 +33,16 @@ writer.UseRawData = False writer.RequireODIN = False ev_size = Configurables.Online__EventSize('EventSize') + +explorer = Configurables.StoreExplorerAlg('Explorer') +explorer.Load = 1 +explorer.PrintFreq = 0.01 +explorer.OutputLevel = 1 + application.setup_hive(FlowManager("EventLoop"), 40) application.setup_monitoring() #application.setup_algorithms(writer, 0.05) -application.setup_algorithms([ev_size,writer], 1.0) +application.setup_algorithms([ev_size,explorer,writer], 1.0) application.monSvc.DimUpdateInterval = 1 application.config.burstPrintCount = 30000 diff --git a/Online/FarmConfig/job/runTask.sh b/Online/FarmConfig/job/runTask.sh index 8fd1d760f..13272d8fa 100755 --- a/Online/FarmConfig/job/runTask.sh +++ b/Online/FarmConfig/job/runTask.sh @@ -76,7 +76,7 @@ elif test -f "${STATIC_OPTS}/${TASK_TYPE}.py"; then execute `gaudi_task ${STATIC_OPTS}/${TASK_TYPE}.py`; # elif test -n "`echo ${UTGID} | grep TestEvent`"; then - execute `gaudi_event_task ${SMICONTROLLERROOT}/scripts/EventProcessor.py`; + execute `gaudi_event_task ${FARMCONFIGROOT}/job/PassThrough.py`; # elif test -f ./${PARTITION_NAME}DefaultTask.sh; then execute . ./${PARTITION_NAME}DefaultTask.sh; diff --git a/Online/FarmConfig/options/EBStorageNFS.opts b/Online/FarmConfig/options/EBStorageNFS.opts new file mode 100644 index 000000000..610b47dd1 --- /dev/null +++ b/Online/FarmConfig/options/EBStorageNFS.opts @@ -0,0 +1,19 @@ +#include "$DATAFLOWROOT/options/StorageWriter.opts" +Monitoring.CounterUpdateInterval = 3; +MBM.ConnectWhen = "initialize"; +// +Writer.Server = "XXEB09.lbdaq.cern.ch:8000"; +// Writer.Server = "devbbdb01.lbdaq.cern.ch:4242"; +Writer.FDBVersion = 0; +EventProc.REQ2 = "EvType=2;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=VIP;Frequency=PERC;Perc=100.0"; +// +// NFS writing +Writer.IdleTimeout = 30; +Writer.HaveFileDB = 2; +Writer.NumThreads = 2; +Writer.NumBuffers = 4; +Writer.MaxFileSizeMB = 4100; +Writer.BufferSizeMB = 1024; +Writer.FileName = "/hlt2/objects/${PARTITION}/${RUN}/Run_${RUN}_${TIME}_${NODE}.mdf"; +Writer.OutputType = "POSIX"; + diff --git a/Online/FarmConfig/options/HLT2Reader.opts b/Online/FarmConfig/options/HLT2Reader.opts index b078116f1..732b457d5 100644 --- a/Online/FarmConfig/options/HLT2Reader.opts +++ b/Online/FarmConfig/options/HLT2Reader.opts @@ -5,8 +5,8 @@ #include "$FARMCONFIGROOT/options/Monitoring.opts" // Manager.Services = {"Dataflow_MBMClient/MEPManager", - "Dataflow_BurstReader/Reader", -// "Dataflow_StorageReader/Reader", +// "Dataflow_BurstReader/Reader", + "Dataflow_StorageReader/Reader", "Dataflow_RunableWrapper/Wrap", "Dataflow_UI/UI" }; @@ -18,6 +18,7 @@ MEPManager.PartitionBuffers = true; MEPManager.PartitionName = @OnlineEnv.PartitionName; MEPManager.PartitionID = @OnlineEnv.PartitionID; MEPManager.Buffers = {"Events"};//@OnlineEnv.HLT2Reader_Buffers; +MEPManager.InhibitCancel = true; // Reader.Buffer = "Events";//@OnlineEnv.Hlt2Reader_Output; Reader.BrokenHosts = ""; @@ -62,14 +63,11 @@ Reader.DataType = "posix"; Reader.Server = "XXEB09.lbdaq.cern.ch:8000"; */ // -/* Reader.DataType = "network"; Reader.Server = "devbbdb01.lbdaq.cern.ch:4242"; Reader.FDBVersion = 1; Reader.PartitionName = "FEST"; Reader.FilePrefix = "${PARTITION}/${RUN}/"; -Reader.AllowedRuns = { "219722", "219723", "219724", "219725", "299999" }; -Reader.AllowedRuns = { "219722", "299999" }; -Reader.NumThreads = 1; +Reader.AllowedRuns = { "219799", "219800", "219810", "219811", "219812", "219813", "219816", "219817", "219818", "219819", "219820" }; +Reader.NumThreads = 2; Reader.NumBuffers = 3; -*/ diff --git a/Online/FarmConfig/options/HLT2Writer.opts b/Online/FarmConfig/options/HLT2Writer.opts index 2732f2843..3358bcc87 100644 --- a/Online/FarmConfig/options/HLT2Writer.opts +++ b/Online/FarmConfig/options/HLT2Writer.opts @@ -19,12 +19,12 @@ EventProc.REQ2 = "EvType=2;TriggerMask=0xffffffff,0xffffffff,0xfffffff Writer.PartitionName = "FEST"; Writer.ThreadFileQueues = false; Writer.IdleTimeout = 30; -Writer.HaveFileDB = 2; Writer.NumThreads = 2; Writer.NumBuffers = 3; Writer.MaxFileSizeMB = 4000; Writer.BufferSizeMB = 32; -Writer.Server = "XXEB09.lbdaq.cern.ch:8100"; -Writer.FileName = "file:/daqarea1/objects/nfs_data/${PARTITION}/${RUN}/Run_${RUN}_${NODE}_${TIME}_${SEQ}.mdf"; +Writer.HaveFileDB = 2; +Writer.FDBVersion = 0; +Writer.Server = "XXEB09.lbdaq.cern.ch:8010"; Writer.FileName = "/hlt2/HLT2/${PARTITION}/${RUN}/Run_${RUN}_${NODE}_${TIME}_${SEQ}.mdf"; Writer.OutputType = "POSIX"; diff --git a/Online/GaudiOnline/components/OnlineEventApp.cpp b/Online/GaudiOnline/components/OnlineEventApp.cpp index 21fdddb94..9f344af02 100644 --- a/Online/GaudiOnline/components/OnlineEventApp.cpp +++ b/Online/GaudiOnline/components/OnlineEventApp.cpp @@ -340,8 +340,9 @@ OnlineEventApp::access_t OnlineEventApp::start_file_access() { /// Pause the application (RUNNING -> READY) int OnlineEventApp::pauseProcessing() { + // Noop: we want to empty the event pipeline and run until a "stop" appears.... m_logger->debug("+++ Pause the application."); - m_halt = EVENTLOOP_PAUSE; + // m_halt = EVENTLOOP_PAUSE; return OnlineApplication::pauseProcessing(); } diff --git a/Online/OnlineBase/OnlineBase/MBM/bmdef.h b/Online/OnlineBase/OnlineBase/MBM/bmdef.h index 27ba5f6ce..41c30fbf6 100755 --- a/Online/OnlineBase/OnlineBase/MBM/bmdef.h +++ b/Online/OnlineBase/OnlineBase/MBM/bmdef.h @@ -173,7 +173,7 @@ extern "C" { int mbm_wait_space_a(BMID bm); int mbm_cancel_request (BMID bm); - int mbm_set_cancelled (BMID bm); + int mbm_set_cancelled (BMID bm, int value); int mbm_is_cancelled (BMID bm); int mbm_stop_consumer (BMID bm); diff --git a/Online/OnlineBase/src/MBM/mbmlib_client.cpp b/Online/OnlineBase/src/MBM/mbmlib_client.cpp index 7f6a3b352..c24d72f8b 100755 --- a/Online/OnlineBase/src/MBM/mbmlib_client.cpp +++ b/Online/OnlineBase/src/MBM/mbmlib_client.cpp @@ -485,9 +485,9 @@ int mbm_cancel_request (BMID bm) { return bm->communication.send_request_server(msg,false); } -int mbm_set_cancelled (BMID bm) { +int mbm_set_cancelled (BMID bm, int value) { MBM_CHECK_BMID(bm); - bm->cancelled = true; + bm->cancelled = (value != 0); return MBM_NORMAL; } @@ -652,8 +652,8 @@ int mbm_get_space_try(BMID bm, long size, int** ptr, RTL_ast_t astadd, void* ast LOCK lck (bm->lock); if ( bm->cancelled ) { *bm->evt_ptr = 0; - *bm->evt_size = 0; - *bm->evt_type = 0; + bm->ast_addr = nullptr; + bm->ast_param = nullptr; return MBM_REQ_CANCEL; } } @@ -671,20 +671,20 @@ int mbm_get_space_try(BMID bm, long size, int** ptr, RTL_ast_t astadd, void* ast } else if ( MBM_NO_ROOM == status ) { *bm->evt_ptr = 0; - *bm->evt_size = 0; - *bm->evt_type = 0; + bm->ast_addr = nullptr; + bm->ast_param = nullptr; //bm->cancelled = false; } else if ( MBM_REQ_CANCEL == status ) { *bm->evt_ptr = 0; - *bm->evt_size = 0; - *bm->evt_type = 0; + bm->ast_addr = nullptr; + bm->ast_param = nullptr; //bm->cancelled = false; } else if ( MBM_ILL_LEN == status ) { *bm->evt_ptr = 0; - *bm->evt_size = 0; - *bm->evt_type = 0; + bm->ast_addr = nullptr; + bm->ast_param = nullptr; //bm->cancelled = false; } else { diff --git a/Online/OnlineKernel/src/MBM/Producer.cpp b/Online/OnlineKernel/src/MBM/Producer.cpp index 71c29bb23..4aa44f99e 100755 --- a/Online/OnlineKernel/src/MBM/Producer.cpp +++ b/Online/OnlineKernel/src/MBM/Producer.cpp @@ -143,6 +143,9 @@ int MBM::Producer::declareEvent() { e.len = flen; return MBM_NORMAL; } + else if ( sc == MBM_REQ_CANCEL ) { + return sc; + } throw std::runtime_error("Failed to declare event for MBM buffer:"+m_buffName+" [Internal Error]"); } throw std::runtime_error("Failed to declare event for MBM buffer:"+m_buffName+" [Buffer not connected]"); @@ -174,9 +177,13 @@ int MBM::Producer::declareEventTry() { // Action to be called on space receival int MBM::Producer::sendSpace() { if ( m_bmid != MBM_INV_DESC ) { - if ( ::mbm_send_space(m_bmid) == MBM_NORMAL ) { + int sc = ::mbm_send_space(m_bmid); + if ( sc == MBM_NORMAL ) { return MBM_NORMAL; } + else if ( sc == MBM_REQ_CANCEL ) { + return sc; + } throw std::runtime_error("Failed to send space for MBM buffer:"+m_buffName+" [Internal Error]"); } throw std::runtime_error("Failed to declare event for MBM buffer:"+m_buffName+" [Buffer not connected]"); @@ -202,8 +209,14 @@ int MBM::Producer::spaceRearm(int new_length) { if ( sc == MBM_NORMAL || sc == MBM_REQ_CANCEL ) { return sc; } + else if ( sc == MBM_REQ_CANCEL ) { + return sc; + } throw std::runtime_error("Failed to wait space for MBM buffer:"+m_buffName+" [Internal Error]"); } + else if ( sc == MBM_REQ_CANCEL ) { + return sc; + } throw std::runtime_error("Failed to get event for MBM buffer:"+m_buffName+" [Internal Error]"); } throw std::runtime_error("Failed to declare event for MBM buffer:"+m_buffName+" [Buffer not connected]"); diff --git a/Online/RawBankSizes/options/EventSizeMon.py b/Online/RawBankSizes/options/EventSizeMon.py index d561cbf4c..675f72dae 100644 --- a/Online/RawBankSizes/options/EventSizeMon.py +++ b/Online/RawBankSizes/options/EventSizeMon.py @@ -27,15 +27,16 @@ class EventSizeMon(GaudiOnline.Application): def setup_algorithms(self): import Gaudi.Configuration as Gaudi import Configurables - input = self.setup_event_input() - monitor = Configurables.Online__EventSize('EventSize') - monitor.RawData = '/Event/Banks/RawData' - monitor.Bins = 100 - monitor.Low = 0.0 - monitor.High = 200e3 - sequence = Gaudi.GaudiSequencer('Processor') - sequence.Members = [input, monitor, self.updateAndReset] - self.app.TopAlg = [sequence] + input = self.setup_event_input() + input.MakeRawEvent = 1 + monitor = Configurables.Online__EventSize('EventSize') + monitor.RawData = '/Event/Banks/RawData' + monitor.Bins = 100 + monitor.Low = 0.0 + monitor.High = 200e3 + sequence = Gaudi.GaudiSequencer('Processor') + sequence.Members = [input, monitor, self.updateAndReset] + self.app.TopAlg = [sequence] self.broker.DataProducers = self.app.TopAlg return self diff --git a/Online/SmiController/src/SmiController.cpp b/Online/SmiController/src/SmiController.cpp index 5583639af..925c8f573 100644 --- a/Online/SmiController/src/SmiController.cpp +++ b/Online/SmiController/src/SmiController.cpp @@ -584,8 +584,8 @@ void SmiController::TaskProxy::handle(const CPP::Event& event) { //============================================================================== /// Start the dependent task's transition timeout void SmiController::TaskProxy::start_timer(int reason, const std::string& transition) { - auto i = task->timeouts.find(transition); - if ( i == task->timeouts.end() ) i = task->timeouts.find("Any"); + auto i = task->timeouts.find(RTL::str_lower(transition)); + if ( i == task->timeouts.end() ) i = task->timeouts.find("any"); int tmo = i==task->timeouts.end() ? 5 : (*i).second; if ( m_timer_id.first ) ::dtq_stop_timer(m_timer_id.first); m_timer_id = TimerID(this,reason); diff --git a/Online/SmiController/src/TasklistHandlers.cpp b/Online/SmiController/src/TasklistHandlers.cpp index 5187b42db..02204587c 100644 --- a/Online/SmiController/src/TasklistHandlers.cpp +++ b/Online/SmiController/src/TasklistHandlers.cpp @@ -20,6 +20,7 @@ #include "SmiController/TasklistHandlers.h" #include "XML/XML.h" #include "RTL/rtl.h" +#include "RTL/strdef.h" // C/C++ include files #include <iostream> @@ -152,7 +153,7 @@ void TasklistAnalyzer::Args::operator()(const xml_h& h) { /// Action operator when analyzing data void TasklistAnalyzer::Timeouts::operator()(const xml_h& h) { TaskParams a = h; - task->timeouts.push_back(Tasklist::Timeout(a.action(),a.timeout())); + task->timeouts.push_back(Tasklist::Timeout(RTL::str_lower(a.action()),a.timeout())); } /// Action operator when analyzing data diff --git a/Online/Storage/Storage/fdb_client.h b/Online/Storage/Storage/fdb_client.h index 4fed9c1d3..fff559456 100644 --- a/Online/Storage/Storage/fdb_client.h +++ b/Online/Storage/Storage/fdb_client.h @@ -77,9 +77,9 @@ namespace Online { /// Read AND Delete object data from disk at the specified location reply_t delete_object(const std::string& location, std::time_t& date, std::size_t& length); /// Read object data from the specified location without removing disk object - reply_t next_object_get(const std::string& location, std::time_t& date, std::size_t& length); + reply_t next_object_get(std::string& location, std::time_t& date, std::size_t& length); /// Read object data from the specified location with removing disk object - reply_t next_object_delete(const std::string& location, std::time_t& date, std::size_t& length); + reply_t next_object_delete(std::string& location, std::time_t& date, std::size_t& length); /// Helper: Delete the specified object data from the database reply_t db_object_delete(const std::string& prefix, std::time_t& date, std::size_t& length); diff --git a/Online/Storage/src/client/fdb_client.cpp b/Online/Storage/src/client/fdb_client.cpp index c3b1dcf94..233eab324 100644 --- a/Online/Storage/src/client/fdb_client.cpp +++ b/Online/Storage/src/client/fdb_client.cpp @@ -88,7 +88,7 @@ reply_t fdb_client::delete_object(const string& location, time_t& date, size_t& } /// Read AND Delete object data from disk at the specified location -reply_t fdb_client::next_object_get(const string& prefix, time_t& date, size_t& length) { +reply_t fdb_client::next_object_get(string& prefix, time_t& date, size_t& length) { string location; if ( version == 0 ) location = "/next?prefix="+prefix; else if ( version > 0 ) location = "/dataflow/next/"+prefix; @@ -109,6 +109,7 @@ reply_t fdb_client::next_object_get(const string& prefix, time_t& date, size_t& if ( hdr ) date = hdr->as_time(); hdr = reply.header(http::constants::content_length); if ( hdr ) length = hdr->as<size_t>(); + prefix = h.value; } return reply; } @@ -121,7 +122,7 @@ reply_t fdb_client::next_object_get(const string& prefix, time_t& date, size_t& } /// Read AND Delete object data from disk at the specified location -reply_t fdb_client::next_object_delete(const string& prefix, time_t& date, size_t& length) { +reply_t fdb_client::next_object_delete(string& prefix, time_t& date, size_t& length) { string location; if ( version == 0 ) location = "/next?prefix="+prefix; else if ( version > 0 ) location = "/dataflow/next/"+prefix; @@ -142,6 +143,7 @@ reply_t fdb_client::next_object_delete(const string& prefix, time_t& date, size_ if ( hdr ) date = hdr->as_time(); hdr = reply.header(http::constants::content_length); if ( hdr ) length = hdr->as<size_t>(); + prefix = h.value; } return reply; } diff --git a/Online/Storage/src/server/fdb_db_server.cpp b/Online/Storage/src/server/fdb_db_server.cpp index 307a853d9..e3ed80e12 100644 --- a/Online/Storage/src/server/fdb_db_server.cpp +++ b/Online/Storage/src/server/fdb_db_server.cpp @@ -121,7 +121,7 @@ http::basic_http_server<db>::handler_t::handle_get(const request_t& req, reply_t } if ( !error_code_ok(ec, this->debug) ) { header_t h(http::constants::error_cause,"Failed to get "+req.uri+" ["+ec.message()+"]"); - ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: %s %-20s = %s", + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %-6s %s %-20s = %s", req.method.c_str(), reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); rep.headers.emplace_back(move(h)); @@ -131,7 +131,7 @@ http::basic_http_server<db>::handler_t::handle_get(const request_t& req, reply_t bool mb = length > 3*MByte; rep = reply_t::stock_reply(reply_t::permanent_redirect); rep.headers.emplace_back(http::constants::location,access_name); - ::lib_rtl_output(LIB_RTL_INFO,"+++ %s: '%s' %.1f %cB [%s]", + ::lib_rtl_output(LIB_RTL_INFO,"+++ %-6s '%s' %.1f %cB [%s]", req.method.c_str(), access_name.c_str(), double(length)/(mb ? MByte : kByte), mb ? 'M' : 'k', req.remote_address().to_string().c_str()); @@ -165,7 +165,7 @@ http::basic_http_server<db>::handler_t::handle_delete(const request_t& req, repl } if ( !error_code_ok(ec, this->debug) ) { header_t h(http::constants::error_cause,"Failed to delete "+req.uri+" ["+ec.message()+"]"); - ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: %s %-20s = %s", + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %-6s %s %-20s = %s", req.method.c_str(), reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); rep.headers.emplace_back(move(h)); @@ -175,7 +175,7 @@ http::basic_http_server<db>::handler_t::handle_delete(const request_t& req, repl bool mb = length > 3*MByte; rep = reply_t::stock_reply(reply_t::permanent_redirect); rep.headers.emplace_back(http::constants::location,access_name); - ::lib_rtl_output(LIB_RTL_INFO,"+++ %s: '%s' %.1f %cB [%s]", + ::lib_rtl_output(LIB_RTL_INFO,"+++ %-6s '%s' %.1f %cB [%s]", req.method.c_str(), obj.c_str(), double(length)/(mb ? MByte : kByte), mb ? 'M' : 'k', req.remote_address().to_string().c_str()); @@ -192,7 +192,7 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t if ( (hdr_date=req.header(http::constants::date)) == nullptr ) { header_t h(http::constants::error_cause,"Missing date header"); rep = reply_t::stock_reply(reply_t::bad_request); - ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: bad_request %-20s = %s", + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %-6s bad_request %-20s = %s", req.method.c_str(), h.name.c_str(), h.value.c_str()); ++monitor->data.num_put_bad_request; rep.headers.emplace_back(move(h)); @@ -200,7 +200,7 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t else if ( (hdr_len=req.header(http::constants::content_length)) == nullptr ) { header_t h(http::constants::error_cause,"Missing content length header"); rep = reply_t::stock_reply(reply_t::bad_request); - ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: bad_request %-20s = %s", + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %-6s bad_request %-20s = %s", req.method.c_str(), h.name.c_str(), h.value.c_str()); ++monitor->data.num_put_bad_request; rep.headers.emplace_back(move(h)); @@ -216,7 +216,7 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t if ( ec == std::errc::permission_denied ) { header_t h(http::constants::error_cause,"Failed to add "+req.uri+" "+ec.message()); rep = reply_t::stock_reply(reply_t::unauthorized); - ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: %s %-20s = %s", + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %-6s %s %-20s = %s", req.method.c_str(), reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); ++monitor->data.num_put_unauthorized; @@ -226,7 +226,7 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t header_t h(http::constants::error_cause, "Failed to add object "+req.uri+" in dbase: "+ec.message()); rep = reply_t::stock_reply(reply_t::bad_request); - ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: %s %-20s = %s", + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %-6s %s %-20s = %s", req.method.c_str(), reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); ++monitor->data.num_put_bad_request; @@ -246,7 +246,7 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t rep.headers.emplace_back(http::constants::location,access_name); rep.headers.emplace_back(http::constants::data_length,len); rep.headers.emplace_back(http::constants::date,date); - ::lib_rtl_output(LIB_RTL_INFO,"+++ %s: '%s' %.1f %cB [%s]", + ::lib_rtl_output(LIB_RTL_INFO,"+++ %-6s '%s' %.1f %cB [%s]", req.method.c_str(), req.uri.c_str(), double(len)/(mb ? MByte : kByte), mb ? 'M' : 'k', remote.c_str()); @@ -265,7 +265,7 @@ http::basic_http_server<db>::handler_t::handle_update(const request_t& req, repl if ( (hdr_state=req.header(http::constants::state)) == nullptr ) { header_t h(http::constants::error_cause,"Missing state header"); rep = reply_t::stock_reply(reply_t::bad_request); - ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: bad_request %-20s = %s", + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %-6s bad_request %-20s = %s", req.method.c_str(), h.name.c_str(), h.value.c_str()); ++monitor->data.num_upda_bad_request; rep.headers.emplace_back(move(h)); @@ -279,7 +279,7 @@ http::basic_http_server<db>::handler_t::handle_update(const request_t& req, repl header_t h(http::constants::error_cause, "Failed to update object "+req.uri+" in dbase: "+ec.message()); rep = reply_t::stock_reply(reply_t::bad_request); - ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: %s %-20s = %s", + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %-6s %s %-20s = %s", req.method.c_str(), reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); ++monitor->data.num_upda_errors; diff --git a/Online/Storage/src/server/fdb_fs_server.cpp b/Online/Storage/src/server/fdb_fs_server.cpp index d2cc917e1..a91566afc 100644 --- a/Online/Storage/src/server/fdb_fs_server.cpp +++ b/Online/Storage/src/server/fdb_fs_server.cpp @@ -283,12 +283,12 @@ http::basic_http_server<fs>::handler_t::handle_get(const request_t& req, reply_t } else { std::string err = std::make_error_code(std::errc(errno)).message(); - ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read '%s' [%s] only got %ld out of %ld bytes", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %-6s Failed to read '%s' [%s] only got %ld out of %ld bytes", req.method.c_str(), req.uri.c_str(), err.c_str(), rd, len); } if ( ctxt->st_data == ctxt->st_size ) { - ::lib_rtl_output(LIB_RTL_INFO, "+++ %s: Last chunk '%s' %.1f MB", + ::lib_rtl_output(LIB_RTL_INFO, "+++ %-6s Last chunk '%s' %.1f MB", req.method.c_str(), req.uri.c_str(), double(ctxt->st_size)/(1024e0*1024e0)); } return write; @@ -298,7 +298,7 @@ http::basic_http_server<fs>::handler_t::handle_get(const request_t& req, reply_t long net = std::chrono::duration_cast<std::chrono::milliseconds>(req.netio-null).count(); long wrt = std::chrono::duration_cast<std::chrono::milliseconds>(req.handling-null).count(); double mb = double(ctxt->st_size)/(1024e0*1024e0); - ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %s: %ld bytes netio: %ld ms %.1f MB/s wrt: %ld ms %.1f MB/s", + ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %-6s %ld bytes netio: %ld ms %.1f MB/s wrt: %ld ms %.1f MB/s", req.method.c_str(), ctxt->st_size, net, mb/std::max(1e-6, double(net)/1e3), wrt, mb/std::max(1e-6, double(wrt)/1e3)); @@ -306,7 +306,7 @@ http::basic_http_server<fs>::handler_t::handle_get(const request_t& req, reply_t monitor->data.bytesGET += ctxt->st_data; return none; } - ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read '%s' size:%ld / %ld %s [Inconsistemt context]", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %-6s Failed to read '%s' size:%ld / %ld %s [Inconsistemt context]", req.method.c_str(), req.uri.c_str(), rep.bytes_sent, rep.bytes_total, rep.stock_status(rep.status).c_str()); rep.context.reset(); @@ -365,7 +365,7 @@ http::basic_http_server<fs>::handler_t::handle_delete(const request_t& req, repl } else { std::string err = std::make_error_code(std::errc(errno)).message(); - ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read '%s' [%s] only got %ld out of %ld bytes", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %-6s Failed to read '%s' [%s] only got %ld out of %ld bytes", req.method.c_str(), req.uri.c_str(), err.c_str(), rd, len); } if ( ctxt->st_data == ctxt->st_size ) { @@ -374,7 +374,7 @@ http::basic_http_server<fs>::handler_t::handle_delete(const request_t& req, repl ec = ctxt->unlink(fname.c_str()); if ( !error_code_ok(ec, this->debug) ) { } - ::lib_rtl_output(LIB_RTL_INFO, "+++ %s: Last chunk '%s' %.1f MB", + ::lib_rtl_output(LIB_RTL_INFO, "+++ %-6s Last chunk '%s' %.1f MB", req.method.c_str(), req.uri.c_str(), double(ctxt->st_size)/(1024e0*1024e0)); } return write; @@ -384,7 +384,7 @@ http::basic_http_server<fs>::handler_t::handle_delete(const request_t& req, repl long net = std::chrono::duration_cast<std::chrono::milliseconds>(req.netio-null).count(); long wrt = std::chrono::duration_cast<std::chrono::milliseconds>(req.handling-null).count(); double mb = double(ctxt->st_size)/(1024e0*1024e0); - ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %s: %ld bytes netio: %ld ms %.1f MB/s wrt: %ld ms %.1f MB/s", + ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %-6s %ld bytes netio: %ld ms %.1f MB/s wrt: %ld ms %.1f MB/s", req.method.c_str(), ctxt->st_size, net, mb/std::max(1e-6, double(net)/1e3), wrt, mb/std::max(1e-6, double(wrt)/1e3)); @@ -392,7 +392,7 @@ http::basic_http_server<fs>::handler_t::handle_delete(const request_t& req, repl monitor->data.bytesDEL += ctxt->st_data; return close; } - ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read '%s' size:%ld / %ld %s [Inconsistemt context]", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %-6s Failed to read '%s' size:%ld / %ld %s [Inconsistemt context]", req.method.c_str(), req.uri.c_str(), rep.bytes_sent, rep.bytes_total, rep.stock_status(rep.status).c_str()); rep.context.reset(); @@ -443,7 +443,7 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t // Wait and then check if another thread created the directory if ( -1 == ::stat(dir.c_str(), &stat) ) { std::string err = std::make_error_code(std::errc(errno)).message(); - ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to create parent directory %s [%s]", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %-6s Failed to create parent directory %s [%s]", req.method.c_str(), dir.c_str(), err.c_str()); if ( errno == EACCES || errno == ENOENT || errno == EPERM || errno == EROFS ) rep = reply_t::stock_reply(reply_t::unauthorized); @@ -454,7 +454,7 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t return write; } } - ::lib_rtl_output(LIB_RTL_INFO, "+++ %s: Created parent directory %s", + ::lib_rtl_output(LIB_RTL_INFO, "+++ %-6s Created parent directory %s", req.method.c_str(), dir.c_str()); } } @@ -476,7 +476,7 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t return write; } // This should not happen: The client must first receive the continue! - ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Received non-empty data on request to write " + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %-6s Received non-empty data on request to write " "'%s' [Protocol Violation]", req.method.c_str(), fname.c_str()); return write; } @@ -490,7 +490,7 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t ctxt->st_data += wrt; if ( wrt != len ) { std::string err = std::make_error_code(std::errc(errno)).message(); - ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to write '%s' " + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %-6s Failed to write '%s' " "[%s] only wrote %ld out of %ld bytes", req.method.c_str(), req.uri.c_str(), err.c_str(), wrt, len); ++monitor->data.numPUT_error; @@ -503,9 +503,9 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t long net = std::chrono::duration_cast<std::chrono::milliseconds>(req.netio-null).count(); long wrt = std::chrono::duration_cast<std::chrono::milliseconds>(req.handling-null).count(); double mb = double(ctxt->st_size)/(1024e0*1024e0); - ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %s: Wrote '%s' %.0f %cB", req.method.c_str(), + ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %-6s Wrote '%s' %.0f %cB", req.method.c_str(), req.uri.c_str(), mb > 10e0 ? mb : mb*1024e0, mb > 10e0 ? 'M' : 'k'); - ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %s: %ld bytes netio: %ld ms %.1f MB/s wrt: %ld ms %.1f MB/s", + ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %-6s %ld bytes netio: %ld ms %.1f MB/s wrt: %ld ms %.1f MB/s", req.method.c_str(), ctxt->st_size, net, mb/std::max(1e-6, double(net)/1e3), wrt, mb/std::max(1e-6, double(wrt)/1e3)); @@ -532,11 +532,11 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t } } ++monitor->data.numPUT_error; - ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: FAILED to update '%s' to state WRITTEN!", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %-6s FAILED to update '%s' to state WRITTEN!", req.method.c_str(), req.uri.c_str()); return write; } - ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Finished request '%s' %ld bytes", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %-6s Finished request '%s' %ld bytes", req.method.c_str(), req.uri.c_str(), ctxt ? ctxt->st_size : 0); /// The request is handled. Trigger the shutdown of the connection return none; -- GitLab From 2d5277fbc7435ef6ed2177dd9c0c7c29175d6f24 Mon Sep 17 00:00:00 2001 From: Default Online user <Markus.Frank@cern.ch> Date: Wed, 13 Oct 2021 11:52:23 +0200 Subject: [PATCH 4/5] Fixes from FEST, fix gcc11 compilation warnings --- .../Dataflow/src/Utils/tell1_check_file.cpp | 2 +- Online/EventData/src/EventAccess.cpp | 2 +- .../FarmConfig/job/AddersFromArchitecture.py | 2 +- Online/FarmConfig/job/EBReader.sh | 2 +- Online/FarmConfig/job/HLT2SFAdder.sh | 1 + Online/FarmConfig/job/Passthrough.py | 25 ++-- Online/FarmConfig/job/RU.sh | 4 +- Online/FarmConfig/options/EBReader.opts | 1 + Online/FarmConfig/options/EBSender.opts | 2 +- Online/FarmConfig/options/EBStorage.opts | 3 +- Online/FarmConfig/options/EBStorageNFS.opts | 2 +- Online/FarmConfig/options/HLT2Reader.opts | 5 +- Online/FarmConfig/options/HLT2Writer.opts | 5 +- .../FarmConfig/options/HLT2WriterPosix.opts | 2 +- Online/FarmConfig/options/Monitoring.opts | 1 + .../options/StorageReader.opts | 0 .../options/StorageWriter.opts | 2 + Online/GaudiOnline/components/InputAlg.cpp | 5 +- .../python/GaudiOnline/OnlineApplication.py | 8 +- .../python/GaudiOnline/Passthrough.py | 4 +- Online/GaudiUPI/src/DialogItem.cpp | 6 +- Online/IPMI/src/NewLib/newnew.h | 9 +- Online/PCIE40Data/PCIE40Data/pcie40.h | 20 +-- Online/PCIE40Data/src/pcie40decoder.cpp | 4 +- Online/PyDIM/src/dimcppmodule.cpp | 22 ++- Online/PyDIM/src/dimmodule.cpp | 7 +- Online/PyDIM/src/pydim_utils.cpp | 14 +- Online/TestBeam/TestBeam/gui/NodeFSMPanel.h | 8 +- Online/TestBeam/src/gui/NodeFSMPanel.cpp | 133 ++++++++++++------ TestBeam/options/DataflowArch_Allen.xml | 79 +++++++++++ TestBeam/options/OnlineEnv.opts | 1 + TestBeam/options/OnlineEnvBase.py | 1 + TestBeam/setup_comp.sh | 12 ++ 33 files changed, 283 insertions(+), 111 deletions(-) rename Online/{Dataflow => FarmConfig}/options/StorageReader.opts (100%) rename Online/{Dataflow => FarmConfig}/options/StorageWriter.opts (96%) create mode 100644 TestBeam/options/DataflowArch_Allen.xml diff --git a/Online/Dataflow/src/Utils/tell1_check_file.cpp b/Online/Dataflow/src/Utils/tell1_check_file.cpp index 0cb5090b5..0d1e7e26c 100644 --- a/Online/Dataflow/src/Utils/tell1_check_file.cpp +++ b/Online/Dataflow/src/Utils/tell1_check_file.cpp @@ -212,7 +212,7 @@ extern "C" int tell1_check_file(int argc, char* argv[]) { ::memcpy(data_ptr, size, peek_size); long length = load_mdf_event(input, MemBuffer(alloc_size,data_ptr), deCompress); if ( length <= 0 ) { - ::free(alloc_ptr); + delete [] alloc_ptr; goto Done; } num_bytes_file += alloc_size; diff --git a/Online/EventData/src/EventAccess.cpp b/Online/EventData/src/EventAccess.cpp index 2e030ad6b..bcac44d03 100644 --- a/Online/EventData/src/EventAccess.cpp +++ b/Online/EventData/src/EventAccess.cpp @@ -329,7 +329,7 @@ EventAccess::convertPCIE40MEP(datapointer_t start, size_t len) { } } printer(*m_logger); pair<long, unique_ptr<pcie40::event_collection_t> > events; - pcie40::decoder_t::logger_t logger = printer; + pcie40::decoder_t::logger_t logger = printer; pcie40::decoder_t decoder(logger); datapointer_t end = start + len; diff --git a/Online/FarmConfig/job/AddersFromArchitecture.py b/Online/FarmConfig/job/AddersFromArchitecture.py index 996441987..33455c13f 100644 --- a/Online/FarmConfig/job/AddersFromArchitecture.py +++ b/Online/FarmConfig/job/AddersFromArchitecture.py @@ -174,7 +174,7 @@ def AddersfromTasks(tasklist, adder_type, partition, dohostdns): # =========================================================================================== elif adder_type=="EBPartAdder": - adder_task_class = Class0 + adder_task_class = Class1 for task_name in tasklist: adder = Adder(task_name, "[a-z][a-z]eb[0-9][0-9]", "EBAdder", partition, "counter", "dataflow01", MON_TOP_NODE) adder.obj.ExpandRate = 1 diff --git a/Online/FarmConfig/job/EBReader.sh b/Online/FarmConfig/job/EBReader.sh index 4b7fdc98e..d16c15c62 100644 --- a/Online/FarmConfig/job/EBReader.sh +++ b/Online/FarmConfig/job/EBReader.sh @@ -24,4 +24,4 @@ else fi; export RUN_NUMBER_SERVICE=${PARTITION}/RunInfo/RunNumber; # -execute `dataflow_task ${CLASS}` -opts=${STATIC_OPTS}/${TASK_TYPE}.opts ${AUTO_STARTUP} ${DEBUG_STARTUP}; +execute `dataflow_task Class1` -opts=${STATIC_OPTS}/${TASK_TYPE}.opts ${AUTO_STARTUP} ${DEBUG_STARTUP}; diff --git a/Online/FarmConfig/job/HLT2SFAdder.sh b/Online/FarmConfig/job/HLT2SFAdder.sh index 78b8f8a01..8ec005e5b 100755 --- a/Online/FarmConfig/job/HLT2SFAdder.sh +++ b/Online/FarmConfig/job/HLT2SFAdder.sh @@ -9,4 +9,5 @@ # # ========================================================================= export ADDER_TYPE=HltSubfarm; +echo "WARNING Starting subfarm adder ${UTGID}"; exec -a ${UTGID} genPython.exe `which gaudirun.py` ./AddersFromArchitecture.py --application=Online::OnlineApplication; diff --git a/Online/FarmConfig/job/Passthrough.py b/Online/FarmConfig/job/Passthrough.py index dd444772b..9aec109dd 100644 --- a/Online/FarmConfig/job/Passthrough.py +++ b/Online/FarmConfig/job/Passthrough.py @@ -36,13 +36,18 @@ ev_size = Configurables.Online__EventSize('EventSize') explorer = Configurables.StoreExplorerAlg('Explorer') explorer.Load = 1 -explorer.PrintFreq = 0.01 +explorer.PrintFreq = 0.0000001 explorer.OutputLevel = 1 +have_odin = False +part = OnlineEnv.PartitionName +if part == 'FEST' or part == 'LHCb2': + have_odin = True + application.setup_hive(FlowManager("EventLoop"), 40) -application.setup_monitoring() +application.setup_monitoring(have_odin=have_odin) #application.setup_algorithms(writer, 0.05) -application.setup_algorithms([ev_size,explorer,writer], 1.0) +application.setup_algorithms([ev_size,explorer,writer], 0.5) application.monSvc.DimUpdateInterval = 1 application.config.burstPrintCount = 30000 @@ -53,21 +58,23 @@ application.config.numEventThreads = 15 application.config.MBM_numConnections = 8 application.config.MBM_numEventThreads = 5 # -application.config.execMode = 1 -application.config.numEventThreads = 15 +application.config.numEventThreads = 5 application.config.MBM_numConnections = 3 application.config.MBM_numEventThreads = 4 # # Enable this for debugging # -# application.config.numEventThreads = 1 -# application.config.MBM_numConnections = 1 -# application.config.MBM_numEventThreads = 1 +_dbg = 1 +if _dbg: + application.config.execMode = 0 + application.config.numEventThreads = 1 + application.config.MBM_numConnections = 1 + application.config.MBM_numEventThreads = 1 # application.config.MBM_requests = [ 'EvType=2;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0', 'EvType=1;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0' ] # -print('Setup complete....') +print('Setup complete.... Have ODIN: '+str(have_odin)) diff --git a/Online/FarmConfig/job/RU.sh b/Online/FarmConfig/job/RU.sh index 05bdbfa7d..5bed71070 100755 --- a/Online/FarmConfig/job/RU.sh +++ b/Online/FarmConfig/job/RU.sh @@ -9,8 +9,8 @@ # # ========================================================================= # -. /group/online/dataflow/EventBuilder/EventBuilderRelease/setup.x86_64_v2-centos7-gcc10-opt.vars; -# . /group/online/dataflow/EventBuilder/EventBuilderRelease/setup.x86_64_v2-centos7-gcc10-do0.vars; +# . /group/online/dataflow/EventBuilder/EventBuilderRelease/setup.x86_64_v2-centos7-gcc10-opt.vars; +. /group/online/dataflow/EventBuilder/EventBuilderRelease/setup.x86_64_v2-centos7-gcc10-do0.vars; # . /group/online/dataflow/EventBuilder/EventBuilderRelease/setup.x86_64-centos7-gcc9-opt.vars; #. /group/online/dataflow/EventBuilder/EventBuilderRelease/setup.${CMTCONFIG}.vars; cd ${EVENTBUILDINGROOT}/options; diff --git a/Online/FarmConfig/options/EBReader.opts b/Online/FarmConfig/options/EBReader.opts index 9bd4eca75..82530fa26 100644 --- a/Online/FarmConfig/options/EBReader.opts +++ b/Online/FarmConfig/options/EBReader.opts @@ -21,6 +21,7 @@ Reader.Directories = {"/group/online/dataflow/cmtuser/data/mdf", "/group/online/dataflow/cmtuser/data/tae" }; Reader.Directories = { "/daqarea1/fest/mep"}; +//Reader.Directories = { "/daqarea1/fest/202110/mdf/30000000"}; Reader.FilePrefix = "00"; // Reader.AllowedRuns = {"*"}; diff --git a/Online/FarmConfig/options/EBSender.opts b/Online/FarmConfig/options/EBSender.opts index 44ae980d0..1d6a30b8e 100755 --- a/Online/FarmConfig/options/EBSender.opts +++ b/Online/FarmConfig/options/EBSender.opts @@ -24,7 +24,7 @@ MBM.PartitionName = @OnlineEnv.PartitionName; MBM.PartitionBuffers = @OnlineEnv.PartitionBuffers; // EventSelector.Input = "Output"; -EventSelector.REQ1 = "EvType=2;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=VIP;Frequency=PERC;Perc=50.0"; +EventSelector.REQ1 = "EvType=2;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=50.0"; //EventSelector.REQ1 = "$EVENTSELECTOR_REQ1"; // Sender.DataSink = "$SENDER_TARGET"; diff --git a/Online/FarmConfig/options/EBStorage.opts b/Online/FarmConfig/options/EBStorage.opts index a9f082ce9..22fcea2a5 100644 --- a/Online/FarmConfig/options/EBStorage.opts +++ b/Online/FarmConfig/options/EBStorage.opts @@ -1,5 +1,6 @@ -#include "$DATAFLOWROOT/options/StorageWriter.opts" +#include "$FARMCONFIGROOT/options/StorageWriter.opts" Monitoring.CounterUpdateInterval = 3; +// MBM.ConnectWhen = "initialize"; //EventProc.DelayCancel = 15; // diff --git a/Online/FarmConfig/options/EBStorageNFS.opts b/Online/FarmConfig/options/EBStorageNFS.opts index 610b47dd1..9263fca7e 100644 --- a/Online/FarmConfig/options/EBStorageNFS.opts +++ b/Online/FarmConfig/options/EBStorageNFS.opts @@ -1,4 +1,4 @@ -#include "$DATAFLOWROOT/options/StorageWriter.opts" +#include "$FARMCONFIGROOT/options/StorageWriter.opts" Monitoring.CounterUpdateInterval = 3; MBM.ConnectWhen = "initialize"; // diff --git a/Online/FarmConfig/options/HLT2Reader.opts b/Online/FarmConfig/options/HLT2Reader.opts index 732b457d5..12e1bdea4 100644 --- a/Online/FarmConfig/options/HLT2Reader.opts +++ b/Online/FarmConfig/options/HLT2Reader.opts @@ -3,6 +3,7 @@ #include "$MBM_SETUP_OPTIONS" #include "$FARMCONFIGROOT/options/Logging.opts" #include "$FARMCONFIGROOT/options/Monitoring.opts" +Monitoring.CounterUpdateInterval = 3; // Manager.Services = {"Dataflow_MBMClient/MEPManager", // "Dataflow_BurstReader/Reader", @@ -68,6 +69,8 @@ Reader.Server = "devbbdb01.lbdaq.cern.ch:4242"; Reader.FDBVersion = 1; Reader.PartitionName = "FEST"; Reader.FilePrefix = "${PARTITION}/${RUN}/"; -Reader.AllowedRuns = { "219799", "219800", "219810", "219811", "219812", "219813", "219816", "219817", "219818", "219819", "219820" }; +Reader.AllowedRuns = { "219723", "219724", "219725", "219726", "219727", "219728", "219729" +// "219799", "219800", "219810", "219811", "219812", "219813", "219816", "219817", "219818", "219819", "219820" +}; Reader.NumThreads = 2; Reader.NumBuffers = 3; diff --git a/Online/FarmConfig/options/HLT2Writer.opts b/Online/FarmConfig/options/HLT2Writer.opts index 3358bcc87..e3157c81d 100644 --- a/Online/FarmConfig/options/HLT2Writer.opts +++ b/Online/FarmConfig/options/HLT2Writer.opts @@ -1,6 +1,7 @@ #pragma print off #include "$MBM_SETUP_OPTIONS" -#include "$DATAFLOWROOT/options/StorageWriter.opts" +#include "$FARMCONFIGROOT/options/StorageWriter.opts" +Monitoring.CounterUpdateInterval = 3; MBM.Buffers = @OnlineEnv.HLT2Writer_Buffers; EventProc.Input = @OnlineEnv.HLT2Writer_Input; @@ -18,7 +19,7 @@ EventProc.REQ2 = "EvType=2;TriggerMask=0xffffffff,0xffffffff,0xfffffff // NFS writing Writer.PartitionName = "FEST"; Writer.ThreadFileQueues = false; -Writer.IdleTimeout = 30; +Writer.IdleTimeout = 120; Writer.NumThreads = 2; Writer.NumBuffers = 3; Writer.MaxFileSizeMB = 4000; diff --git a/Online/FarmConfig/options/HLT2WriterPosix.opts b/Online/FarmConfig/options/HLT2WriterPosix.opts index 24f0b0f52..25c7b88f4 100644 --- a/Online/FarmConfig/options/HLT2WriterPosix.opts +++ b/Online/FarmConfig/options/HLT2WriterPosix.opts @@ -1,4 +1,4 @@ -#include "$DATAFLOWROOT/options/StorageWriter.opts" +#include "$FARMCONFIGROOT/options/StorageWriter.opts" Writer.MaxFileSizeMB = 4000; Writer.NumBuffers = 6; Writer.NumThreads = 4; diff --git a/Online/FarmConfig/options/Monitoring.opts b/Online/FarmConfig/options/Monitoring.opts index 86cf2961d..b62bc0857 100755 --- a/Online/FarmConfig/options/Monitoring.opts +++ b/Online/FarmConfig/options/Monitoring.opts @@ -3,3 +3,4 @@ Monitoring.PartitionName = @OnlineEnv.PartitionName; Monitoring.UniqueServiceNames = 1; Monitoring.ExpandCounterServices = 1; Monitoring.ExpandNameInfix = "<proc>/"; +Monitoring.CounterUpdateInterval = 5; diff --git a/Online/Dataflow/options/StorageReader.opts b/Online/FarmConfig/options/StorageReader.opts similarity index 100% rename from Online/Dataflow/options/StorageReader.opts rename to Online/FarmConfig/options/StorageReader.opts diff --git a/Online/Dataflow/options/StorageWriter.opts b/Online/FarmConfig/options/StorageWriter.opts similarity index 96% rename from Online/Dataflow/options/StorageWriter.opts rename to Online/FarmConfig/options/StorageWriter.opts index b29ef1927..6ab1a357c 100644 --- a/Online/Dataflow/options/StorageWriter.opts +++ b/Online/FarmConfig/options/StorageWriter.opts @@ -4,6 +4,8 @@ #include "$FARMCONFIGROOT/options/Monitoring.opts" Logger.OutputLevel = @OnlineEnv.OutputLevel; +Monitoring.CounterUpdateInterval = 5; +Monitoring.DimUpdateInterval = 20; // Manager.Services = {"Dataflow_MBMClient/MBM", "Dataflow_MBMSelector/EventProc", diff --git a/Online/GaudiOnline/components/InputAlg.cpp b/Online/GaudiOnline/components/InputAlg.cpp index a71599c79..a073f5519 100644 --- a/Online/GaudiOnline/components/InputAlg.cpp +++ b/Online/GaudiOnline/components/InputAlg.cpp @@ -115,7 +115,6 @@ StatusCode Online::InputAlg::process(EventContext const& /* ctxt */) const { evt_desc_t e(m_io->pop()); if ( e.second ) { try { - StatusCode sc; bool declare_banks = m_declareEvt.value(); bool make_raw_evt = m_makeRawEvt.value(); bool expand_tae = m_expandTAE.value(); @@ -143,7 +142,7 @@ StatusCode Online::InputAlg::process(EventContext const& /* ctxt */) const { /// Invoke here the conversion to raw events if ( make_raw_evt ) { const auto* pev = &event.second; - sc = m_evtTool->put("/Event/DAQ/RawEvent", *(const lb_banks_t*)pev); + StatusCode sc = m_evtTool->put("/Event/DAQ/RawEvent", *(const lb_banks_t*)pev); if ( !sc.isSuccess() ) { } } @@ -158,7 +157,7 @@ StatusCode Online::InputAlg::process(EventContext const& /* ctxt */) const { if ( make_raw_evt ) { std::size_t eid = e.first; const auto* hdr = e.second->at(eid).tell1_event; - sc = this->expand_tae_tell1(hdr); + StatusCode sc = this->expand_tae_tell1(hdr); if ( !sc.isSuccess() ) { } } diff --git a/Online/GaudiOnline/python/GaudiOnline/OnlineApplication.py b/Online/GaudiOnline/python/GaudiOnline/OnlineApplication.py index ca2881596..185724c86 100644 --- a/Online/GaudiOnline/python/GaudiOnline/OnlineApplication.py +++ b/Online/GaudiOnline/python/GaudiOnline/OnlineApplication.py @@ -143,7 +143,7 @@ class Application(object): self.input = input return self.input - def setup_monitoring(self, task_name=None): + def setup_monitoring(self, task_name=None, have_odin=True): config = self.config mon = Configurables.MonitorSvc('MonitorSvc') mon.PartitionName = self.partitionName @@ -152,18 +152,22 @@ class Application(object): mon.UniqueServiceNames = True mon.UseDStoreNames = True mon.DimUpdateInterval = 5 + mon.CounterUpdateInterval = 5 if task_name: mon.ProgramName = task_name self.app.ExtSvc.insert(0, mon) self.monSvc = mon + self.updateAndReset = None update = Configurables.Online__UpdateAndReset('UpdateAndReset') update.resetOnStart = False; update.saverCycle = 900 update.saveHistograms = 0 + update.disableReadOdin = not have_odin if task_name: update.MyName = task_name - self.updateAndReset = update; + self.updateAndReset = update + return self def setup_algorithms(self,*args,**kwd): return self diff --git a/Online/GaudiOnline/python/GaudiOnline/Passthrough.py b/Online/GaudiOnline/python/GaudiOnline/Passthrough.py index 0313b994b..90e3a5ad1 100644 --- a/Online/GaudiOnline/python/GaudiOnline/Passthrough.py +++ b/Online/GaudiOnline/python/GaudiOnline/Passthrough.py @@ -25,9 +25,11 @@ class Passthrough(Application): passThrough.RawGuard = '/Event/Banks/RawDataGuard' passThrough.AcceptRate = acceptRate self.passThrough = passThrough + sequence = Gaudi.GaudiSequencer('Output') sequence.Members = [self.input] - sequence.Members.append(self.updateAndReset) + if self.updateAndReset: + sequence.Members.append(self.updateAndReset) sequence.Members.append(self.passThrough) if isinstance(writer, (list, tuple)): for item in writer: diff --git a/Online/GaudiUPI/src/DialogItem.cpp b/Online/GaudiUPI/src/DialogItem.cpp index 8f3641b79..f4eca1ce9 100755 --- a/Online/GaudiUPI/src/DialogItem.cpp +++ b/Online/GaudiUPI/src/DialogItem.cpp @@ -26,7 +26,7 @@ namespace { union _CNV { void* ptr; long* _l; - float* _f; + double* _f; _CNV(void* p) { ptr=p; } long l() { return *_l; } float f() { return *_f; } @@ -34,8 +34,8 @@ namespace { }; } -static inline ClientData _cnv(int i) { return _CNV(&i).cdata(); } -static inline ClientData _cnv(float f) { return _CNV(&f).cdata(); } +static inline ClientData _cnv(int i) { long l = i; return _CNV(&l).cdata(); } +static inline ClientData _cnv(float f) { double d = f; return _CNV(&d).cdata(); } static inline ClientData _cnv(const string& s) { return _CNV((void*)&s).ptr; } DialogItem::DialogItem (const string& fmt,const string& text,const string& def,const string& lo,const string& hi,bool list_only) { diff --git a/Online/IPMI/src/NewLib/newnew.h b/Online/IPMI/src/NewLib/newnew.h index d3676f4f1..62758ac8d 100644 --- a/Online/IPMI/src/NewLib/newnew.h +++ b/Online/IPMI/src/NewLib/newnew.h @@ -15,8 +15,8 @@ * Created on: Feb 10, 2016 * Author: beat */ -//#include <string> #include "log.h" + void *operator new(size_t s) { void *t = malloc(s); @@ -27,3 +27,10 @@ void *operator new(size_t s) memset(t,0,s); return t; } + +void operator delete(void* pointer) +{ + if ( pointer ) { + ::free(pointer); + } +} diff --git a/Online/PCIE40Data/PCIE40Data/pcie40.h b/Online/PCIE40Data/PCIE40Data/pcie40.h index bcaa056de..7afef879a 100644 --- a/Online/PCIE40Data/PCIE40Data/pcie40.h +++ b/Online/PCIE40Data/PCIE40Data/pcie40.h @@ -426,13 +426,13 @@ namespace Online { static constexpr detsource_t sourceid_top5_Rich = (uint16_t)((sourceid_Rich<<11)&sourceid_TOP5); static constexpr std::size_t maxTell40ODIN = 1; // True counts - static constexpr std::size_t maxTell40VP = 215; // 208 - static constexpr std::size_t maxTell40UT = 225; // 216 - static constexpr std::size_t maxTell40FT = 250; // 240 - static constexpr std::size_t maxTell40Ecal = 28; // 28 - static constexpr std::size_t maxTell40Hcal = 8; // 8 - static constexpr std::size_t maxTell40Muon = 10; // 10 - static constexpr std::size_t maxTell40Rich = 350; + static constexpr std::size_t maxTell40VP = 216; // 208 + static constexpr std::size_t maxTell40UT = 226; // 202 + static constexpr std::size_t maxTell40FT = 300; // 288 + static constexpr std::size_t maxTell40Ecal = 100; // 92 + static constexpr std::size_t maxTell40Hcal = 100; // 92 + static constexpr std::size_t maxTell40Muon = 50; // 44 + static constexpr std::size_t maxTell40Rich = 350; // 172 static constexpr std::size_t maxTell40Other = 100; static constexpr std::size_t maxTell40 = maxTell40ODIN + maxTell40VP + maxTell40UT + maxTell40FT + maxTell40Ecal + maxTell40Hcal + maxTell40Muon + maxTell40Rich + maxTell40Other; @@ -684,7 +684,7 @@ namespace Online { /// Access collection Offset according to source ID inline std::size_t event_t::collection_offset_source_id(uint16_t src_id) { - switch(src_id) { + switch( src_id¶ms::sourceid_TOP5 ) { case params::sourceid_top5_ODIN: return params::collectionOffsetODIN; case params::sourceid_top5_VP: return params::collectionOffsetVP; case params::sourceid_top5_UT: return params::collectionOffsetUT; @@ -706,6 +706,10 @@ namespace Online { case bank_t::UT: case bank_t::FTCluster: case bank_t::Rich: + case bank_t::HC: + case bank_t::HCError: + case bank_t::Calo: + case bank_t::CaloError: case bank_t::EcalPacked: case bank_t::HcalPacked: case bank_t::Muon: diff --git a/Online/PCIE40Data/src/pcie40decoder.cpp b/Online/PCIE40Data/src/pcie40decoder.cpp index 57eef47d9..23bc90e3a 100644 --- a/Online/PCIE40Data/src/pcie40decoder.cpp +++ b/Online/PCIE40Data/src/pcie40decoder.cpp @@ -151,7 +151,7 @@ public: } return &b; #else - return &c->banks[c->length-1]; + return &c->banks[c->capacity - c->specials]; #endif } @@ -235,7 +235,7 @@ public: e->flags.detail.tae_central = sodin->tae_central(); e->flags.detail.tae_window = sodin->tae_window(); } - //b->setMagic(); + b->setMagic(); b->setSize(length); b->setSourceID(src_id); b->setData(data); diff --git a/Online/PyDIM/src/dimcppmodule.cpp b/Online/PyDIM/src/dimcppmodule.cpp index f15b70349..edc4edf87 100755 --- a/Online/PyDIM/src/dimcppmodule.cpp +++ b/Online/PyDIM/src/dimcppmodule.cpp @@ -381,22 +381,22 @@ DimRpc_setData (DimRpc_Object* self, PyObject* args) { * The conversion is done based on the arguments supplied when the RPC * command was created. */ - char *buff=NULL; + char *buff = nullptr; unsigned int buff_size=0; if (!self->cpp_dimRpc) { // should never reach this point PyErr_SetString(PyExc_AttributeError, "C++ Dim RPC object is NULL"); - return NULL; + return nullptr; } if (iterator_to_allocated_buffer(args, self->format_out, (char **)&buff, &buff_size) ) { self->cpp_dimRpc->setData(buff, buff_size); - delete buff; + delete [] buff; } else { PyErr_SetString(PyExc_AttributeError, "Could not convert arguments to C buffer"); - return NULL; + return nullptr; } Py_RETURN_NONE; @@ -746,18 +746,18 @@ DimRpcInfo_setData (DimRpcInfo_Object* self, PyObject* args) { * The conversion is done based on the arguments supplied when the RPC * command was created. */ - char *buff=NULL; + char *buff = nullptr; unsigned int buff_size=0; if (!self->cpp_dimRpcInfo) { /* should never reach this point */ PyErr_SetString(PyExc_AttributeError, "C++ Dim RPC object is NULL"); - return NULL; + return nullptr; } //printPyObject(args); - if (iterator_to_allocated_buffer(args, self->format_in, - (char **)&buff, &buff_size) ) { + if ( iterator_to_allocated_buffer(args, self->format_in, + (char**)&buff, &buff_size) ) { /* The setData() method of DimInfo is blocking. This creates a * deadlock between the calling thread that holds the python * global interpretor lock and the DIM global lock @@ -765,13 +765,12 @@ DimRpcInfo_setData (DimRpcInfo_Object* self, PyObject* args) { Py_BEGIN_ALLOW_THREADS self->cpp_dimRpcInfo->setData(buff, buff_size); Py_END_ALLOW_THREADS - delete buff; + delete [] buff; } else { PyErr_SetString(PyExc_AttributeError, "Could not convert arguments to C buffer"); - return NULL; + return nullptr; } - Py_RETURN_NONE; } @@ -896,7 +895,6 @@ initdimcpp() #else module = Py_InitModule3("dimcpp", dimcpp_methods, "DIM C++ methods"); #endif - PyEval_InitThreads(); if (module == NULL) { ::printf("Could not initialise dimcpp module\n"); PyMODINIT_RETURN(module); diff --git a/Online/PyDIM/src/dimmodule.cpp b/Online/PyDIM/src/dimmodule.cpp index fbbf17479..88c993bf8 100755 --- a/Online/PyDIM/src/dimmodule.cpp +++ b/Online/PyDIM/src/dimmodule.cpp @@ -988,13 +988,10 @@ dim_dic_get_id(PyObject* /* self */, PyObject* /* args */) { * @return client_name The client name or an empty string if the * command was not successful. */ - char name[256]; - int res; - - res = dic_get_id(name); + char name[256] { 0 }; + int res = dic_get_id(name); if (!res) name[0] = 0; - return Py_BuildValue("s", name); } diff --git a/Online/PyDIM/src/pydim_utils.cpp b/Online/PyDIM/src/pydim_utils.cpp index 6663194f2..6e8c74904 100755 --- a/Online/PyDIM/src/pydim_utils.cpp +++ b/Online/PyDIM/src/pydim_utils.cpp @@ -126,7 +126,7 @@ static int listOrTuple2Int(PyObject* pyObj, int** buffer) size = PyTuple_Size(pyObj); if (!size) res = 0; - (*buffer) = (int*)malloc(size*sizeof(int)+1); + (*buffer) = new int[size+1]; if ( !(*buffer) ) res = 0; for (i=0; i<size; i++) { @@ -139,7 +139,7 @@ static int listOrTuple2Int(PyObject* pyObj, int** buffer) size = PyList_Size(pyObj); if (!size) res = 0; - (*buffer) = (int*)malloc(size*sizeof(int)+1); + (*buffer) = new int[size+1]; if ( !(*buffer) ) res = 0; for (i=0; i<size; i++) { @@ -156,7 +156,7 @@ static int listOrTuple2Int(PyObject* pyObj, int** buffer) return 1; } else { - free(*buffer); + delete [] (*buffer); buffer = NULL; return 0; } @@ -778,18 +778,18 @@ iterator_to_allocated_buffer(PyObject *iter, /* list or tuple */ * Input: PyTuple or PyList * Output: pointer to the newly created buffer and its size */ - *buffer = NULL; + *buffer = nullptr; *size = 0; *size = getSizeFromFormatAndObjects(iter, format); if (!*size) { // could not figure out the size of the needed buffer - *buffer = NULL; + *buffer = nullptr; return 0; } - *buffer= (char*)malloc(*size); + *buffer= new char[*size]; if (!iterator_to_buffer(iter, *buffer, *size, format)) { - *buffer = NULL; + *buffer = nullptr; return 0; } // the call succeded diff --git a/Online/TestBeam/TestBeam/gui/NodeFSMPanel.h b/Online/TestBeam/TestBeam/gui/NodeFSMPanel.h index 299ac4c55..fc4e08189 100644 --- a/Online/TestBeam/TestBeam/gui/NodeFSMPanel.h +++ b/Online/TestBeam/TestBeam/gui/NodeFSMPanel.h @@ -74,7 +74,7 @@ namespace testbeam { std::mutex dim_protection; CPP::Interactor* gui = nullptr; TGTextButton* apply = nullptr; - LineEntry maindns, dns, node, part, replace, script, arch, numslave; + LineEntry maindns, dns, node, part, replace, runnumber, script, arch, numslave; TaskEntry tmSrv, logSrv, logViewer, mbmmon, mbmdmp, tanSrv, storage, ctrl, did; std::vector<Child> children; TGComboBox* ctrl_command = nullptr; @@ -94,6 +94,7 @@ namespace testbeam { long m_numSlaves = 0; int m_pid = -1; + int m_currentRun = 1234; int m_autoRun = 0; bool m_applied = false; std::string m_runinfo; @@ -121,6 +122,7 @@ namespace testbeam { int m_mainDNS_id = -1; int m_tmSrvList_id = 0; + int m_runnumber_id = 0; public: enum IDS { @@ -141,6 +143,8 @@ namespace testbeam { SCRIPT_INPUT = PART_ID_OFFSET+14, REPLACE_LABEL = PART_ID_OFFSET+15, REPLACE_INPUT = PART_ID_OFFSET+16, + RUNNUMBER_LABEL = PART_ID_OFFSET+15, + RUNNUMBER_INPUT = PART_ID_OFFSET+16, TMSRV_ID_OFFSET = 200, TMSRV_LABEL = TMSRV_ID_OFFSET+1, @@ -216,6 +220,8 @@ namespace testbeam { void updateChildren(const std::string& value); void updateTaskStatus(TaskEntry& entry, const std::string& state); + void runnumberChanged(const char* value); + void setArchitecture(const std::string& value); void architectureChanged(const char* value); diff --git a/Online/TestBeam/src/gui/NodeFSMPanel.cpp b/Online/TestBeam/src/gui/NodeFSMPanel.cpp index 4cc12a4ae..dbf298561 100644 --- a/Online/TestBeam/src/gui/NodeFSMPanel.cpp +++ b/Online/TestBeam/src/gui/NodeFSMPanel.cpp @@ -26,6 +26,7 @@ #include "RTL/strdef.h" #include "RTL/rtl.h" #include "dim/dic.h" +#include "dim/dis.h" /// ROOT include files #include "TSystem.h" @@ -53,10 +54,40 @@ static constexpr const char* TASK_DEAD = "DEAD"; static constexpr const char* TASK_RUNNING = "RUNNING"; namespace { + string guiserv_name() { + return "TestBeamGUI_"+RTL::processName(); + } + string _tmKill() { + struct stat stat; + return ( 0 == ::stat(TMKILL_COMMAND, &stat) ) + ? TMKILL_COMMAND + : "LD_LIBRARY_PATH=NONE /usr/local/bin/tmKill"; + } + string _tmStart() { + struct stat stat; + return ( 0 == ::stat(TMSTART_COMMAND, &stat) ) + ? TMSTART_COMMAND + : "LD_LIBRARY_PATH=NONE /usr/local/bin/tmStart"; + } + string _tmSrv() { + struct stat stat; + return 0 == ::stat(TMSRV_COMMAND, &stat) + ? TMSRV_COMMAND : "/usr/local/sbin/tmSrv"; + } string _selectedText(TGComboBox* c) { TGLBEntry* selected = c->GetSelectedEntry(); return selected ? selected->GetTitle() : ""; } + /// DIM service update handler + void run_no_update(void* tag, void** buf, int* size, int* first) { + NodeFSMPanel* h = *(NodeFSMPanel**)tag; + if ( *first ) { + } + if ( h ) { + *size = sizeof(h->m_currentRun); + *buf = &h->m_currentRun; + } + } /// DIM command service callback void dim_feed(void* tag, void* address, int* size) { NodeFSMPanel::dim_info_t* h = *(NodeFSMPanel::dim_info_t**)tag; @@ -208,14 +239,20 @@ void NodeFSMPanel::init() { replace.label = new TGLabel( para_group, "Replacements", REPLACE_LABEL); replace.input = new TGTextEntry(para_group, m_replacements.c_str(), REPLACE_INPUT); - arch.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "architectureChanged(const char*)"); - script.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "scriptChanged(const char*)"); - maindns.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "mainDnsChanged(const char*)"); - dns.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "dnsChanged(const char*)"); - node.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "hostChanged(const char*)"); - numslave.input->Connect("TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "numslaveChanged(const char*)"); - part.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "partitionChanged(const char*)"); - replace.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "replacementChanged(const char*)"); + str.str(""); + str << m_currentRun; + runnumber.label = new TGLabel( para_group, "Run number", RUNNUMBER_LABEL); + runnumber.input = new TGTextEntry(para_group, str.str().c_str(), RUNNUMBER_INPUT); + + arch.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "architectureChanged(const char*)"); + script.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "scriptChanged(const char*)"); + maindns.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "mainDnsChanged(const char*)"); + dns.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "dnsChanged(const char*)"); + node.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "hostChanged(const char*)"); + numslave.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "numslaveChanged(const char*)"); + part.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "partitionChanged(const char*)"); + replace.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "replacementChanged(const char*)"); + runnumber.input->Connect("TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "runnumberChanged(const char*)"); apply = new TGTextButton(para_group, "Apply parameters", APPLY_PARAMS); apply->Connect("Clicked()", "testbeam::NodeFSMPanel", this, "applyParams()"); @@ -321,6 +358,10 @@ void NodeFSMPanel::init() { para_group->AddFrame(numslave.input, new TGTableLayoutHints(1, 2, 4, 5, kLHintsLeft|kLHintsCenterY|kLHintsExpandY,_PAD,_PAD,_PADY,_PAD)); para_group->AddFrame(replace.label, new TGTableLayoutHints(0, 1, 5, 6, kLHintsLeft|kLHintsCenterY,_PAD,_PADY,_PADY,_PAD)); para_group->AddFrame(replace.input, new TGTableLayoutHints(1, 2, 5, 6, kLHintsLeft|kLHintsCenterY|kLHintsExpandY,_PAD,_PAD,_PADY,_PAD)); + + para_group->AddFrame(runnumber.label, new TGTableLayoutHints(0, 1, 6, 7, kLHintsLeft|kLHintsCenterY,_PAD,_PAD,_PADY,_PAD)); + para_group->AddFrame(runnumber.input, new TGTableLayoutHints(1, 5, 6, 7, kLHintsLeft|kLHintsCenterY|kLHintsExpandY|kLHintsExpandX, _PAD,_PAD,_PADY,_PAD)); + para_group->AddFrame(script.label, new TGTableLayoutHints(0, 1, 8, 9, kLHintsLeft|kLHintsCenterY|kLHintsExpandX|kLHintsExpandY,_PAD,_PAD,_PADY,_PAD)); para_group->AddFrame(script.input, new TGTableLayoutHints(1, 5, 8, 9, kLHintsLeft|kLHintsCenterY|kLHintsExpandX|kLHintsExpandY,_PAD,_PAD,_PADY,_PAD)); para_group->AddFrame(arch.label, new TGTableLayoutHints(0, 1, 9,10, kLHintsLeft|kLHintsCenterY,_PAD,_PAD,_PADY,_PAD)); @@ -448,10 +489,11 @@ void NodeFSMPanel::init() { dns.input->Resize(250, dns.input->GetDefaultHeight()); node.input->Resize(250, node.input->GetDefaultHeight()); part.input->Resize(250, part.input->GetDefaultHeight()); + numslave.input->Resize(150, numslave.input->GetDefaultHeight()); replace.input->Resize(250, replace.input->GetDefaultHeight()); + runnumber.input->Resize(150, replace.input->GetDefaultHeight()); script.input->Resize(600, script.input->GetDefaultHeight()); arch.input->Resize(600, arch.input->GetDefaultHeight()); - numslave.input->Resize(150, numslave.input->GetDefaultHeight()); double height = tmSrv.kill->GetDefaultHeight(); tmSrv.kill ->Resize(120, height); @@ -553,10 +595,12 @@ void NodeFSMPanel::init() { gClient->NeedRedraw(c.state); gClient->NeedRedraw(c.meta); } + ::dis_start_serving(guiserv_name().c_str()); } /// Default destructor NodeFSMPanel::~NodeFSMPanel() { + if ( 0 != m_runnumber_id ) ::dis_remove_service(m_runnumber_id); if ( 0 != m_storageCom->svcID ) ::dic_release_service(m_storageCom->svcID); if ( 0 != m_tanSrvCom->svcID ) ::dic_release_service(m_tanSrvCom->svcID); if ( 0 != m_tmSrvList_id ) ::dic_release_service(m_tmSrvList_id); @@ -614,6 +658,8 @@ void NodeFSMPanel::enableParams() { storage.utgid->SetEnabled(kTRUE); storage.start->SetEnabled(kFALSE); apply->SetText("Apply parameters"); + if ( 0 != m_runnumber_id ) ::dis_remove_service(m_runnumber_id); + m_runnumber_id = 0; if ( 0 != m_storageCom->svcID ) ::dic_release_service(m_storageCom->svcID); m_storageCom->svcID = 0; if ( 0 != m_tanSrvCom->svcID ) ::dic_release_service(m_tanSrvCom->svcID); @@ -692,6 +738,7 @@ void NodeFSMPanel::applyParams() { storage.utgid->SetEnabled(kFALSE); storage.start->SetEnabled(kTRUE); set_utgid_names(); + if ( 0 != m_runnumber_id ) ::dis_remove_service(m_runnumber_id); if ( 0 != m_tanSrvCom->svcID ) ::dic_release_service(m_tanSrvCom->svcID); if ( 0 != m_tmSrvList_id ) ::dic_release_service(m_tmSrvList_id); if ( 0 != m_tmSrvCom->svcID ) ::dic_release_service(m_tmSrvCom->svcID); @@ -708,6 +755,9 @@ void NodeFSMPanel::applyParams() { m_ctrlCom->svcID = ::dic_info_service(svc.c_str(), MONITORED,0,0,0,dim_feed,(long)m_ctrlCom, 0, 0); m_ctrlTasks->svcID = ::dic_info_service((m_ctrlName+"/tasks").c_str(), MONITORED,1,0,0,dim_feed,(long)m_ctrlTasks, 0, 0); m_mainDNS_id = m_mainDNS.empty() ? -1 : ::dic_add_dns(m_mainDNS.c_str(), ::dim_get_dns_port()); + m_runnumber_id = ::dis_add_service((m_partition+"/RunInfo/RunNumber").c_str(),"I",0,0,run_no_update,(long)this); + ::dis_update_service(m_runnumber_id); + ::dis_start_serving(guiserv_name().c_str()); } void NodeFSMPanel::updateDependentValues() { @@ -973,30 +1023,16 @@ void NodeFSMPanel::autoStop() { void NodeFSMPanel::tmSrv_start() { if ( m_tmSrvCom->data == TASK_DEAD ) { stringstream cmd; - struct stat stat; - string tmSrv = TMSRV_COMMAND; - string tmStart = TMSTART_COMMAND; - if ( 0 != ::stat("/opt/FMC/bin/tmStart", &stat) ) { - tmStart = "LD_LIBRARY_PATH=NONE /usr/local/bin/tmStart"; - tmSrv = "/usr/local/sbin/tmSrv"; - } if ( m_mainDNS.empty() ) { cmd << "export DIM_DNS_NODE=" << m_node << ";export UTGID=" << m_tmSrvName << ";" - << tmSrv << " -l 2 -N " << m_node << " -p 2 -u online -U root --no-auth --no-ptrace-workaround&"; + << _tmSrv() << " -l 2 -N " << m_node << " -p 2 -u online -U root --no-auth --no-ptrace-workaround&"; system(cmd.str().c_str()); } else { - cmd << tmStart << " -N " << m_mainDNS << " -m " << m_node << " " << startDefaults() + cmd << _tmStart() << " -N " << m_mainDNS << " -m " << m_node << " " << startDefaults() << " -n root -g root -DDIM_DNS_NODE=" << m_node << " -u " << m_tmSrvName << " " - << tmSrv << " -l 2 -N " << m_node << " -p 2 -u online -U root --no-auth --no-ptrace-workaround"; + << _tmSrv() << " -l 2 -N " << m_node << " -p 2 -u online -U root --no-auth --no-ptrace-workaround"; system(cmd.str().c_str()); -#if 0 - FiniteStateMachine::TaskManager(m_node, m_mainDNS_id) - .start(m_tmSrvName, - " -N " + m_mainDNS + " -m " + m_node + " " + startDefaults(), - "/opt/FMC/sbin/tmSrv", - "-l 2 -N " + m_node + " -p 2 -u online -U root --no-auth --no-ptrace-workaround"); -#endif } return; } @@ -1011,32 +1047,27 @@ void NodeFSMPanel::tmSrv_kill() { if ( m_mainDNS.empty() ) { cmd = "pkill -9 tmSrv"; system(cmd.c_str()); + return; } - else { - cmd = string(TMKILL_COMMAND) + " -N "+m_mainDNS+" -m "+m_node+" " + killDefaults() + m_tmSrvName; - system(cmd.c_str()); - //FiniteStateMachine::TaskManager(m_node, m_mainDNS_id).kill(m_tmSrvName, 9); - } + cmd = _tmKill() + " -N "+m_mainDNS+" -m "+m_node+" " + killDefaults() + m_tmSrvName; + system(cmd.c_str()); + //FiniteStateMachine::TaskManager(m_node, m_mainDNS_id).kill(m_tmSrvName, 9); } /// Start logSrv process void NodeFSMPanel::logSrv_start() { if ( m_logSrvCom->data == TASK_DEAD ) { stringstream cmd; - struct stat stat; - string tmStart = TMSTART_COMMAND; - if ( 0 != ::stat("/opt/FMC/bin/tmStart", &stat) ) { - tmStart = "LD_LIBRARY_PATH=NONE /usr/local/bin/tmStart"; - } if ( m_mainDNS.empty() ) { - cmd << "export DIM_DNS_NODE=" << m_node << ";export UTGID=" << m_logSrvName << ";"; - cmd << "export LOGFIFO=" << m_logFifo << "; " - << "export DIM_HOST_NODE=" << m_node << "; "; + cmd << "export DIM_DNS_NODE=" << m_node << "; " + << "export UTGID=" << m_logSrvName << ";"; + cmd << "export LOGFIFO=" << m_logFifo << "; " + << "export DIM_HOST_NODE=" << m_node << "; "; cmd << "/group/online/dataflow/scripts/NodeLogger.sh&"; system(cmd.str().c_str()); } else { - cmd << tmStart << " -N " << m_mainDNS << " -m " << m_node + cmd << _tmStart() << " -N " << m_mainDNS << " -m " << m_node << " -e -o -n online -DDIM_DNS_NODE=" << m_node << " -u " << m_logSrvName << " " << "-DLOGFIFO=" << m_logFifo << " " << "-DDIM_HOST_NODE=" << m_node << " " @@ -1054,13 +1085,10 @@ void NodeFSMPanel::logSrv_start() { /// Kill logSrv process void NodeFSMPanel::logSrv_kill() { string cmd; - struct stat stat; if ( m_mainDNS.empty() ) cmd = "pkill -9 logSrv"; - else if ( 0 == ::stat("/opt/FMC/bin/tmKill", &stat) ) - cmd = string(TMKILL_COMMAND)+" -N "+m_mainDNS+" -m "+m_node+" " + killDefaults() + m_logSrvName; else - cmd = string("LD_LIBRARY_PATH=NONE /usr/local/bin/tmKill")+" -N "+m_mainDNS+" -m "+m_node+" " + killDefaults() + m_logSrvName; + cmd = _tmKill() + " -N "+m_mainDNS+" -m "+m_node+" " + killDefaults() + m_logSrvName; system(cmd.c_str()); } @@ -1233,6 +1261,19 @@ void NodeFSMPanel::logViewer_kill() { killProcess(cmd.str()); } +void NodeFSMPanel::runnumberChanged(const char* value) { + string runno = value; + int run = 0; + if ( 1 == ::sscanf(runno.c_str(),"%d",&run) ) { + m_currentRun = run; + ::dis_update_service(m_runnumber_id); + return; + } + GuiException(kMBOk,kMBIconAsterisk,"Invalid Value", + "The given run number is invalid.\n" + "Current run is: %d\n", m_currentRun).send(gui); +} + void NodeFSMPanel::setArchitecture(const std::string& value) { m_architecture = value; if ( arch.input ) { @@ -1271,6 +1312,10 @@ void NodeFSMPanel::setPartition(const std::string& value) { void NodeFSMPanel::partitionChanged(const char* value) { m_partition = value; updateDependentValues(); + if ( 0 != m_runnumber_id ) ::dis_remove_service(m_runnumber_id); + m_runnumber_id = ::dis_add_service((m_partition+"/RunInfo/RunNumber").c_str(),"I",0,0,run_no_update,(long)this); + ::dis_start_serving(guiserv_name().c_str()); + ::dis_update_service(m_runnumber_id); } void NodeFSMPanel::setReplacement(const std::string& value) { diff --git a/TestBeam/options/DataflowArch_Allen.xml b/TestBeam/options/DataflowArch_Allen.xml new file mode 100644 index 000000000..c0cbe4940 --- /dev/null +++ b/TestBeam/options/DataflowArch_Allen.xml @@ -0,0 +1,79 @@ +<tasks_inventory> + <!-- Task definitions for Dataflow Architecture: Markus_EBPass --> + <!-- This file is generated, do not modify by hand!!! --> + + <task name="EBReader" user="online" group="onliners" instances="2"> + <command>runFarmTask.sh</command> + <argument name="-type" value="${NAME}"/> + <argument name="-runinfo" value="${RUNINFO}"/> + <argument name="-partition" value="${PARTITION}"/> + <fmcparam name="utgid" value="${PARTITION}_${NODE}_${NAME}_${INSTANCE}"/> + <fmcparam name="wd" value="/group/online/dataflow/scripts"/> + <fmcparam name="stdout" value="/tmp/logGaudi.fifo"/> + <fmcparam name="stderr" value="/tmp/logGaudi.fifo"/> + <timeout action="Any" value="20"/> + </task> + + <task name="EBPass" user="online" group="onliners" instances="2"> + <command>runFarmTask.sh</command> + <argument name="-type" value="${NAME}"/> + <argument name="-runinfo" value="${RUNINFO}"/> + <argument name="-partition" value="${PARTITION}"/> + <fmcparam name="utgid" value="${PARTITION}_${NODE}_${NAME}_${INSTANCE}"/> + <fmcparam name="wd" value="/group/online/dataflow/scripts"/> + <fmcparam name="stdout" value="/tmp/logGaudi.fifo"/> + <fmcparam name="stderr" value="/tmp/logGaudi.fifo"/> + <timeout action="Any" value="20"/> + </task> + + <task name="EBStorage" user="online" group="onliners"> + <command>runFarmTask.sh</command> + <argument name="-type" value="${NAME}"/> + <argument name="-runinfo" value="${RUNINFO}"/> + <argument name="-partition" value="${PARTITION}"/> + <fmcparam name="utgid" value="${PARTITION}_${NODE}_${NAME}_${INSTANCE}"/> + <fmcparam name="wd" value="/group/online/dataflow/scripts"/> + <fmcparam name="stdout" value="/tmp/logGaudi.fifo"/> + <fmcparam name="stderr" value="/tmp/logGaudi.fifo"/> + <timeout action="Any" value="20"/> + </task> + + <task name="EBSender" user="online" group="onliners"> + <command>runFarmTask.sh</command> + <argument name="-type" value="${NAME}"/> + <argument name="-runinfo" value="${RUNINFO}"/> + <argument name="-partition" value="${PARTITION}"/> + <fmcparam name="utgid" value="${PARTITION}_${NODE}_${NAME}_${INSTANCE}"/> + <fmcparam name="wd" value="/group/online/dataflow/scripts"/> + <fmcparam name="stdout" value="/tmp/logGaudi.fifo"/> + <fmcparam name="stderr" value="/tmp/logGaudi.fifo"/> + <timeout action="Any" value="20"/> + </task> + + <task name="EBMBM" user="online" group="onliners"> + <command>runFarmTask.sh</command> + <argument name="-type" value="${NAME}"/> + <argument name="-runinfo" value="${RUNINFO}"/> + <argument name="-partition" value="${PARTITION}"/> + <fmcparam name="utgid" value="${PARTITION}_${NODE}_${NAME}_${INSTANCE}"/> + <fmcparam name="wd" value="/group/online/dataflow/scripts"/> + <fmcparam name="stdout" value="/tmp/logGaudi.fifo"/> + <fmcparam name="stderr" value="/tmp/logGaudi.fifo"/> + <timeout action="Any" value="20"/> + </task> + + <task name="EBAdder" user="online" group="onliners"> + <command>runFarmTask.sh</command> + <argument name="-type" value="${NAME}"/> + <argument name="-runinfo" value="${RUNINFO}"/> + <argument name="-partition" value="${PARTITION}"/> + <argument name="-environ" value="ARCHITECTURE=${ARCHITECTURE}"/> + <fmcparam name="utgid" value="${PARTITION}_${NODE}_${NAME}_${INSTANCE}"/> + <fmcparam name="wd" value="/group/online/dataflow/scripts"/> + <fmcparam name="stdout" value="/tmp/logGaudi.fifo"/> + <fmcparam name="stderr" value="/tmp/logGaudi.fifo"/> + <timeout action="Any" value="20"/> + + </task> + +</tasks_inventory> diff --git a/TestBeam/options/OnlineEnv.opts b/TestBeam/options/OnlineEnv.opts index 5a4c0e0ba..cae14c321 100644 --- a/TestBeam/options/OnlineEnv.opts +++ b/TestBeam/options/OnlineEnv.opts @@ -10,3 +10,4 @@ OnlineEnv.OutputLevel = 3; OnlineEnv.AcceptRate = 1.0; OnlineEnv.OnlineVersion = "None"; OnlineEnv.passThroughDelay = 0; +OnlineEnv.HasMonitoring = false; diff --git a/TestBeam/options/OnlineEnvBase.py b/TestBeam/options/OnlineEnvBase.py index ea02b39fd..7bbc93331 100644 --- a/TestBeam/options/OnlineEnvBase.py +++ b/TestBeam/options/OnlineEnvBase.py @@ -8,3 +8,4 @@ MooreVersion = "v0" MooreOnlineVersion = "v0" MooreStartupMode = 0 OutputLevel = 3 +HasMonitoring = False diff --git a/TestBeam/setup_comp.sh b/TestBeam/setup_comp.sh index 183f27a90..457aed1eb 100644 --- a/TestBeam/setup_comp.sh +++ b/TestBeam/setup_comp.sh @@ -41,9 +41,12 @@ fi; # # H3=`echo $DIM_DNS_NODE | tr a-z A-Z | cut -b 1-3`; +EB=`echo $DIM_DNS_NODE | tr a-z A-Z | cut -b 3-4`; MAINDNS=${DIM_DNS_NODE}; if test "${H3}" = "HLT"; then MAINDNS="ecstms01"; +elif test "${EB}" = "EB"; then + MAINDNS="ecstms01"; fi; # # @@ -64,6 +67,15 @@ start_gui() { -runinfo=\${MONITOR_BASE}/options/OnlineEnvBase.py \ -architecture=\${MONITOR_BASE}/options/DataflowArch_${1}.xml" & } +start_gui_Allen() { + xterm -title "TestBeam GUI" -geo 180x20 -e ". ${INSTALLATION}/setup.${BINARY_TAG}.vars; \ + gentest.exe libTestBeamGui.so testbeam_node_gui \ + -maxinst=1 -instances=1 -partition=ALLEN -host=${HOST} -maindns=${MAINDNS} \ + -ctrl_script=/group/online/dataflow/scripts/runFarmTask.sh \ + -runinfo=/group/online/dataflow/options/ALLEN/OnlineEnvBase.py \ + -architecture=/group/online/dataflow/options/ALLEN/Architecture.xml" & +} +# start_gui_MDF() { xterm -title "TestBeam GUI" -geo 180x20 -e ". ${INSTALLATION}/setup.${BINARY_TAG}.vars; \ gentest.exe libTestBeamGui.so testbeam_node_gui \ -- GitLab From 505d83420e8bc974dccbce27b603d333662ab34a Mon Sep 17 00:00:00 2001 From: Default Online user <Markus.Frank@cern.ch> Date: Wed, 13 Oct 2021 14:29:57 +0200 Subject: [PATCH 5/5] Remove internal classes from evaluator. Use STL provided classes --- Online/IPMI/src/NewLib/helper.h | 13 +- Online/IPMI/src/NewLib/ipmi_session.h | 4 +- Online/Parsers/src/Evaluator/Evaluator.cpp | 69 +++---- Online/Parsers/src/Evaluator/hash_map.src | 217 --------------------- Online/Parsers/src/Evaluator/stack.src | 47 ----- Online/Parsers/src/Evaluator/string.src | 144 -------------- Online/PyDIM/src/pydim_utils.cpp | 4 +- 7 files changed, 49 insertions(+), 449 deletions(-) delete mode 100644 Online/Parsers/src/Evaluator/hash_map.src delete mode 100644 Online/Parsers/src/Evaluator/stack.src delete mode 100644 Online/Parsers/src/Evaluator/string.src diff --git a/Online/IPMI/src/NewLib/helper.h b/Online/IPMI/src/NewLib/helper.h index 0157a6961..42432203d 100644 --- a/Online/IPMI/src/NewLib/helper.h +++ b/Online/IPMI/src/NewLib/helper.h @@ -73,8 +73,9 @@ //template <class ipmi_intf> ;//class ipmi_intf; #define IPM_DEV_MANUFACTURER_ID(x) \ ((uint32_t) ((x[2] & 0x0F) << 16 | x[1] << 8 | x[0])) + #ifdef HAVE_PRAGMA_PACK -#pragma pack(1) +#pragma pack(push,1) #endif struct ipm_devid_rsp { @@ -88,10 +89,10 @@ struct ipm_devid_rsp uint8_t product_id[2]; uint8_t aux_fw_rev[4]; }; + #ifdef HAVE_PRAGMA_PACK -#pragma pack(0) +#pragma pack(push,0) #endif - class valstr { public: @@ -152,6 +153,12 @@ template <class T> class ipmi_helper uint16_t ipmi_get_oem_id(ipmi_intf<T> *intf); IPMI_OEM ipmi_get_oem(ipmi_intf<T> *intf); }; + +#ifdef HAVE_PRAGMA_PACK +#pragma pack(pop) +#pragma pack(pop) +#endif + #define ipmi_open_file_read(file) ipmi_open_file(file, 0) #define ipmi_open_file_write(file) ipmi_open_file(file, 1) diff --git a/Online/IPMI/src/NewLib/ipmi_session.h b/Online/IPMI/src/NewLib/ipmi_session.h index fb7bbb15a..02c17137e 100644 --- a/Online/IPMI/src/NewLib/ipmi_session.h +++ b/Online/IPMI/src/NewLib/ipmi_session.h @@ -71,7 +71,7 @@ typedef enum * From table 22.25 of the IPMIv2 specification */ #ifdef HAVE_PRAGMA_PACK -#pragma pack(1) +#pragma pack(push,1) #endif struct get_session_info_rsp { @@ -148,7 +148,7 @@ struct get_session_info_rsp } channel_data; } /*ATTRIBUTE_PACKING*/; #ifdef HAVE_PRAGMA_PACK -#pragma pack(0) +#pragma pack(pop) #endif #define IPMI_AUTHSTATUS_PER_MSG_DISABLED 0x10 #define IPMI_AUTHSTATUS_PER_USER_DISABLED 0x08 diff --git a/Online/Parsers/src/Evaluator/Evaluator.cpp b/Online/Parsers/src/Evaluator/Evaluator.cpp index ce26a3fb7..ce6edf572 100644 --- a/Online/Parsers/src/Evaluator/Evaluator.cpp +++ b/Online/Parsers/src/Evaluator/Evaluator.cpp @@ -7,9 +7,9 @@ #include <memory> #include <mutex> #include <cmath> // for pow() -#include "stack.src" -#include "string.src" -#include "hash_map.src" +#include <stack> +#include <string> +#include <unordered_map> #include <string.h> #include <ctype.h> #include <errno.h> @@ -29,22 +29,23 @@ #define EVAL XmlTools::Evaluator //--------------------------------------------------------------------------- -struct Item { - enum { UNKNOWN, VARIABLE, EXPRESSION, FUNCTION, STRING } what; - double variable; - string expression; - void *function; +namespace { + + struct Item { + enum { UNKNOWN, VARIABLE, EXPRESSION, FUNCTION, STRING } what; + double variable; + std::string expression; + void *function; - explicit Item() : what(UNKNOWN), variable(0),expression(), function(0) {} - explicit Item(double x) : what(VARIABLE), variable(x),expression(), function(0) {} - explicit Item(string x) : what(EXPRESSION),variable(0),expression(x),function(0) {} - explicit Item(void *x) : what(FUNCTION), variable(0),expression(), function(x) {} -}; + explicit Item() : what(UNKNOWN), variable(0),expression(), function(0) {} + explicit Item(double x) : what(VARIABLE), variable(x),expression(), function(0) {} + explicit Item(std::string x) : what(EXPRESSION),variable(0),expression(x),function(0) {} + explicit Item(void *x) : what(FUNCTION), variable(0),expression(), function(x) {} + }; -typedef char * pchar; -typedef hash_map<string,Item> dic_type; + typedef char * pchar; + typedef std::unordered_map<std::string,Item> dic_type; -namespace { struct ParseResult { pchar theExpression = 0; pchar thePosition = 0; @@ -103,7 +104,7 @@ enum { ENDL, LBRA, OR, AND, EQ, NE, GE, GT, LE, LT, static int engine(pchar, pchar, double &, pchar &, const dic_type &); -static int variable(const string & name, double & result, +static int variable(const std::string & name, double & result, const dic_type & dictionary) /*********************************************************************** * * @@ -140,7 +141,7 @@ static int variable(const string & name, double & result, } } -static int function(const string & name, stack<double> & par, +static int function(const std::string & name, std::stack<double> & par, double & result, const dic_type & dictionary) /*********************************************************************** * * @@ -243,7 +244,7 @@ static int operand(pchar begin, pchar end, double & result, } c = *pointer; *pointer = '\0'; - string name(begin); + std::string name(begin); *pointer = c; // G E T V A R I A B L E @@ -257,8 +258,8 @@ static int operand(pchar begin, pchar end, double & result, // G E T F U N C T I O N - stack<pchar> pos; // position stack - stack<double> par; // parameter stack + std::stack<pchar> pos; // position stack + std::stack<double> par; // parameter stack double value; pchar par_begin = pointer+1, par_end; @@ -320,7 +321,7 @@ static int operand(pchar begin, pchar end, double & result, * val - stack of values. * * * ***********************************************************************/ -static int maker(int op, stack<double> & val) +static int maker(int op, std::stack<double> & val) { if (val.size() < 2) return EVAL::ERROR_SYNTAX_ERROR; double val2 = val.top(); val.pop(); @@ -431,9 +432,9 @@ static int engine(pchar begin, pchar end, double & result, { 4, 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 } // ^ }; - stack<int> op; // operator stack - stack<pchar> pos; // position stack - stack<double> val; // value stack + std::stack<int> op; // operator stack + std::stack<pchar> pos; // position stack + std::stack<double> val; // value stack double value; pchar pointer = begin; int iWhat, iCur, iPrev = 0, iTop, EVAL_STATUS; @@ -587,7 +588,7 @@ static void setItem(const char * prefix, const char * name, // A D D I T E M T O T H E D I C T I O N A R Y - string item_name = prefix + string(pointer,n); + std::string item_name = prefix + std::string(pointer,n); dic_type::iterator iter = (s->theDictionary).find(item_name); if (iter != (s->theDictionary).end()) { iter->second = item; @@ -695,8 +696,8 @@ namespace XmlTools { void Evaluator::setEnviron(const char* name, const char* value) { Struct* s = reinterpret_cast<Struct*>(p); std::lock_guard<std::mutex> guard(s->theLock); - string prefix = "${"; - string item_name = prefix + string(name) + string("}"); + std::string prefix = "${"; + std::string item_name = prefix + std::string(name) + std::string("}"); dic_type::iterator iter = (s->theDictionary).find(item_name); Item item; item.what = Item::STRING; @@ -720,7 +721,7 @@ namespace XmlTools { const char* Evaluator::getEnviron(const char* name) { Struct* s = reinterpret_cast<Struct*>(p); std::lock_guard<std::mutex> guard(s->theLock); - string item_name = name; + std::string item_name = name; //std::cout << " ++++++++++++++++++++++++++++ Try to resolve env:" << name << std::endl; dic_type::iterator iter = (s->theDictionary).find(item_name); if (iter != (s->theDictionary).end()) { @@ -729,7 +730,7 @@ namespace XmlTools { } if ( ::strlen(item_name.c_str()) > 3 ) { // Need to remove braces from ${xxxx} for call to getenv() - string env_name(name+2,::strlen(name)-3); + std::string env_name(name+2,::strlen(name)-3); const char* env_str = ::getenv(env_name.c_str()); if ( 0 != env_str ) { parse_result().theStatus = EVAL::OK; @@ -786,7 +787,7 @@ namespace XmlTools { Struct * s = reinterpret_cast<Struct*>(p); std::lock_guard<std::mutex> guard(s->theLock); return - ((s->theDictionary).find(string(pointer,n)) == (s->theDictionary).end()) ? + ((s->theDictionary).find(std::string(pointer,n)) == (s->theDictionary).end()) ? false : true; } @@ -798,7 +799,7 @@ namespace XmlTools { if (n == 0) return false; Struct * s = reinterpret_cast<Struct*>(p); std::lock_guard<std::mutex> guard(s->theLock); - return ((s->theDictionary).find(sss[npar]+string(pointer,n)) == + return ((s->theDictionary).find(sss[npar]+std::string(pointer,n)) == (s->theDictionary).end()) ? false : true; } @@ -809,7 +810,7 @@ namespace XmlTools { if (n == 0) return; Struct * s = reinterpret_cast<Struct*>(p); std::lock_guard<std::mutex> guard(s->theLock); - (s->theDictionary).erase(string(pointer,n)); + (s->theDictionary).erase(std::string(pointer,n)); } //--------------------------------------------------------------------------- @@ -820,7 +821,7 @@ namespace XmlTools { if (n == 0) return; Struct * s = reinterpret_cast<Struct*>(p); std::lock_guard<std::mutex> guard(s->theLock); - (s->theDictionary).erase(sss[npar]+string(pointer,n)); + (s->theDictionary).erase(sss[npar]+std::string(pointer,n)); } //--------------------------------------------------------------------------- diff --git a/Online/Parsers/src/Evaluator/hash_map.src b/Online/Parsers/src/Evaluator/hash_map.src deleted file mode 100644 index ddb1376d4..000000000 --- a/Online/Parsers/src/Evaluator/hash_map.src +++ /dev/null @@ -1,217 +0,0 @@ -// -*- C++ -*- -// --------------------------------------------------------------------------- - -#ifdef DEBUG_MODE -#include <iostream> -#endif - -#ifndef HEP_HASH_MAP_SRC -#define HEP_HASH_MAP_SRC - -#include <string.h> -#include <utility> -#include "string.src" - -/* - * Simplified hash_map class. - * It provides only basic functions of the standard <hash_map> and - * is intended to be used as a replacement of the standard class where - * full functionality of <hash_map> is not required, but it is essential - * to have highly portable and effective code. - * - * This file should be used exclusively inside *.cc files. - * Usage inside header files can result to a clash with standard <hash_map>. - * - * @author Evgeni Chernyaev <Evgueni.Tcherniaev@cern.ch> - */ -template<class K, class T> -class hash_map { -private: - hash_map(const hash_map& c) - : table(0), cur_size(0), max_size(0), max_load(0), - grow(0.0), default_value(c.default_value) - { - } - hash_map& operator=(const hash_map& c) { - table = 0; - cur_size = 0; - max_size = 0; - max_load = 0; - grow = 0.0; - default_value = c.default_value; - return *this; - } -public: - struct Entry { // Hash_map entry - std::pair<const K,T> data; - Entry* next; - Entry(K k, T v, Entry* n) : data(k,v), next(n) {} - }; - - class hash_map_iterator { // Hash_map iterator - Entry* entry; - public: - hash_map_iterator() : entry(0) {} - hash_map_iterator(Entry & e) : entry(&e) {} - std::pair<const K,T> & operator * () const { return entry->data; } - std::pair<const K,T> * operator ->() const { return &(operator*()); } - bool operator==(hash_map_iterator i) const { - return (entry == i.entry); - } - bool operator!=(hash_map_iterator i) const { - return (entry != i.entry); - } - }; - -public: - typedef unsigned int size_type; - typedef std::pair<const K,T> value_type; - typedef hash_map_iterator iterator; - typedef hash_map_iterator const_iterator; - -private: - Entry** table; // Hash table: pointers to entries - size_type cur_size; // Number of entries - size_type max_size; // Bucket_count - current size of the table - float max_load; // Keep (n) <= (max_size * max_load) - float grow; // When necessary, resize(max_size * grow) - const T default_value; // Default value used by [] - - size_type hash(const char * key) const { - size_type res = 0; - while(*key) { res = res*31 + *key++; } - return res; - } - - size_type hash(const string & key) const { - return hash(key.c_str()); - } - - bool eq(const char * a, const char * b) const { - return (strcmp(a, b) == 0); - } - - bool eq(const string & a, const string & b) const { - return (a == b); - } - -public: - - // Constructor. - hash_map(const T & dv = T(), size_type n = 107) - : table(0), cur_size(0), max_size(0), default_value(dv) - { - set_load(); - resize(n); - } - - // Destructor. - ~hash_map() { - for(size_type i=0; i<max_size; i++) { - Entry* n = table[i]; - while(n) { Entry* p = n; n = p->next; delete p; } - } - delete [] table; - } - - // Sets load and grow parameters. - void set_load(float m = 0.7, float g = 1.7) { max_load = m; grow = g; } - - // Returns number of elements. - size_type size() const { return cur_size; } - - // Returns size of the hash table. - size_type bucket_count() const { return max_size; } - - // Resizes the hash table. - void resize(size_type s) { - if (s <= max_size) return; - Entry** tmp = table; - table = new Entry* [s]; - for (size_type k=0; k<s; k++) table[k] = 0; - for (size_type i=0; i<max_size; i++) { - Entry* n = tmp[i]; - while(n) { - Entry* p = n; - n = p->next; - size_type ii = hash(p->data.first) % s; - p->next = table[ii]; - table[ii] = p; - } - } - max_size = s; - delete [] tmp; - } - - // Subscripting. - T & operator[](const K & key) { - size_type i = hash(key) % max_size; - for (Entry* p=table[hash(key) % max_size]; p; p=p->next) { - if (eq(key,p->data.first)) return p->data.second; - } - if (cur_size++ >= max_size*max_load) { - resize(size_type(max_size*grow)); - i = hash(key) % max_size; - } - table[i] = new Entry(key, default_value, table[i]); - return table[i]->data.second; - } - - // Finds element with given key. - iterator find(const K & key) const { - size_type i = hash(key) % max_size; - for (Entry* p=table[i]; p; p=p->next) { - if (eq(key,p->data.first)) return iterator(*p); - } - return end(); - } - - // Erases element with given key. - size_type erase(const K & key) { - size_type i = hash(key) % max_size; - Entry* p = table[i]; - if (p == 0) return 0; - if (eq(key,p->data.first)) { - table[i] = p->next; delete p; cur_size--; return 1; - } - Entry** pp = &table[i]; - for (p=p->next; p; p=p->next) { - if (eq(key,p->data.first)) { - *pp = p->next; delete p; cur_size--; return 1; - }else{ - pp = &(p->next); - } - } - return 0; - } - - // Clears the hash table. - void clear() { - for(size_type i=0; i<max_size; i++) { - for (Entry* p=table[i]; p;) { - Entry* pp = p; p = p->next; delete pp; - } - table[i] = 0; - } - cur_size = 0; - } - - // Returns end iterator. - iterator end() const { return iterator(); } - -#ifdef DEBUG_MODE - // Prints content of the hash table. - void print() { - std::cout << endl; - for(size_type i=0; i<max_size; i++) { - for (Entry* p=table[i]; p; p=p->next) { - std::cout - << '"' << p->data.first << '"' << " " << p->data.second - << std::endl; - } - } - } -#endif -}; - -#endif /* HEP_HASH_MAP_SRC */ diff --git a/Online/Parsers/src/Evaluator/stack.src b/Online/Parsers/src/Evaluator/stack.src deleted file mode 100644 index 8079c56d5..000000000 --- a/Online/Parsers/src/Evaluator/stack.src +++ /dev/null @@ -1,47 +0,0 @@ -// -*- C++ -*- -// --------------------------------------------------------------------------- - -#ifndef HEP_STACK_SRC -#define HEP_STACK_SRC - -/* - * Simplified stack class. - * It is intended to be used as a replacement of the standard class where - * full functionality of <stack> is not required, but it is essential - * to have highly portable and effective code. - * - * This file should be used exclusively inside *.cc files. - * Usage inside header files can result to a clash with standard <stack>. - * - * @author Evgeni Chernyaev <Evgueni.Tcherniaev@cern.ch> - */ -template<class T> -class stack { -private: - int k, max_size; - T * v; - - stack(const stack& c) = delete; - stack& operator=(const stack&) = delete; - -public: - stack() : k(0), max_size(20), v(new T[20]) {} - ~stack() { delete [] v; } - - int size() const { return k; } - T top () const { return v[k-1]; } - T & top () { return v[k-1]; } - void pop () { k--; } - void push(T a) { - if (k == max_size) { - T * w = v; - max_size *= 2; - v = new T[max_size]; - for (int i=0; i<k; i++) v[i] = w[i]; - delete [] w; - } - v[k++] = a; - } -}; - -#endif /* HEP_STACK_SRC */ diff --git a/Online/Parsers/src/Evaluator/string.src b/Online/Parsers/src/Evaluator/string.src deleted file mode 100644 index 0d27fb862..000000000 --- a/Online/Parsers/src/Evaluator/string.src +++ /dev/null @@ -1,144 +0,0 @@ -// -*- C++ -*- -// --------------------------------------------------------------------------- - -#ifndef HEP_STRING_SRC -#define HEP_STRING_SRC - -#include <iostream> -#include <string.h> - -/* - * Simplified string class. - * It provides only few basic functions of the standard <string> and - * is intended to be used as a replacement of the standard class where - * full functionality of <string> is not required, but it is essential - * to have highly portable and effective code. - * - * This file should be used exclusively inside *.cc files. - * Usage inside header files can result to a clash with standard <string>. - */ -struct string { - struct srep { - char* s; // pointer to data - int n; // reference count - srep() : s(0), n(1) {} - } *p; - - // Default constructor. - string() { p = new srep; p->s = 0; } - - // Constructor from character string. - string(const char* s) { - size_t len = strlen(s)+1; - p = new srep; p->s = new char[len]; strncpy(p->s, s, len); - } - - // Constructor from character substring. - string(const char* s, unsigned int n) { - p = new srep; p->s = new char[n+1]; strncpy(p->s, s, n); *(p->s+n) = '\0'; - } - - // Copy constructor from string. - string(const string& x) { - x.p->n++; - p = x.p; - } - - // Destructor. - ~string() { - if (--p->n == 0) { - delete [] p->s; - delete p; - } - } - - // Assignment from character string. - string& operator=(const char* s) { - if (p->n > 1) { // disconnect self - p->n--; - p = new srep; - }else{ - delete [] p->s; // free old string - } - p->s = new char[strlen(s)+1]; - strcpy(p->s, s); - return *this; - } - - // Assignment from string. - string& operator=(const string & x) { - if ( this != &x ) { - x.p->n++; // protect against "st = st" - if (--p->n == 0) { delete [] p->s; delete p; } - p = x.p; - } - return *this; - } - - // Returns C-style character string. - const char* c_str() const { return p->s; } -}; - -// -// Concatinations. -// -inline string operator+(char a, const string & b) { - string s; - size_t len = strlen(b.c_str())+2; - s.p->s = new char[len]; - s.p->s[0] = a; - strncpy(s.p->s+1, b.c_str(),len-1); - return s; -} - -inline string operator+(const char * a, const string & b) { - int lena = strlen(a); - int lenb = strlen(b.c_str()); - string s; - s.p->s = new char[lena+lenb+1]; - strncpy(s.p->s, a, lena+1); - strncpy(s.p->s+lena, b.c_str(), lenb+1); - return s; -} - -inline string operator+(const string & a, const char * b) { - int lena = strlen(a.c_str()), lenb = strlen(b); - string s; s.p->s = new char[lena+lenb+1]; - strncpy(s.p->s, a.c_str(),lena); - strncpy(s.p->s+lena, b,lenb+1); - return s; -} - -inline string operator+(const string & a, const string & b) { - int lena = strlen(a.c_str()); - int lenb = strlen(b.c_str()); - string s; s.p->s = new char[lena+lenb+1]; - strncpy(s.p->s, a.c_str(), lena); - strncpy(s.p->s+lena, b.c_str(), lenb+1); - return s; -} - -// -// Comparisons. -// -inline bool operator==(const string & x, const char* s) { - return strcmp(x.p->s, s) == 0; -} -inline bool operator==(const string & x, const string & y) { - return strcmp(x.p->s, y.p->s) == 0; -} -inline bool operator!=(const string & x, const char* s) { - return strcmp(x.p->s, s) != 0; -} -inline bool operator!=(const string & x, const string & y) { - return strcmp(x.p->s, y.p->s) != 0; -} - -// -// Output to a stream. -// -std::ostream & operator<<(std::ostream & s, const string & x) { - return s << x.p->s; -} - -#endif /* HEP_STRING_SRC */ diff --git a/Online/PyDIM/src/pydim_utils.cpp b/Online/PyDIM/src/pydim_utils.cpp index 6e8c74904..9bace0ac1 100755 --- a/Online/PyDIM/src/pydim_utils.cpp +++ b/Online/PyDIM/src/pydim_utils.cpp @@ -201,8 +201,8 @@ static PyObject* pyCallFunction (PyObject* pyFunc, PyObject* args) PyGILState_STATE gstate; PyObject* res; - gstate = PyGILState_Ensure(); - res = PyEval_CallObject(pyFunc, args); + gstate = ::PyGILState_Ensure(); + res = PyObject_CallObject(pyFunc, args); if (!res) PyErr_Print(); PyGILState_Release(gstate); -- GitLab