diff --git a/Online/Dataflow/include/Dataflow/DiskReader.h b/Online/Dataflow/include/Dataflow/DiskReader.h index 8251532d682c775938d7730c4c37216d0e14f68c..e5ce2e622a97f34e3848bd539579521f7d1bb643 100644 --- a/Online/Dataflow/include/Dataflow/DiskReader.h +++ b/Online/Dataflow/include/Dataflow/DiskReader.h @@ -120,7 +120,7 @@ namespace Online { /// Property: Partition name string std::string m_partitionName; /// Property: Run-type string - std::string m_runType; + std::string m_activity; /// Property: Stream identifier std::string m_stream; /// Property: Name of the 'GO' command which should be used. If empty feature is disabled. diff --git a/Online/Dataflow/options/StorageReaderNFS.opts b/Online/Dataflow/options/StorageReaderNFS.opts deleted file mode 100644 index 52c1c53114541a3613df8c0ebf3dbbc0659774eb..0000000000000000000000000000000000000000 --- a/Online/Dataflow/options/StorageReaderNFS.opts +++ /dev/null @@ -1,40 +0,0 @@ -#pragma print off -#include "$INFO_OPTIONS" -#include "$FARMCONFIGROOT/options/Logging.opts" -#include "$FARMCONFIGROOT/options/Monitoring.opts" -// -Manager.Services = {"Dataflow_MBMClient/MBM", - "Dataflow_StorageReader/Reader", - "Dataflow_RunableWrapper/Wrap" - }; -// -Manager.Runable = "Wrap"; -Wrap.Callable = "Reader"; -// -Reader.PartitionName = @OnlineEnv.PartitionName; -Reader.RunType = @OnlineEnv.RunType; -Reader.Stream = "HLT1"; -Reader.AllowedRuns = { "*" }; -Reader.DeleteFiles = false; -Reader.SaveRest = false; -Reader.FilePrefix = "/objects/data/${PARTITION}/${RUN1000}"; -Reader.FilePrefix = "/daqarea1/objects/nfs_data/${PARTITION}"; -Reader.FilePrefix = "/daqarea1/objects/nfs_data/LHCb/HLT1"; -Reader.PartitionName = @OnlineEnv.PartitionName; -Reader.PauseSleep = 1; -Reader.InitialSleep = 1; -Reader.MaxPauseWait = 1; -Reader.RequireConsumers = 0; -Reader.PackingFactor = 40; -Reader.Buffer = "Events"; -Reader.Server = "XXEB09.lbdaq.cern.ch:8010"; -Reader.DataType = "nfs"; -Reader.NumBuffers = 2; -Reader.NumThreads = 1; -// -MBM.PartitionBuffers = true; -MBM.PartitionName = @OnlineEnv.PartitionName; -MBM.PartitionID = @OnlineEnv.PartitionID; -MBM.Buffers = { "Events" }; -// -Logger.OutputLevel = @OnlineEnv.OutputLevel; diff --git a/Online/Dataflow/options/StorageWriterNFS.opts b/Online/Dataflow/options/StorageWriterNFS.opts deleted file mode 100644 index 11111f05b944fd9ed8214329b318ad1cc07934c7..0000000000000000000000000000000000000000 --- a/Online/Dataflow/options/StorageWriterNFS.opts +++ /dev/null @@ -1,43 +0,0 @@ -#pragma print off -#include "$INFO_OPTIONS" -#include "$FARMCONFIGROOT/options/Logging.opts" -#include "$FARMCONFIGROOT/options/Monitoring.opts" - -Logger.OutputLevel = @OnlineEnv.OutputLevel; -// -Manager.Services = {"Dataflow_MBMClient/MBM", - "Dataflow_MBMSelector/EventProc", - "Dataflow_RunableWrapper/Wrap" - }; -Manager.Algorithms = {"Dataflow_EmptyProcessor/Empty"}; -Manager.Runable = "Wrap"; -Wrap.Callable = "EventProc"; -// -EventProc.REQ1 = "EvType=1;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0"; -EventProc.REQ2 = "EvType=2;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0"; -EventProc.Input = "Output"; -// -// -OnlineEnv.PartitionBuffers = true; -MBM.PartitionBuffers = true; -MBM.PartitionName = @OnlineEnv.PartitionName; -MBM.PartitionID = @OnlineEnv.PartitionID; -MBM.Buffers = {"Output"}; -// -// -Manager.Algorithms = {"Dataflow_StorageWriter/Writer"}; -Writer.PartitionName = @OnlineEnv.PartitionName; -Writer.RunType = @OnlineEnv.RunType; -Writer.Stream = "HLT1"; -// -Writer.MinFileSizeMB = 10; -Writer.PollTimeout = 100000; // micro-seconds -Writer.NumThreads = 1; -Writer.CancelTimeout = 100; // seconds -Writer.Server = "XXEB09.lbdaq.cern.ch:8010"; -Writer.BufferSizeMB = 980; -Writer.NumBuffers = 3; -Writer.FileName = "file:/daqarea1/objects/nfs_data/${PARTITION}/${STREAM}/${RUN1000}/${RUN}/Run_${RUN}_${NODE}_${TIME}_${PID}_${SEQ}.mdf"; -Writer.OutputType = "NFS"; -Writer.MaxFileSizeMB = 5000; -Writer.HaveFileDB = 2; diff --git a/Online/Dataflow/src/Storage/StorageWriter.cpp b/Online/Dataflow/src/Storage/StorageWriter.cpp index 4ffe6af0755c315c95c10a5329551bbc1d68f15a..303c424b639ff9cf1b85a8a477c975252a95a818 100644 --- a/Online/Dataflow/src/Storage/StorageWriter.cpp +++ b/Online/Dataflow/src/Storage/StorageWriter.cpp @@ -86,7 +86,7 @@ namespace { struct StorageWriter::POSIX_FILE { RawFile file { }; std::mutex lock; - long length { 0 }; + uint64_t length { 0 }; uint32_t run { 0 }; std::time_t last_write { 0 }; POSIX_FILE() = default; @@ -131,7 +131,7 @@ struct StorageWriter::POSIX_FILE { /// ROOT implementation struct StorageWriter::ROOT_FILE { std::unique_ptr file; - long length = 0; + uint64_t length = 0; uint32_t run = 0; std::time_t last_write = 0; std::mutex lock; @@ -223,7 +223,7 @@ StorageWriter::StorageWriter(const std::string& nam, Context& ctxt) this->declareProperty("MaxEvents", m_max_events = std::numeric_limits::max()); this->declareProperty("Stream", m_stream = "RAW"); - this->declareProperty("RunType", m_run_type = ""); + this->declareProperty("Activity", m_activity = ""); this->declareProperty("RunList", m_run_list); this->declareProperty("PartitionName", m_partition_name = "LHCb"); this->declareProperty("FileName", m_file_name = fname); @@ -729,7 +729,7 @@ std::string StorageWriter::makeFileName(int run) { file_name = RTL::str_replace(file_name, "${TIME}", text); file_name = RTL::str_replace(file_name, "${NODE}", RTL::nodeNameShort()); file_name = RTL::str_replace(file_name, "${STREAM}", this->m_stream); - file_name = RTL::str_replace(file_name, "${RUNTYPE}", this->m_run_type); + file_name = RTL::str_replace(file_name, "${ACTIVITY}", this->m_activity); file_name = RTL::str_replace(file_name, "${PARTITION}", part); ::snprintf(text, sizeof(text), "%010d", int((run/1000)*1000)); file_name = RTL::str_replace(file_name, "${RUN1000}", text); @@ -744,7 +744,9 @@ std::string StorageWriter::makeFileName(int run) { /// Thread invocation routine to save assembled buffers to the disk server template -int StorageWriter::process_posix_buffers(std::mutex& queue_lock, std::map >& open_files) { +int StorageWriter::process_posix_buffers(std::mutex& queue_lock, + std::map >& open_files) +{ auto add_db_entry = [this] (size_t len, const std::string& fname) { if ( this->m_have_file_db ) { http::HttpReply reply; @@ -844,14 +846,20 @@ int StorageWriter::process_posix_buffers(std::mutex& queue_lock, std::map to_erase; now = ::time(0); for(auto& output : open_files ) { - if ( this->m_curr_run >= output.second->run ) { - if ( (now - output.second->last_write) > this->m_idle_tmo ) { - // if the current run is still the active run AND - // there are still events (very slowly) flowing, do not close prematurely. - if ( (now - this->m_last_event_stamp) > this->m_idle_tmo ) { - close_output(output.second, "output-idle", 1); - to_erase.emplace_back(output.first); - } + auto run = output.first; + if ( this->m_curr_run >= run ) { + // if the current run is still the active run AND + // there are still events (very slowly) flowing, do not close prematurely + // unless the file is nearly full (90%). + // This is mainly for HLT2, where accepted events may be very few only.... + auto& file = output.second; + bool close_file = (now - file->last_write) > this->m_idle_tmo; + close_file &= + ((now - this->m_last_event_stamp) > this->m_idle_tmo) || + ( file->length > this->m_max_file_size_MB * MBYTE * 0.9); + if ( close_file ) { + close_output(file, "output-idle", 1); + to_erase.emplace_back(run); } } } @@ -872,7 +880,7 @@ int StorageWriter::process_posix_buffers(std::mutex& queue_lock, std::maplength > 0 && output->length + len > m_max_file_size_MB * MBYTE ) + if ( output.get() && output->length > 0 && output->length + len > this->m_max_file_size_MB * MBYTE ) close_output(output, "size-limit", 1); if ( !output.get() ) { @@ -944,7 +952,7 @@ int StorageWriter::process_network_buffers() { } } if ( b.buffer ) { - long len = b.pointer - b.buffer; + std::size_t len = b.pointer - b.buffer; /// Skip files with less than a minimum bytes if ( len > this->m_min_file_size_MB*MBYTE ) { if ( this->write_buffer(b) != DF_SUCCESS ) { @@ -987,7 +995,7 @@ int StorageWriter::process_buffers() { case ROOT_STORAGE: if ( this->m_threadFileQueues ) { std::mutex queue_lock; - std::map > files; + std::map > files; return this->process_posix_buffers(queue_lock, files); } return this->process_posix_buffers(this->m_output_lock,this->m_open_root_files); @@ -995,7 +1003,7 @@ int StorageWriter::process_buffers() { default: if ( this->m_threadFileQueues ) { std::mutex queue_lock; - std::map > files; + std::map > files; return this->process_posix_buffers(queue_lock, files); } return this->process_posix_buffers(this->m_output_lock,this->m_open_posix_files); diff --git a/Online/Dataflow/src/Storage/StorageWriter.h b/Online/Dataflow/src/Storage/StorageWriter.h index a9b537520edc5908da1216aef6a56a06d03b0aff..227fe6ef8d34b35e8bebc727e6a25279edcad657 100644 --- a/Online/Dataflow/src/Storage/StorageWriter.h +++ b/Online/Dataflow/src/Storage/StorageWriter.h @@ -68,8 +68,8 @@ namespace Online { struct POSIX_FILE; struct ROOT_FILE; - std::map > m_open_posix_files; - std::map > m_open_root_files; + std::map > m_open_posix_files; + std::map > m_open_root_files; protected: /// Property: FDB server name @@ -77,7 +77,7 @@ namespace Online { /// Property: Partition name string std::string m_partition_name { }; /// Property: Run-type string - std::string m_run_type { }; + std::string m_activity { }; /// Property: Stream identifier std::string m_stream { }; /// File Name Pattern @@ -93,9 +93,9 @@ namespace Online { /// Property: Number of event sender threads std::size_t m_num_threads { 0 }; /// Property: Minimal file size in MBytes - long m_min_file_size_MB { 0 }; + std::size_t m_min_file_size_MB { 0 }; /// Property: Maximal file size in MBytes (used if writing to posix file system) - long m_max_file_size_MB { 512 }; + std::size_t m_max_file_size_MB { 512 }; /// Property: Sleep in milliseconds between retries when write connection fails int m_write_error_sleep { 0 }; /// Property: Number of retries when write connection fails @@ -185,7 +185,7 @@ namespace Online { int process_network_buffers(); template int process_posix_buffers(std::mutex& queue_lock, - std::map >& open_files); + std::map >& open_files); /// Get current buffer. If empty check for new one... Buffer& get_buffer(uint32_t run, int64_t length); diff --git a/Online/Dataflow/src/components/FileWriter.cpp b/Online/Dataflow/src/components/FileWriter.cpp index 38c367711f4c22b3dc7436438a79dc121a3bb199..f90b5881a3549a98ac468c60ec276b981df3e323 100644 --- a/Online/Dataflow/src/components/FileWriter.cpp +++ b/Online/Dataflow/src/components/FileWriter.cpp @@ -113,7 +113,7 @@ FileWriter::FileWriter(const std::string& nam, Context& ctxt) this->declareProperty("NodePattern", this->m_nodePattern = "..eb.."); this->declareProperty("ExitOnError", this->m_ExitOnError = true); this->declareProperty("Stream", this->m_stream = "RAW"); - this->declareProperty("RunType", this->m_runType = ""); + this->declareProperty("Activity", this->m_activity = ""); this->declareProperty("PartitionName", this->m_partitionName = "LHCb"); this->m_thread = std::make_unique([this] { this->close_files(); }); } @@ -146,7 +146,7 @@ int FileWriter::initialize() { for (std::string dir_name : m_deviceList) { dir_name = RTL::str_replace(dir_name, "${NODE}", node); dir_name = RTL::str_replace(dir_name, "${STREAM}", this->m_stream); - dir_name = RTL::str_replace(dir_name, "${RUNTYPE}", this->m_runType); + dir_name = RTL::str_replace(dir_name, "${ACTIVITY}", this->m_activity); dir_name = RTL::str_replace(dir_name, "${PARTITION}", this->m_partitionName); this->info("Check directory: %s",dir_name.c_str()); @@ -417,7 +417,7 @@ FileWriter::FileDescr *FileWriter::openFile(RunDesc* run_desc) { file_name = RTL::str_replace(file_name, "${TIME}", file_time); file_name = RTL::str_replace(file_name, "${NODE}", node); file_name = RTL::str_replace(file_name, "${STREAM}", this->m_stream); - file_name = RTL::str_replace(file_name, "${RUNTYPE}", this->m_runType); + file_name = RTL::str_replace(file_name, "${ACTIVITY}", this->m_activity); file_name = RTL::str_replace(file_name, "${PARTITION}", this->m_partitionName); ::snprintf(text, sizeof(text), "%010d", int((runn/1000)*1000)); file_name = RTL::str_replace(file_name, "${RUN1000}", text); diff --git a/Online/Dataflow/src/components/FileWriter.h b/Online/Dataflow/src/components/FileWriter.h index 6f1995d1707fe25f5e99304b6f6924d42bbd0566..7584c2e5cec7ce3749dc07bbdd132c2f6e6775df 100644 --- a/Online/Dataflow/src/components/FileWriter.h +++ b/Online/Dataflow/src/components/FileWriter.h @@ -62,7 +62,7 @@ namespace Online { /// Property: Partition name string std::string m_partitionName; /// Property: Run-type string - std::string m_runType; + std::string m_activity; /// Property: Stream identifier std::string m_stream; /// Property: diff --git a/Online/Dataflow/src/framework/DiskReader.cpp b/Online/Dataflow/src/framework/DiskReader.cpp index 7da515ff4b4582a44c45d1ae034af5f86925661c..1d80952a92b7d1ec6d816863f68f2bf3e742cc2b 100644 --- a/Online/Dataflow/src/framework/DiskReader.cpp +++ b/Online/Dataflow/src/framework/DiskReader.cpp @@ -128,7 +128,7 @@ DiskReader::DiskReader(const string& nam, DataflowContext& ctxt) declareProperty("PatchOdin", m_patch_odin = 0); declareProperty("RunNumberService", m_runNumberService = ""); declareProperty("Stream", m_stream = "RAW"); - declareProperty("RunType", m_runType = ""); + declareProperty("Activity", m_activity = ""); declareProperty("PartitionName", m_partitionName = "LHCb"); m_allowedRuns.push_back("*"); ::lib_rtl_create_lock(0, &m_lock); @@ -145,7 +145,7 @@ DiskReader::~DiskReader() { string DiskReader::interprete_file_tags(const string& file) const { string file_name = file; file_name = RTL::str_replace(file_name, "${STREAM}", this->m_stream); - file_name = RTL::str_replace(file_name, "${RUNTYPE}", this->m_runType); + file_name = RTL::str_replace(file_name, "${ACTIVITY}", this->m_activity); file_name = RTL::str_replace(file_name, "${PARTITION}", this->m_partitionName); return file_name; } diff --git a/Online/FarmConfig/job/AligDrv.sh b/Online/FarmConfig/job/AligDrv.sh index 7a87d4dbfc33d1339059e4f17af9da7cd9cb2794..a2332e8743cef2b15b5c9ac00207c38de7c40d6a 100755 --- a/Online/FarmConfig/job/AligDrv.sh +++ b/Online/FarmConfig/job/AligDrv.sh @@ -19,7 +19,7 @@ fix_python() } # # -act=${RUN_TYPE} +act=${ACTIVITY} export PYTHONPATH=/group/online/dataflow/options/LHCbA:$PYTHONPATH; # # diff --git a/Online/FarmConfig/job/AligWrk.sh b/Online/FarmConfig/job/AligWrk.sh index 0cc8d0fa0885a8b96885270ae3c07a156bc6a479..47b1a28668f15b7634972a35517fba406076a1b7 100755 --- a/Online/FarmConfig/job/AligWrk.sh +++ b/Online/FarmConfig/job/AligWrk.sh @@ -21,7 +21,7 @@ fix_python() # # renice -n 11 -p $$ >>/dev/null -act=${RUN_TYPE} +act=${ACTIVITY} export PYTHONPATH=/group/online/dataflow/options/LHCbA:$PYTHONPATH; # # diff --git a/Online/FarmConfig/job/AlignReader.sh b/Online/FarmConfig/job/AlignReader.sh index 0fc3fc7be33f629ff317f0c4e81b102d31f39ce6..c61e0ab841b2f49611a1e8f9ea1b8e98796ebdd8 100755 --- a/Online/FarmConfig/job/AlignReader.sh +++ b/Online/FarmConfig/job/AlignReader.sh @@ -9,6 +9,6 @@ if test "$NODE" = "ignore-test-HLTC1020"; then echo "[WARNING] Running TEST version of HLT2 reader"; fi; ##echo "[error] Go service name is: ${GO_SERVICE_NAME}"; -act=${RUN_TYPE} +act=${ACTIVITY} export AlignDir="/localdisk/Alignment/`echo $act|cut -d "|" -f 2`" exec -a ${UTGID} ${DATAFLOW_task} -class=Class2 -opts=../options/${TASK_TYPE}.opts diff --git a/Online/FarmConfig/job/BU.sh b/Online/FarmConfig/job/BU.sh index 32a8b4bda64019db2decc4ba4121949ee8254b9e..c02276a3765b2a3698c37f75dd1d78c1c3cd0585 100755 --- a/Online/FarmConfig/job/BU.sh +++ b/Online/FarmConfig/job/BU.sh @@ -13,7 +13,7 @@ unset PYTHONPATH; unset PYTHONHOME; eval `/usr/bin/python2 -c "import os;s=os.environ['UTGID'];print 'export BU_OPTIONS='+s[s.find('BU'):]+'.opts'"`; if test "${PARTITION}" = "TDET"; then - . /group/online/dataflow/EventBuilder/EventBuilderDev/setup.x86_64_v2-centos7-gcc10-opt.vars; + . /group/online/dataflow/EventBuilder/EventBuilderDev/setup.x86_64_v2-centos7-gcc10-do0.vars; elif test "${PARTITION}" = "FEST"; then . /group/online/dataflow/EventBuilder/EventBuilderDev/setup.x86_64_v2-centos7-gcc10-opt.vars; else diff --git a/Online/FarmConfig/job/EBSender.sh b/Online/FarmConfig/job/EBSender.sh index 212d6385fe911d4e6ee946b22f22f3c310b69bea..a680aa87bd276f99bb2c3b0399b0f6d36117e18e 100755 --- a/Online/FarmConfig/job/EBSender.sh +++ b/Online/FarmConfig/job/EBSender.sh @@ -26,7 +26,7 @@ rfilen() fi done } -act=${RUN_TYPE} +act=${ACTIVITY} # rfilen ${act} export EVENTSELECTOR_REQ1="EvType=2;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0"; diff --git a/Online/FarmConfig/job/RU.sh b/Online/FarmConfig/job/RU.sh index 1f980dc7ad4ce651aa9fb5785a3a0b53debbd024..57b32e3b5c11ba734aeab8c26a19c4ec90db5335 100755 --- a/Online/FarmConfig/job/RU.sh +++ b/Online/FarmConfig/job/RU.sh @@ -10,7 +10,7 @@ # ========================================================================= # if test "${PARTITION}" = "TDET"; then - . /group/online/dataflow/EventBuilder/EventBuilderDev/setup.x86_64_v2-centos7-gcc10-opt.vars; + . /group/online/dataflow/EventBuilder/EventBuilderDev/setup.x86_64_v2-centos7-gcc10-do0.vars; elif test "${PARTITION}" = "FEST"; then . /group/online/dataflow/EventBuilder/EventBuilderDev/setup.x86_64_v2-centos7-gcc10-opt.vars; else diff --git a/Online/FarmConfig/job/createEnvironment.sh b/Online/FarmConfig/job/createEnvironment.sh index 9d515f553510d26cc34689a034a115e0ee4cd7bb..ae1af78fce2a50c8423d4c0ac299043a3ea27142 100755 --- a/Online/FarmConfig/job/createEnvironment.sh +++ b/Online/FarmConfig/job/createEnvironment.sh @@ -122,7 +122,6 @@ try: _p('export HLT_ARCHITECTURE="'+str(Online.HltArchitecture)+'"') _p('export PARTITION='+str(Online.PartitionName)) _p('export PARTITION_NAME='+str(Online.PartitionName)) - _p('export RUN_TYPE="'+str(Online.Activity)+'"') _p('export MOOREONLINE_VERSION='+str(Online.MooreOnlineVersion)) _p('export MOORE_VERSION='+str(Online.MooreVersion)) _p('export ONLINE_VERSION='+str(OnlineVersion)) @@ -132,8 +131,9 @@ try: _p('export TAE_PROCESSING="TAE"') if hasattr(Online,'HLTType'): _p('export HLT_TYPE="'+str(Online.HLTType)+'"') - if hasattr(Online,'RunType'): - _p('export RUN_TYPE="'+str(Online.RunType)+'"') + if hasattr(Online,'Activity'): + _p('export RUN_TYPE="'+str(Online.Activity)+'"') + _p('export ACTIVITY="'+str(Online.Activity)+'"') except Exception as X: _p('echo '+str(X)) diff --git a/Online/FarmConfig/job/interSetup.sh b/Online/FarmConfig/job/interSetup.sh index e301bdaffbd9bd12b024e8f7bdeb6d1339464ad9..4ac850a5e537502ce42aa9627c7019d2d95f452f 100755 --- a/Online/FarmConfig/job/interSetup.sh +++ b/Online/FarmConfig/job/interSetup.sh @@ -57,7 +57,7 @@ try: _p('export LOGFIFO='+str('/run/fmc/logGaudi.fifo')) _p('export ONLINE_ENV_DIR='+dir) _p('export PARTITION_NAME='+str(Online.PartitionName)) - _p('export RUN_TYPE='+str(Online.Activity)) + _p('export ACTIVITY='+str(Online.Activity)) _p('export MOOREONLINE_VERSION='+str(Online.MooreOnlineVersion)) _p('export MOORE_VERSION='+str(Online.MooreVersion)) _p('export ONLINE_VERSION='+str(Online.OnlineVersion)) diff --git a/Online/FarmConfig/options/StorageReader.opts b/Online/FarmConfig/options/StorageReader.opts index 7b7cd1b752d51ae4d45b3a2aa19c4aee3428345d..82e3d60ab0f809500783fe36e72ad212c4e6e798 100644 --- a/Online/FarmConfig/options/StorageReader.opts +++ b/Online/FarmConfig/options/StorageReader.opts @@ -11,9 +11,9 @@ Manager.Services = {"Dataflow_MBMClient/MBM", Manager.Runable = "Wrap"; Wrap.Callable = "Reader"; // -Reader.PartitionName = @OnlineEnv.PartitionName; -Reader.RunType = @OnlineEnv.RunType; -Reader.Stream = "HLT1"; +Reader.PartitionName = @OnlineEnv.PartitionName; +Reader.Activity = @OnlineEnv.Activity; +Reader.Stream = "HLT1"; Reader.AllowedRuns = { "*" }; Reader.DeleteFiles = false; Reader.SaveRest = false; diff --git a/Online/FarmConfig/options/StorageWriter.opts b/Online/FarmConfig/options/StorageWriter.opts index 36bf6872d1c4bbdd9030ce431ee7df1dd7a63306..a19c2d5eb324b85854de1f99ac5afccfe9334f13 100644 --- a/Online/FarmConfig/options/StorageWriter.opts +++ b/Online/FarmConfig/options/StorageWriter.opts @@ -32,7 +32,7 @@ MBM.Buffers = {"Output"}; // Manager.Algorithms = {"Dataflow_StorageWriter/Writer"}; Writer.PartitionName = @OnlineEnv.PartitionName; -Writer.RunType = @OnlineEnv.RunType; +Writer.Activity = @OnlineEnv.Activity; Writer.Stream = "HLT1"; Writer.FileName = "/objects/data/${PARTITION}/${STREAM}/${RUN1000}/${RUN}/Run_${RUN}_${NODE}_${TIME}_${PID}_${SEQ}.mdf"; Writer.Server = "XXEB09.lbdaq.cern.ch:8000"; diff --git a/Online/GauchoAppl/src/components/BusyPub.cpp b/Online/GauchoAppl/src/components/BusyPub.cpp index 7931b5061b08f29180d08cb39a1a8144c2ce3057..5f04d97ff9b56b4d4bddc0414e650cde93819a08 100644 --- a/Online/GauchoAppl/src/components/BusyPub.cpp +++ b/Online/GauchoAppl/src/components/BusyPub.cpp @@ -89,25 +89,25 @@ void BusyPub::analyze(mem_buff& ,MonitorItems* mmap) { auto i = mmap->find("BusySvc/BusyFraction"); if (i == mmap->end()) return; - double bsy = (CounterSerDes::de_serialize((*i).second))->d_data; + double bsy = (CounterSerDes::de_serialize((*i).second))->scalars.d_data; i = mmap->find("BusySvc/BogoMIPS"); if (i == mmap->end()) return; - double bogo = (CounterSerDes::de_serialize((*i).second))->d_data; + double bogo = (CounterSerDes::de_serialize((*i).second))->scalars.d_data; i = mmap->find("BusySvc/NumCores"); if (i == mmap->end()) return; - long nCores = (CounterSerDes::de_serialize((*i).second))->i_data; + long nCores = (CounterSerDes::de_serialize((*i).second))->scalars.i_data; i = mmap->find("BusySvc/TotMemory"); - long memtot = (CounterSerDes::de_serialize((*i).second))->l_data; + long memtot = (CounterSerDes::de_serialize((*i).second))->scalars.l_data; i = mmap->find("BusySvc/MemAvail"); - long memavail = (CounterSerDes::de_serialize((*i).second))->l_data; + long memavail = (CounterSerDes::de_serialize((*i).second))->scalars.l_data; i = mmap->find("BusySvc/FreeMemory"); - long memfree = (CounterSerDes::de_serialize((*i).second))->l_data; + long memfree = (CounterSerDes::de_serialize((*i).second))->scalars.l_data; i = mmap->find("BusySvc/SwapSpaceTot"); - long swaptot = (CounterSerDes::de_serialize((*i).second))->l_data; + long swaptot = (CounterSerDes::de_serialize((*i).second))->scalars.l_data; i = mmap->find("BusySvc/SwapSpaceFree"); - long swapfree = (CounterSerDes::de_serialize((*i).second))->l_data; + long swapfree = (CounterSerDes::de_serialize((*i).second))->scalars.l_data; i = mmap->find("BusySvc/MemBuffers"); - long buffers = (CounterSerDes::de_serialize((*i).second))->l_data; + long buffers = (CounterSerDes::de_serialize((*i).second))->scalars.l_data; m_PubDescrs[0]->m_outbuff = float(bsy/bogo);m_PubDescrs[0]->update(); m_PubDescrs[1]->m_outbuff = 1.0-(float(memfree)/float(memtot));m_PubDescrs[1]->update(); m_PubDescrs[2]->m_outbuff = (float(buffers)/float(memtot));m_PubDescrs[2]->update(); @@ -117,25 +117,25 @@ void BusyPub::analyze(mem_buff& ,MonitorItems* mmap) { for (size_t cli= 0;clifind("BusySvc/"+m_NodeClasses[cli]+"/BusyFraction"); if (i == mmap->end()) continue; - bsy = (CounterSerDes::de_serialize((*i).second))->d_data; + bsy = (CounterSerDes::de_serialize((*i).second))->scalars.d_data; i = mmap->find("BusySvc/"+m_NodeClasses[cli]+"/BogoMIPS"); if (i == mmap->end()) continue; - bogo = (CounterSerDes::de_serialize((*i).second))->d_data; + bogo = (CounterSerDes::de_serialize((*i).second))->scalars.d_data; i = mmap->find("BusySvc/"+m_NodeClasses[cli]+"/NumCores"); if (i == mmap->end()) continue; - nCores = (CounterSerDes::de_serialize((*i).second))->i_data; + nCores = (CounterSerDes::de_serialize((*i).second))->scalars.i_data; i = mmap->find("BusySvc/"+m_NodeClasses[cli]+"/TotMemory"); - memtot = (CounterSerDes::de_serialize((*i).second))->l_data; + memtot = (CounterSerDes::de_serialize((*i).second))->scalars.l_data; i = mmap->find("BusySvc/"+m_NodeClasses[cli]+"/MemAvail"); - memavail = (CounterSerDes::de_serialize((*i).second))->l_data; + memavail = (CounterSerDes::de_serialize((*i).second))->scalars.l_data; i = mmap->find("BusySvc/"+m_NodeClasses[cli]+"/FreeMemory"); - memfree = (CounterSerDes::de_serialize((*i).second))->l_data; + memfree = (CounterSerDes::de_serialize((*i).second))->scalars.l_data; i = mmap->find("BusySvc/"+m_NodeClasses[cli]+"/SwapSpaceTot"); - swaptot = (CounterSerDes::de_serialize((*i).second))->l_data; + swaptot = (CounterSerDes::de_serialize((*i).second))->scalars.l_data; i = mmap->find("BusySvc/"+m_NodeClasses[cli]+"/SwapSpaceFree"); - swapfree = (CounterSerDes::de_serialize((*i).second))->l_data; + swapfree = (CounterSerDes::de_serialize((*i).second))->scalars.l_data; i = mmap->find("BusySvc/"+m_NodeClasses[cli]+"/MemBuffers"); - buffers = (CounterSerDes::de_serialize((*i).second))->l_data; + buffers = (CounterSerDes::de_serialize((*i).second))->scalars.l_data; m_PubDescrs[(cli+1)*6+0]->m_outbuff = float(bsy/bogo);m_PubDescrs[(cli+1)*6+0]->update(); m_PubDescrs[(cli+1)*6+1]->m_outbuff = 1.0-(float(memfree)/float(memtot));m_PubDescrs[(cli+1)*6+1]->update(); m_PubDescrs[(cli+1)*6+2]->m_outbuff = (float(buffers)/float(memtot));m_PubDescrs[(cli+1)*6+2]->update(); diff --git a/Online/GauchoAppl/src/components/CntrPub.cpp b/Online/GauchoAppl/src/components/CntrPub.cpp index 2f20de56c81ed4ef3a23952bf9331fd767b9adde..4998639bfdee83da29b5d031f7b75c47f0ffd93d 100644 --- a/Online/GauchoAppl/src/components/CntrPub.cpp +++ b/Online/GauchoAppl/src/components/CntrPub.cpp @@ -70,17 +70,17 @@ void CntrPub::analyze(mem_buff&, Online::MonitorItems* mmap) auto *d = Online::CounterSerDes::de_serialize(it.second); if (sdes->type == 1) { - sdes->rdata = d->d_data; + sdes->rdata = d->scalars.d_data; } else { if (d->type == C_INT) { - sdes->idata = d->i_data; + sdes->idata = d->scalars.i_data; } else if (d->type == C_LONGLONG) { - sdes->idata = int(d->l_data); + sdes->idata = int(d->scalars.l_data); } } sdes->svc->updateService(); diff --git a/Online/GauchoBase/include/Gaucho/CounterDeserialize.h b/Online/GauchoBase/include/Gaucho/CounterDeserialize.h index e715096112a1287fbcff74c8cdd73eb2f3340b31..b4fbc0844e08e9c758b663300e0f01b7e2b7381c 100644 --- a/Online/GauchoBase/include/Gaucho/CounterDeserialize.h +++ b/Online/GauchoBase/include/Gaucho/CounterDeserialize.h @@ -23,15 +23,27 @@ namespace Online { public: std::string name = ""; unsigned int type = 0; - int i_data = 0; - unsigned int ui_data = 0; - long l_data = 0; - unsigned long ul_data = 0; - float f_data = 0.0; - double d_data = 0.0; + + /// Union containing scalars. Everything smaller than int *is* int + union _scalars { + int i_data; + unsigned int ui_data; + long l_data; + unsigned long ul_data; + float f_data; + double d_data; + _scalars() { this->ul_data = 0; } + } scalars; + + /// Array holder std::unique_ptr ptr {}; int nel = 0; - std::pair ip_data; + + /// Stuff for pairs + std::pair ip_data; + std::pair lp_data; + + /// Union containing gaudi accumulator structures union { struct _avg_data { long entries; @@ -56,7 +68,6 @@ namespace Online { long entries_false; } binomial; } gaudi; - std::pair lp_data; CntrDescr() = default; ~CntrDescr() = default; }; diff --git a/Online/GauchoBase/include/Gaucho/MonTypes.h b/Online/GauchoBase/include/Gaucho/MonTypes.h index 41a77e673937666b378c03f609d3b5e2460ee3b6..8a581f06f168859f3ed1a147c0a69d7bc6a02bcc 100644 --- a/Online/GauchoBase/include/Gaucho/MonTypes.h +++ b/Online/GauchoBase/include/Gaucho/MonTypes.h @@ -18,7 +18,7 @@ /// -- ONLY ADD NEW TYPES TO THE VERY END! /// enum MONTYPE { - H_ILLEGAL = 0xFEED0000, + H_ILLEGAL = 0xFEED0000, H_1DIM, H_2DIM, H_PROFILE, @@ -34,7 +34,7 @@ enum MONTYPE { C_LONGSTAR, C_FLOATSTAR, C_DOUBLESTAR, - C_RATEDOUBLE, // 0xFEED0010 + C_RATEDOUBLE = 0xFEED0010, C_RATEDOUBLESTAR, H_3DIM, H_2DPROFILE, @@ -44,17 +44,20 @@ enum MONTYPE { C_ATOMICLONG, C_ATOMICFLOAT, C_ATOMICDOUBLE, + C_GAUDIACCLONG, C_GAUDIACCINT, C_GAUDIACCFLOAT, C_GAUDIACCDOUBLE, C_GAUDIAVGACCl, C_GAUDIAVGACCd, - C_GAUDIBINACC_temp, // 0xFEED0020 + C_GAUDIBINACC_temp = 0xFEED0020, + C_UINT, C_ULONG, C_UINTSTAR, C_ULONGSTAR, + H_GAUDIH1F, H_GAUDIH1D, H_GAUDIH2F, @@ -67,7 +70,7 @@ enum MONTYPE { C_GAUDIACCuLONG, C_GAUDIACCuINT, - C_GAUDIBINACCd, // 0xFEED0030 + C_GAUDIBINACCd = 0xFEED0030, C_GAUDIAVGACClu, C_GAUDIAVGACCf, C_GAUDISTATACCd, @@ -76,19 +79,19 @@ enum MONTYPE { C_GAUDISIGMAACCd, C_GAUDIMSGACClu, // 0xFEED0037 - C_GAUDIACCCHAR = 0xFEED0050, + C_GAUDIACCCHAR = 0xFEED0050, C_GAUDIACCuCHAR, C_GAUDIACCSHORT, C_GAUDIACCuSHORT, - C_GAUDIAVGACCc = 0xFEED0070, + C_GAUDIAVGACCc = 0xFEED0070, C_GAUDIAVGACCcu, C_GAUDIAVGACCs, C_GAUDIAVGACCsu, C_GAUDIAVGACCi, C_GAUDIAVGACCiu, - C_GAUDISIGMAACCc = 0xFEED0090, + C_GAUDISIGMAACCc = 0xFEED0090, C_GAUDISIGMAACCcu, C_GAUDISIGMAACCs, C_GAUDISIGMAACCsu, @@ -97,7 +100,7 @@ enum MONTYPE { C_GAUDISIGMAACClu, C_GAUDISIGMAACCf, - C_GAUDISTATACCc = 0xFEED00B0, + C_GAUDISTATACCc = 0xFEED00B0, C_GAUDISTATACCcu, C_GAUDISTATACCs, C_GAUDISTATACCsu, @@ -106,14 +109,14 @@ enum MONTYPE { C_GAUDISTATACCl, C_GAUDISTATACCf, - C_GAUDIBINACCc = 0xFEED00D0, + C_GAUDIBINACCc = 0xFEED00D0, C_GAUDIBINACCcu, C_GAUDIBINACCs, C_GAUDIBINACCsu, C_GAUDIBINACCi, C_GAUDIBINACCiu, - C_GAUDIBINACCl = C_GAUDIBINACC_temp, - C_GAUDIBINACClu = C_GAUDIBINACCiu+2, + C_GAUDIBINACCl = C_GAUDIBINACC_temp, + C_GAUDIBINACClu = C_GAUDIBINACCiu+2, C_GAUDIBINACCf, //C_GAUDIBINACCd, diff --git a/Online/GauchoBase/main/taskCounters.cpp b/Online/GauchoBase/main/taskCounters.cpp index c13191e29639b712189519cc9506b86ad24b4f21..b826435ef846a312205a43bce828f71b0430ea82 100644 --- a/Online/GauchoBase/main/taskCounters.cpp +++ b/Online/GauchoBase/main/taskCounters.cpp @@ -81,50 +81,62 @@ int main( int argc, char* argv[] ) { // Taskname, DNS Node for ( size_t i = 0; i < objs.size(); i++ ) { char text[128]; switch ( objs[i]->type ) { + case C_GAUDIACCCHAR: + case C_GAUDIACCuCHAR: + ::snprintf( text, sizeof( text ), "(CHAR) %d", int((signed char)objs[i]->scalars.i_data) ); + break; + case C_GAUDIACCSHORT: + case C_GAUDIACCuSHORT: + ::snprintf( text, sizeof( text ), "(SHORT) %d", int((signed short)objs[i]->scalars.i_data) ); + break; case C_INT: - snprintf( text, sizeof( text ), "(INT) %d", objs[i]->i_data ); + case C_ATOMICINT: + case C_GAUDIACCINT: + ::snprintf( text, sizeof( text ), "(INT) %d", (int)objs[i]->scalars.i_data ); + break; + case C_UINT: + case C_GAUDIACCuINT: + ::snprintf( text, sizeof( text ), "(UINT) %d", int((unsigned int)objs[i]->scalars.ui_data) ); break; case C_LONGLONG: - case C_ATOMICINT: case C_ATOMICLONG: case C_GAUDIACCLONG: - case C_GAUDIACCINT: - ::snprintf( text, sizeof( text ), "(LONGLONG) %ld", long( objs[i]->l_data ) ); + ::snprintf( text, sizeof( text ), "(LONG) %ld", long( objs[i]->scalars.l_data ) ); break; + case C_ULONG: case C_GAUDIACCuLONG: - case C_GAUDIACCuINT: - ::snprintf( text, sizeof( text ), "(Gaudi Counter or uLONGLONG) %ld", long( objs[i]->ul_data ) ); + ::snprintf( text, sizeof( text ), "(ULONG) %ld", (unsigned long)( objs[i]->scalars.ul_data ) ); break; case C_FLOAT: case C_ATOMICFLOAT: - snprintf( text, sizeof( text ), "(FLOAT) %f", objs[i]->f_data ); + ::snprintf( text, sizeof( text ), "(FLOAT) %f", objs[i]->scalars.f_data ); break; case C_DOUBLE: case C_ATOMICDOUBLE: - snprintf( text, sizeof( text ), "(DOUBLE) %f", objs[i]->d_data ); + ::snprintf( text, sizeof( text ), "(DOUBLE) %f", objs[i]->scalars.d_data ); break; case C_RATEDOUBLE: - snprintf( text, sizeof( text ), "(RATEDOUBLE) %f Hz", objs[i]->d_data ); + ::snprintf( text, sizeof( text ), "(RATEDOUBLE) %f Hz", objs[i]->scalars.d_data ); break; case C_RATEFLOAT: - snprintf( text, sizeof( text ), "(RATEFLOAT) %f Hz", objs[i]->f_data ); + ::snprintf( text, sizeof( text ), "(RATEFLOAT) %f Hz", objs[i]->scalars.f_data ); break; case C_INTSTAR: case C_UINTSTAR: for ( int j = 0, nel = objs[i]->nel, *p = (int*)objs[i]->ptr.get(); j < nel; j++ ) { - snprintf( text, sizeof( text ), "(INT*) index %d value %d", j, p[j] ); + ::snprintf( text, sizeof( text ), "(INT*) index %d value %d", j, p[j] ); } break; case C_LONGSTAR: case C_ULONGSTAR: { long* p = (long*)objs[i]->ptr.get(); int nel = objs[i]->nel; - for ( int j = 0; j < nel; j++ ) { snprintf( text, sizeof( text ), "(LONG*) index %d value %li", j, p[j] ); } + for ( int j = 0; j < nel; j++ ) { ::snprintf( text, sizeof( text ), "(LONG*) index %d value %li", j, p[j] ); } } break; case C_FLOATSTAR: { float* p = (float*)objs[i]->ptr.get(); int nel = objs[i]->nel; - for ( int j = 0; j < nel; j++ ) { snprintf( text, sizeof( text ), "(FLOAT*) index %d value %f", j, p[j] ); } + for ( int j = 0; j < nel; j++ ) { ::snprintf( text, sizeof( text ), "(FLOAT*) index %d value %f", j, p[j] ); } } break; case C_DOUBLESTAR: case C_RATEDOUBLESTAR: { @@ -207,7 +219,7 @@ int main( int argc, char* argv[] ) { // Taskname, DNS Node break; case C_GAUDIMSGACClu: - ::snprintf( text, sizeof( text ), "(Gaudi Msg Counter) %lu", long( objs[i]->ul_data ) ); + ::snprintf( text, sizeof( text ), "(Gaudi Msg Counter) %lu", long( objs[i]->scalars.ul_data ) ); break; default: snprintf( text, sizeof( text ), "Unknown Counter" ); diff --git a/Online/GauchoBase/src/CounterDeserialize.cpp b/Online/GauchoBase/src/CounterDeserialize.cpp index 4aa4e8aaa62f3005145b259ba02fdd440397f264..358abafbe997752a76ca52ba32eb4d99a9a8d09c 100644 --- a/Online/GauchoBase/src/CounterDeserialize.cpp +++ b/Online/GauchoBase/src/CounterDeserialize.cpp @@ -46,44 +46,46 @@ CntrDescr* CounterSerDes::de_serialize( void* ptr, char* nam ) { DimBuffBase* p = (DimBuffBase*)ptr; if ( nam == 0 ) { nam = add_ptr( p, p->nameoff ); } auto h = std::make_unique(); - h->i_data = 0; - h->l_data = 0; - h->f_data = 0.0; - h->d_data = 0.0; - h->name = nam; - h->type = p->type; void* dat = add_ptr( p, p->dataoff ); long* plng = add_ptr( p, p->dataoff ); int bsiz = p->reclen - p->dataoff; + + h->name = nam; + h->type = p->type; + h->scalars.ul_data = 0; switch ( p->type ) { + case C_GAUDIACCCHAR: + case C_GAUDIACCuCHAR: + + case C_GAUDIACCSHORT: + case C_GAUDIACCuSHORT: + case C_INT: - h->i_data = *(int*)dat; + case C_ATOMICINT: + case C_GAUDIACCINT: + h->scalars.i_data = *(int*)dat; break; case C_UINT: - h->ui_data = *(unsigned int*)dat; - break; - case C_ULONG: - h->ul_data = *(unsigned long*)dat; + case C_GAUDIACCuINT: + h->scalars.ui_data = *(unsigned int*)dat; break; case C_LONGLONG: - case C_ATOMICINT: case C_ATOMICLONG: case C_GAUDIACCLONG: - case C_GAUDIACCINT: - h->l_data = *(long*)dat; + h->scalars.l_data = *(long*)dat; break; + case C_ULONG: case C_GAUDIACCuLONG: case C_GAUDIMSGACClu: - case C_GAUDIACCuINT: - h->ul_data = *(unsigned long*)dat; + h->scalars.ul_data = *(unsigned long*)dat; break; case C_FLOAT: case C_RATEFLOAT: - h->f_data = (float)( *(double*)dat ); + h->scalars.f_data = (float)( *(double*)dat ); break; case C_DOUBLE: case C_RATEDOUBLE: - h->d_data = *(double*)dat; + h->scalars.d_data = *(double*)dat; break; case C_INTSTAR: case C_UINTSTAR: diff --git a/Online/OnlineAlign/src/Fitter.cpp b/Online/OnlineAlign/src/Fitter.cpp index 0c1b1f5a462deb59c7e19c684af3e64219a0bc48..2c6b07c50e780922281e9a3b342b53e25aa592bf 100644 --- a/Online/OnlineAlign/src/Fitter.cpp +++ b/Online/OnlineAlign/src/Fitter.cpp @@ -147,7 +147,7 @@ double Fitter::getIterationResult() { std::vector cdesc; task.counters(this->m_counterNames, cdesc); if ( cdesc.size() > 0 ) { - double result = cdesc[0]->d_data; + double result = cdesc[0]->scalars.d_data; always() << "Got " << cdesc.size() << " Counters Found matching:" << " DNS: '" << this->m_counterTaskDNS << "'" << " Task: '" << this->m_counterTaskName << "'"