From 72d17b77231de8c90d6c31b400b08c2c68477de4 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Thu, 4 Mar 2021 16:38:07 +0100 Subject: [PATCH 01/26] Add Storage client --- Online/Dataflow/CMakeLists.txt | 3 +- Online/Dataflow/Dataflow/Receiver.h | 4 +- Online/Dataflow/options/StorageWriter.opts | 9 +- Online/Dataflow/src/Storage/StorageWriter.cpp | 249 ++++++++++++------ Online/Dataflow/src/Storage/StorageWriter.h | 18 +- .../Dataflow/src/components/MBMSelector.cpp | 10 +- .../Dataflow/src/components/MEPSelector.cpp | 10 +- .../src/components/UpgradeMEPReader.cpp | 70 +++-- .../src/components/UpgradeMEPSelector.cpp | 10 +- .../src/framework/DataflowManager.cpp | 5 + Online/Dataflow/src/framework/DiskReader.cpp | 37 +-- Online/Dataflow/src/framework/Receiver.cpp | 44 +++- Online/GaudiOnline/GaudiOnline/EventAccess.h | 12 +- Online/GaudiOnline/GaudiOnline/EventTraits.h | 6 +- Online/GaudiOnline/components/InputAlg.cpp | 6 +- .../GaudiOnline/components/OnlineEventApp.cpp | 91 +++---- Online/GaudiOnline/components/OutputAlg.cpp | 12 +- Online/GaudiOnline/components/Passthrough.cpp | 20 +- Online/GaudiOnline/src/EventAccess.cpp | 19 +- Online/GaudiOnline/src/MBMEventAccess.cpp | 17 +- Online/GaudiOnline/src/MBMEventOutput.cpp | 18 +- Online/OnlineBase/src/MBM/mbm_extract.cpp | 33 ++- Online/PCIE40Data/PCIE40Data/RawBank40.h | 1 + Online/PCIE40Data/PCIE40Data/pcie40.h | 148 +++++++---- Online/PCIE40Data/main/pcie40_decode_file.cpp | 9 + Online/SmiController/options/TestMEPProd.opts | 1 + .../SmiController/scripts/EventProcessor.py | 6 +- Online/SmiController/scripts/TestMEPProd.sh | 3 + .../scripts/TestStorageWriter.sh | 3 +- Online/Storage/src/server/fdb_SQLite.cpp | 16 +- Online/Storage/src/server/fdb_SQLite.h | 4 +- Online/Storage/src/server/fdb_db_server.cpp | 4 +- Online/Tell1Data/Tell1Data/RawFile.h | 2 +- Online/Tell1Data/src/RawFile.cpp | 13 + 34 files changed, 617 insertions(+), 296 deletions(-) create mode 100644 Online/SmiController/scripts/TestMEPProd.sh diff --git a/Online/Dataflow/CMakeLists.txt b/Online/Dataflow/CMakeLists.txt index a7b9474fe..bf780fad6 100755 --- a/Online/Dataflow/CMakeLists.txt +++ b/Online/Dataflow/CMakeLists.txt @@ -23,6 +23,7 @@ gaudi_depends_on_subdirs(Online/dim Online/UPI Online/Storage Online/Tell1Data + Online/PCIE40Data Online/GauchoBase Online/OnlineKernel Online/PCIE40Data) @@ -185,7 +186,7 @@ gaudi_add_library(DataflowStorage src/Storage/*.cpp INCLUDE_DIRS NO_PUBLIC_HEADERS - LINK_LIBRARIES dim Boost OnlineBase Options DataflowLib Tell1Data StorageClient) + LINK_LIBRARIES dim Boost OnlineBase Options DataflowLib Tell1Data PCIE40Data StorageClient) target_include_directories(DataflowStorage BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) gaudi_generate_componentslist(DataflowStorage) diff --git a/Online/Dataflow/Dataflow/Receiver.h b/Online/Dataflow/Dataflow/Receiver.h index 26c2dc115..ed80d6a50 100755 --- a/Online/Dataflow/Dataflow/Receiver.h +++ b/Online/Dataflow/Dataflow/Receiver.h @@ -19,12 +19,10 @@ // Framework include files #include "Dataflow/DataflowComponent.h" +#include "MBM/Producer.h" #include "RTL/rtl.h" #include <mutex> -// Forward declarations -namespace MBM { class Producer; } - /// Online namespace declaration namespace Online { diff --git a/Online/Dataflow/options/StorageWriter.opts b/Online/Dataflow/options/StorageWriter.opts index b82311311..b48a52f45 100644 --- a/Online/Dataflow/options/StorageWriter.opts +++ b/Online/Dataflow/options/StorageWriter.opts @@ -1,5 +1,7 @@ #pragma print off #include "$INFO_OPTIONS" +#include "$FARMCONFIGROOT/options/Logging.opts" +#include "$FARMCONFIGROOT/options/Monitoring.opts" // Manager.Services = {"Dataflow_MBMClient/MBM", "Dataflow_MBMSelector/EventProc", @@ -9,8 +11,9 @@ Manager.Algorithms = {"Dataflow_EmptyProcessor/Empty"}; Manager.Runable = "Wrap"; Wrap.Callable = "EventProc"; // -EventProc.REQ1 = "EvType=2;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0"; -EventProc.REQ2 = "EvType=3;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0"; +EventProc.REQ1 = "EvType=1;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0"; +EventProc.REQ2 = "EvType=2;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0"; +EventProc.REQ3 = "EvType=3;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0"; EventProc.Input = "Events"; // // @@ -34,7 +37,7 @@ Writer.PollTimeout = 100000; // micro-seconds Writer.NumBuffers = 6; Writer.NumThreads = 4; Writer.CancelTimeout = 100; // seconds +Writer.ForceMDF = true; // -Logger.OutputLevel = @OnlineEnv.OutputLevel; Logger.OutputLevel = 0; // diff --git a/Online/Dataflow/src/Storage/StorageWriter.cpp b/Online/Dataflow/src/Storage/StorageWriter.cpp index 337186a2b..33b056d44 100644 --- a/Online/Dataflow/src/Storage/StorageWriter.cpp +++ b/Online/Dataflow/src/Storage/StorageWriter.cpp @@ -20,6 +20,8 @@ #include <Dataflow/Plugins.h> #include <Storage/fdb_client.h> #include <Tell1Data/Tell1Decoder.h> +#include <PCIE40Data/pcie40decoder.h> +#include <PCIE40Data/sodin.h> #include <RTL/rtl.h> /// C/C++ include files @@ -50,6 +52,7 @@ StorageWriter::StorageWriter(const string& nam, Context& ctxt) declareProperty("NumBuffers", m_num_buffers = 2); declareProperty("NumThreads", m_num_threads = 1); declareProperty("MinFileSizeMB", m_minFileSizeMB = 3000); + declareProperty("ForceMDF", m_force_mdf = 0); } /// Default destructor @@ -71,7 +74,7 @@ int StorageWriter::initialize() { m_free.clear(); for(size_t i=0; i<m_num_buffers; ++i) { Buffer b; - b.buffer = (unsigned char*)::malloc(m_bufferSize+1024); + b.buffer = (uint8_t*)::malloc(m_bufferSize+1024); b.pointer = b.buffer; m_free.push_back(b); } @@ -170,57 +173,160 @@ void StorageWriter::flush_buffer(Buffer& buff) { buff = m_shutdown ? Buffer() : get_buffer(); } +/// Manage buffer to ensure the next data block will fit +int StorageWriter::ensure_buffer(Buffer& buff, long length) { + if ( nullptr == buff.buffer ) { + info("Execute: Failed to allocate buffer. Drop MEP buffer. [%s]", + m_shutdown ? "Shutdown requested" : "??????????"); + return DF_CONTINUE; + } + if ( (buff.pointer - buff.buffer) > (maxBufferSize() - length) ) { + /// Do not exceed the max buffer size. Flush data and start with clean buffer + flush_buffer(buff); + m_curr_run = 0; + m_curr_orbit = 0; + m_curr_bunch = 0; + } + if ( nullptr == buff.buffer ) { + info("Execute: Failed to allocate buffer. Drop MEP buffer. [%s]", + m_shutdown ? "Shutdown requested" : "??????????"); + return DF_CONTINUE; + } + if ( (buff.pointer - buff.buffer) > (maxBufferSize() - length) ) { + error("Execute: Failed to allocate buffer [%ld bytes]. " + "BUFFER SIZE TOO SMALL! Change options!", length); + return DF_ERROR; + } + return DF_SUCCESS; +} + +/// Append data to current buffer. If too small allocate a new buffer +int StorageWriter::save_buffer(Buffer& buff, const void* data, long length) { + int sc = ensure_buffer(buff, length); + if ( sc == DF_SUCCESS ) { + ::memcpy(buff.pointer, data, length); + buff.pointer += length; + } + return sc; +} + +/// Convert PCIE40 MEP to MDF and save it. +int StorageWriter::save_pcie40_as_mdf(Buffer& buff, const uint8_t* start, long len) { + auto* mep_start = (pcie40::mep_header_t*)start; + auto* mep_end = pcie40::add_ptr<pcie40::mep_header_t>(start, len); + std::unique_ptr<pcie40::event_collection_t> ev; + pcie40::decoder_t decoder; + long nev = 0; + + for(const pcie40::mep_header_t *m, *mep = mep_start; mep < mep_end; ) { + if ( mep->is_valid() ) { + const pcie40::multi_fragment_t *mfp = mep->multi_fragment(0); + uint16_t packing = mfp->header.packing; + nev += packing; + decoder.decode(ev, mep); + /// Save the events one by one to the buffer + for( auto* e=ev->begin(); e != ev->end(); e=ev->next(e) ) { + size_t length = e->total_length(); + int sc = ensure_buffer(buff, length); + if ( sc == DF_SUCCESS ) { + if ( 0 == m_curr_run ) { + const auto* odin = e->bank_collection(0)->at(0); + if ( odin && odin->data() ) { + const auto* sodin = odin->begin<pcie40::sodin_t>(); + m_curr_run = sodin->run_number(); + m_curr_orbit = sodin->orbit_id(); + m_curr_bunch = sodin->bunch_id(); + } + } + for( size_t i=0, n=e->num_bank_collections(); i<n; ++i) { + buff.pointer = (uint8_t*)e->bank_collection(i)->copy_data(buff.pointer); + } + } + } + /// Move to the next MEP if any + if ( m=mep->next(); m > mep ) { + mep = m; + continue; + } + } + break; + } + return nev > 0 ? DF_SUCCESS : DF_ERROR; +} + +/// Save MDF frame or MDF burst +int StorageWriter::save_mdf_buffer(Buffer& buff, const uint8_t* start, long len) { + + /// Auto detect data type: now check for MDF data type + for( const uint8_t* end = start + len; start < end; ) { + auto* header = (EventHeader*)start; + auto* hdr = header->subHeader().H1; + long length = header->size0(); + long run = hdr->runNumber(); + if ( start+length > end ) { + break; + } + else if ( 0 == m_curr_run ) { + /// First event in this buffer: store the current run number + m_curr_run = run; + m_curr_orbit = hdr->orbitNumber(); + m_curr_bunch = hdr->bunchID(); + } + else if ( run != m_curr_run ) { + /// Run number change. Flush data and start with clean buffer + flush_buffer(buff); + m_curr_run = run; + m_curr_orbit = hdr->orbitNumber(); + m_curr_bunch = hdr->bunchID(); + } + else if ( (buff.pointer - buff.buffer) > (maxBufferSize() - length) ) { + /// Do not exceed the max buffer size. Flush data and start with clean buffer + flush_buffer(buff); + m_curr_run = run; + m_curr_orbit = hdr->orbitNumber(); + m_curr_bunch = hdr->bunchID(); + } + int sc = save_buffer(buff, header, length); + if ( DF_SUCCESS != sc ) { + return DF_SUCCESS; + } + start += length; + } + return DF_SUCCESS; +} + /// Data processing overload: Write the data record to disk int StorageWriter::execute(const Context::EventData& event) { - int sc = DF_SUCCESS; + int status = DF_SUCCESS; try { - size_t len = event.length; - auto* start = (unsigned char*)event.data; - auto& buff = get_buffer(); + size_t len = event.length; + auto* start = (uint8_t*)event.data; + auto& buff = get_buffer(); + /// Check buffer validity if ( nullptr == buff.buffer ) { info("Execute: Failed to allocate buffer. Drop event. [%s]", m_shutdown ? "Shutdown requested" : "??????????"); - return DF_SUCCESS; + return status; } - for( unsigned char* end = start + len; start < end; ) { - auto* header = (EventHeader*)start; - auto* hdr = header->subHeader().H1; - long length = header->size0(); - long run = hdr->runNumber(); - if ( start+length > end ) { - break; - } - else if ( 0 == m_curr_run ) { - /// First event in this buffer: store the current run number - m_curr_run = run; - m_curr_orbit = hdr->orbitNumber(); - m_curr_bunch = hdr->bunchID(); - } - else if ( run != m_curr_run ) { - /// Run number change. Flush data and start with clean buffer - flush_buffer(buff); - m_curr_run = run; - m_curr_orbit = hdr->orbitNumber(); - m_curr_bunch = hdr->bunchID(); - } - else if ( (buff.pointer - buff.buffer) > (maxBufferSize() -length) ) { - /// Do not exceed the max buffer size. Flush data and start with clean buffer - flush_buffer(buff); - m_curr_run = run; - m_curr_orbit = hdr->orbitNumber(); - m_curr_bunch = hdr->bunchID(); - } - if ( nullptr == buff.buffer ) { - info("Execute: Failed to allocate buffer. Drop event. [%s]", - m_shutdown ? "Shutdown requested" : "??????????"); - return DF_SUCCESS; - } - ::memcpy(buff.pointer, header, length); - buff.pointer += length; - start += length; + /// Auto detect data type: first check for PCIE40 MEP format + auto* mep_hdr = (pcie40::mep_header_t*)start; + if( mep_hdr->magic == mep_hdr->MagicPattern ) { + status = m_force_mdf + ? save_pcie40_as_mdf(buff, start, mep_hdr->size*sizeof(uint32_t)) + : save_buffer (buff, start, mep_hdr->size*sizeof(uint32_t)); + return status; + } + /// Auto detect data type: first check for MDF/BURST format + auto* mdf_hdr = (EventHeader*)start; + if ( mdf_hdr->size0() == mdf_hdr->size1() && + mdf_hdr->size0() == mdf_hdr->size2() ) { + status = save_mdf_buffer(buff, start, len); + return status; } - return DF_SUCCESS; + + error("Execute: Cannot determine event type. Drop event buffer."); + return status; } catch(const exception& e) { error(e,"Execute: Error processing event."); @@ -228,7 +334,7 @@ int StorageWriter::execute(const Context::EventData& event) { catch(...) { error("Execute: UNKOWN error processing event."); } - return sc; + return status; } /// Thread invocation routine to save assembled buffers to the disk server @@ -266,6 +372,17 @@ void StorageWriter::process_buffers() { } } +/// Print server's HttpReply structure +void StorageWriter::print_reply(const char* prefix, const http::HttpReply& reply) const { + string line; + stringstream str; + reply.print(str,""); + do { + getline(str, line); + error("writeBuffer-reply<%s>: %s", prefix, line.c_str()); + } while( str.good() && !str.eof() ); +} + /// Construct file name string StorageWriter::makeFileName() const { char fname[1024]; @@ -295,17 +412,12 @@ string StorageWriter::makeFileName() const { /// Write multi event buffer to file. Eventually open a new file.... int StorageWriter::writeBuffer(const Buffer& buff) { -#if 0 - buff.pointer = m_buffer; - m_curr_run = 0; - return DF_SUCCESS; -#else string fname = makeFileName(); - - /// Implement here a retry mechanism starting with registering the DB record int num_retries = m_write_error_retry; size_t len = buff.pointer-buff.buffer; bool process = (m_cancelled == 0) || (::time(0) - m_cancelled < m_cancel_tmo); + + /// Implement here a retry mechanism starting with registering the DB record while ( process ) { try { string url; @@ -314,17 +426,8 @@ int StorageWriter::writeBuffer(const Buffer& buff) { cl.fdbclient = storage::client::create<storage::client::sync>(srv.host, srv.port, 10000); cl.create_client = std::function(storage::client::create<storage::client::sync>); auto reply = cl.save_object_record(fname, url, len); - if ( reply.status != reply.temp_redirect ) { - string line; - stringstream str; - reply.print(str,""); - do { - getline(str, line); - error("writeBuffer: Server reply: %s", line.c_str()); - } while( str.good() && !str.eof() ); - } - else { - /// OK. this was now successful. Send the data. + if ( reply.status == reply.temp_redirect ) { + /// OK. the registration was now successful. Send the data. /// Implement here a retry mechanism sending the data. process = (m_cancelled == 0) || (::time(0) - m_cancelled < m_cancel_tmo); while ( process ) { @@ -339,13 +442,7 @@ int StorageWriter::writeBuffer(const Buffer& buff) { return DF_SUCCESS; } else { - string line; - stringstream str; - reply.print(str,""); - do { - getline(str, line); - error("writeBuffer: Server reply: %s", line.c_str()); - } while( str.good() && !str.eof() ); + print_reply("file-server", reply); } } catch(const std::exception& e) { @@ -354,12 +451,16 @@ int StorageWriter::writeBuffer(const Buffer& buff) { catch(...) { error("writeBuffer: Exception while sending data: UNKNOWN Exception"); } - if ( --num_retries < 0 ) break; ::lib_rtl_sleep(m_write_error_sleep); - process = (m_cancelled == 0) || (::time(0) - m_cancelled < m_cancel_tmo); + process = (m_cancelled == 0); + process |= (::time(0) - m_cancelled < m_cancel_tmo); + process &= (--num_retries < 0); } break; } + else { + print_reply("dbase-server", reply); + } } catch(const std::exception& e) { error("writeBuffer: Exception while connecting: %s",e.what()); @@ -367,11 +468,11 @@ int StorageWriter::writeBuffer(const Buffer& buff) { catch(...) { error("writeBuffer: Exception while connecting: UNKNOWN Exception"); } - if ( --num_retries < 0 ) break; ::lib_rtl_sleep(m_write_error_sleep); - process = (m_cancelled == 0) || (::time(0) - m_cancelled < m_cancel_tmo); + process = (m_cancelled == 0); + process |= (::time(0) - m_cancelled < m_cancel_tmo); + process &= (--num_retries < 0); } return DF_ERROR; -#endif } diff --git a/Online/Dataflow/src/Storage/StorageWriter.h b/Online/Dataflow/src/Storage/StorageWriter.h index b1e8b284f..bffe85a36 100644 --- a/Online/Dataflow/src/Storage/StorageWriter.h +++ b/Online/Dataflow/src/Storage/StorageWriter.h @@ -23,6 +23,7 @@ /// C/C++ include files +#include <cstdint> #include <memory> #include <thread> #include <mutex> @@ -48,9 +49,9 @@ namespace Online { typedef std::unique_ptr<std::thread> thread_t; struct Buffer { /// Pointer to the start of the event buffer - unsigned char* buffer = nullptr; + uint8_t* buffer = nullptr; /// Current pointer inside the event buffer - unsigned char* pointer = nullptr; + uint8_t* pointer = nullptr; /// Default constructor Buffer() = default; }; @@ -75,6 +76,8 @@ namespace Online { int m_poll_tmo {100}; /// Property: Cancel timeout to empty pending buffers int m_cancel_tmo {100}; + /// Property: Force output in MDF format + int m_force_mdf {0}; /// Buffer handling thread std::vector<thread_t> m_threads; @@ -106,10 +109,21 @@ namespace Online { std::string makeFileName() const; /// Get current buffer. If empty check for new one... Buffer& get_buffer(); + /// Manage buffer to ensure the next data block will fit + int ensure_buffer(Buffer& buff, long length); + /// Append data to current buffer. If too small allocate a new buffer + int save_buffer(Buffer& buffer, const void* data, long length); + /// Convert PCIE40 MEP to MDF and save it. + int save_pcie40_as_mdf(Buffer& buffer, const uint8_t* start, long length); + /// Save MDF frame or MDF burst + int save_mdf_buffer(Buffer& buffer, const uint8_t* start, long length); + /// Maximum available buffer size long maxBufferSize() const { return m_bufferSize; } /// Write multi event buffer to file. Eventually open a new file.... int writeBuffer(const Buffer& buffer); + /// Print server's HttpReply structure + void print_reply(const char* prefix, const http::HttpReply& reply) const; public: /// Initializing constructor diff --git a/Online/Dataflow/src/components/MBMSelector.cpp b/Online/Dataflow/src/components/MBMSelector.cpp index f71ec3352..eda6d2e1b 100755 --- a/Online/Dataflow/src/components/MBMSelector.cpp +++ b/Online/Dataflow/src/components/MBMSelector.cpp @@ -205,7 +205,15 @@ int MBMSelector::start() { /// Initialize the event processor int MBMSelector::stop() { - detail::deletePtr(m_consumer); + if ( m_consumer ) { + try { + for( int i=0; i<m_nreqs; ++i ) + m_consumer->delRequest(m_Reqs[i]); + } + catch(...) { } + delete m_consumer; + m_consumer = nullptr; + } m_nreqs = 0; return EventRunable::stop(); } diff --git a/Online/Dataflow/src/components/MEPSelector.cpp b/Online/Dataflow/src/components/MEPSelector.cpp index e2eb57667..89dd32b2a 100755 --- a/Online/Dataflow/src/components/MEPSelector.cpp +++ b/Online/Dataflow/src/components/MEPSelector.cpp @@ -533,7 +533,15 @@ int MEPSelector::start() { /// Initialize the event processor int MEPSelector::stop() { - detail::deletePtr(m_consumer); + if ( m_consumer ) { + try { + for(int i=0; i<m_nreqs; ++i) + m_consumer->delRequest(m_Reqs[i]); + } + catch(...) { } + delete m_consumer; + m_consumer = nullptr; + } if ( m_buffer ) { ::free(m_buffer); m_buffer = 0; diff --git a/Online/Dataflow/src/components/UpgradeMEPReader.cpp b/Online/Dataflow/src/components/UpgradeMEPReader.cpp index 6a3a068d2..e6f1aad77 100644 --- a/Online/Dataflow/src/components/UpgradeMEPReader.cpp +++ b/Online/Dataflow/src/components/UpgradeMEPReader.cpp @@ -38,12 +38,16 @@ namespace Online { */ class UpgradeMEPReader : public DiskReader { public: + /// Header prefix length in MBM file + size_t m_headerPrefixSize = 0; + /// Runable implementation : Run the class implementation virtual int i_run() override; public: /// Standard Constructor - using DiskReader::DiskReader; + UpgradeMEPReader(const std::string& nam, DataflowContext& ctxt); + /// Standard Destructor virtual ~UpgradeMEPReader() = default; }; @@ -80,6 +84,13 @@ DECLARE_DATAFLOW_NAMED_COMPONENT_NS(Online,Dataflow_UpgradeMEPReader,UpgradeMEPR using namespace std; +/// Standard Constructor +UpgradeMEPReader::UpgradeMEPReader(const std::string& nam, DataflowContext& ctxt) + : DiskReader(nam, ctxt) +{ + declareProperty("HeaderPrefixSize", m_headerPrefixSize = 0); +} + /// IRunable implementation : Run the class implementation int Online::UpgradeMEPReader::i_run() { DiskIO io(*this); @@ -90,12 +101,13 @@ int Online::UpgradeMEPReader::i_run() { int partid = context.mbm->partitionID(); bool files_processed = false; MBM::BufferInfo* mbmInfo = m_mbmInfo[m_buffer]; - + size_t file_length = 0; + m_eventsIN = 0; m_goto_paused = true; - m_deleteFiles = false; - m_saveRest = false; - m_mmapFiles = true; + //m_deleteFiles = false; + //m_saveRest = false; + //m_mmapFiles = true; m_receiveEvts = true; status = waitForGo(); @@ -167,40 +179,53 @@ int Online::UpgradeMEPReader::i_run() { const char* ptr = current.name().c_str()+idx; ::sscanf(ptr, "%07d", &runno); updateRunNumber(runno); + file_length = current.data_size(); } } if ( current.isOpen() ) { MBM::EventDesc& dsc = m_producer->event(); MBMAllocator allocator(this); off_t file_position = current.position(); - uint16_t packing = 0; - int data_len = 0, event_type; pcie40::mep_header_t mep_hdr, *pmep_hdr; + int data_len = 0, event_type; + uint16_t packing = 0; + long frame_len = 0; // Check if data are still availible - if ( current.pointer()+sizeof(pcie40::mep_header_t) > current.end() ) { - current.reset(false); + if ( m_mmapFiles && current.pointer()+sizeof(pcie40::mep_header_t) > current.end() ) { + current.reset(!m_mmapFiles); continue; } - - current.read(&mep_hdr, sizeof(mep_hdr)); - data_len = mep_hdr.size; + if ( m_headerPrefixSize > 0 ) { + current.read(&frame_len, sizeof(long)); + } + if ( sizeof(mep_hdr) != current.read(&mep_hdr, sizeof(mep_hdr)) ) { + current.reset(!m_mmapFiles); + continue; + } + data_len = mep_hdr.size*sizeof(uint32_t); event_type = EVENT_TYPE_MEP; if ( data_len <= 0 ) { - current.reset(false); + current.reset(!m_mmapFiles); mep_number = 0; continue; } - else if ( current.pointer()-sizeof(mep_hdr)+data_len > current.end() ) { - current.reset(false); + else if ( data_len < 128 ) { + current.reset(!m_mmapFiles); mep_number = 0; continue; } - else if ( data_len < 128 ) { - current.reset(false); + else if ( m_mmapFiles && current.pointer()-sizeof(mep_hdr)+data_len > current.end() ) { + current.reset(!m_mmapFiles); mep_number = 0; continue; } - unsigned char* data = allocator(data_len); + else if ( file_length - file_position < data_len - sizeof(mep_hdr) ) { + current.reset(!m_mmapFiles); + mep_number = 0; + continue; + } + unsigned char* data = allocator(data_len); + long rd_len = data_len-sizeof(mep_hdr); if ( data == 0 || dsc.len == 0 ) { // MBM Error // Set back the file position to the beginning of the event and continue. // If there was a cancel in between, the file shall be saved. @@ -208,8 +233,13 @@ int Online::UpgradeMEPReader::i_run() { continue; } dsc.len = data_len; - ::memcpy(data,&mep_hdr,sizeof(mep_hdr)); - current.read(data+sizeof(mep_hdr), data_len-sizeof(mep_hdr)); + ::memcpy(data, &mep_hdr, sizeof(mep_hdr)); + if ( rd_len != current.read(data+sizeof(mep_hdr), rd_len) ) { + current.reset(!m_mmapFiles); + continue; + } + //current.position(file_position + frame_len); + pmep_hdr = (pcie40::mep_header_t*)data; packing = pmep_hdr->multi_fragment(0)->header.packing; ++mep_number; diff --git a/Online/Dataflow/src/components/UpgradeMEPSelector.cpp b/Online/Dataflow/src/components/UpgradeMEPSelector.cpp index 196a8b71c..4e19ec474 100644 --- a/Online/Dataflow/src/components/UpgradeMEPSelector.cpp +++ b/Online/Dataflow/src/components/UpgradeMEPSelector.cpp @@ -228,7 +228,15 @@ int UpgradeMEPSelector::start() { /// Initialize the event processor int UpgradeMEPSelector::stop() { - detail::deletePtr(m_consumer); + if ( m_consumer ) { + try { + for( int i=0; i<m_nreqs; ++i ) + m_consumer->delRequest(m_Reqs[i]); + } + catch(...) { } + delete m_consumer; + m_consumer = nullptr; + } if ( m_buffer ) { ::free(m_buffer); m_buffer = 0; diff --git a/Online/Dataflow/src/framework/DataflowManager.cpp b/Online/Dataflow/src/framework/DataflowManager.cpp index 57e7b9748..16b24efdd 100755 --- a/Online/Dataflow/src/framework/DataflowManager.cpp +++ b/Online/Dataflow/src/framework/DataflowManager.cpp @@ -23,6 +23,7 @@ #include "Dataflow/Incidents.h" #include "Dataflow/Plugins.h" #include "Tell1Data/Tell1Decoder.h" +#include "MBM/bmdef.h" // C/C++ include files #include <functional> @@ -442,6 +443,10 @@ int DataflowManager::execute(const DataflowContext::EventData& event) { setRunNumber(mdf->subHeader().H1->runNumber()); m_numBadEvent = 0; } + else if ( event.type == EVENT_TYPE_MEP ) { + setRunNumber(0); + m_numBadEvent = 0; + } else { // We got a MEP event. Should never happen! if ( ++m_numBadEvent <= m_maxBadEvent ) { diff --git a/Online/Dataflow/src/framework/DiskReader.cpp b/Online/Dataflow/src/framework/DiskReader.cpp index 0e28b017e..cd12af3ad 100644 --- a/Online/Dataflow/src/framework/DiskReader.cpp +++ b/Online/Dataflow/src/framework/DiskReader.cpp @@ -61,6 +61,7 @@ unsigned char* DiskReader::MBMAllocator::operator()(size_t length) { catch (const exception& e) { reader->error("Exception while reading input files (spaceRearm): %s BuffSize:%ld. ", e.what(), length); + ::lib_rtl_sleep(1000); } if ( status != MBM_NORMAL ) { return 0; @@ -94,29 +95,29 @@ DiskReader::DiskReader(const string& nam, DataflowContext& ctxt) m_producer(0), m_eventsIN(0), m_evtCountFile(0), m_runSvcID(0), m_currentRun(0), m_goSvcID(0), m_goValue(0), m_disabled(false), m_goto_paused(false), m_mbmInfos(0) { - declareProperty("InputType", m_inputType = AUTO_INPUT_TYPE ); - declareProperty("Buffer", m_buffer = "Mep"); + declareProperty("InputType", m_inputType = AUTO_INPUT_TYPE ); + declareProperty("Buffer", m_buffer = "Mep"); declareProperty("Directories", m_dirList ); - declareProperty("FilePrefix", m_filePrefix = "Run_"); - declareProperty("BrokenHosts", m_brokenHostsFile = ""); - declareProperty("DeleteFiles", m_deleteFiles = true); - declareProperty("SaveRest", m_saveRest = true); - declareProperty("ConsumerWait", m_maxConsWait = 20); - declareProperty("MaxPauseWait", m_maxPauseWait = 1000); + declareProperty("FilePrefix", m_filePrefix = "Run_"); + declareProperty("BrokenHosts", m_brokenHostsFile = ""); + declareProperty("DeleteFiles", m_deleteFiles = true); + declareProperty("SaveRest", m_saveRest = true); + declareProperty("ConsumerWait", m_maxConsWait = 20); + declareProperty("MaxPauseWait", m_maxPauseWait = 1000); declareProperty("AllowedRuns", m_allowedRuns); - declareProperty("InitialSleep", m_initialSleep = 10); - declareProperty("PauseSleep", m_pauseSleep = 5); - declareProperty("Rescan", m_rescan = 1); - declareProperty("GoService", m_goService = ""); - declareProperty("PackingFactor", m_packingFactor = 1); - declareProperty("AllocationSizekB", m_preAllocSize = 2048); + declareProperty("InitialSleep", m_initialSleep = 10); + declareProperty("PauseSleep", m_pauseSleep = 5); + declareProperty("Rescan", m_rescan = 1); + declareProperty("GoService", m_goService = ""); + declareProperty("PackingFactor", m_packingFactor = 1); + declareProperty("AllocationSizekB", m_preAllocSize = 2048); declareProperty("EventsPerFile", m_max_events_per_file = -1); declareProperty("CheckedBuffers", m_mbmNames); - declareProperty("OnOpenFailedDelete", m_openFailDelete = 0); + declareProperty("OnOpenFailedDelete", m_openFailDelete = 0); declareProperty("RequireConsumers", m_requireConsumers = true); - declareProperty("MuDelay", m_muDelay = 0); - declareProperty("MMapFiles", m_mmapFiles = 0); - declareProperty("ReuseFile", m_reuse_file = false); + declareProperty("MuDelay", m_muDelay = 0); + declareProperty("MMapFiles", m_mmapFiles = 0); + declareProperty("ReuseFile", m_reuse_file = false); m_allowedRuns.push_back("*"); ::lib_rtl_create_lock(0, &m_lock); } diff --git a/Online/Dataflow/src/framework/Receiver.cpp b/Online/Dataflow/src/framework/Receiver.cpp index a507faf93..42d0d644f 100755 --- a/Online/Dataflow/src/framework/Receiver.cpp +++ b/Online/Dataflow/src/framework/Receiver.cpp @@ -21,7 +21,6 @@ #include "Dataflow/Incidents.h" #include "Tell1Data/Tell1Decoder.h" #include "DD4hep/Primitives.h" -#include "MBM/Producer.h" #include "RTL/Lock.h" #include "WT/wtdef.h" #include "RTL/rtl.h" @@ -47,11 +46,11 @@ Receiver::Receiver(const string& nam, Context& ctxt) : Component(nam, ctxt) { ::wtc_init(); declareProperty("Buffer", m_buffer); - declareProperty("UseEventRequests", m_useRequests=false); - declareProperty("DeclareAsynchonously",m_declareAsynch=false); - declareProperty("RoutingMask", m_routingMask=0); - declareProperty("VetoMask", m_vetoMask=0); - declareProperty("ErrorDelay", m_errorDelay=100); + declareProperty("UseEventRequests", m_useRequests = false); + declareProperty("DeclareAsynchonously",m_declareAsynch = false); + declareProperty("RoutingMask", m_routingMask = 0); + declareProperty("VetoMask", m_vetoMask = 0); + declareProperty("ErrorDelay", m_errorDelay = 100); } /// Initialize the MBM server @@ -95,13 +94,13 @@ int Receiver::start() { /// Stop the data flow component. Default implementation is empty. int Receiver::stop() { m_recvEvents = false; - if ( m_prod ) m_prod->clear(); return Component::stop(); } /// Finalize the MBM server int Receiver::finalize() { m_recvEvents = false; + if ( m_prod ) m_prod->clear(); unsubscribeIncidents(); ::wtc_flush(WT_FACILITY_DAQ_EVENT); ::wtc_remove(WT_FACILITY_DAQ_EVENT); @@ -261,15 +260,32 @@ int Receiver::declareData(RecvEntry& entry) { int ret = m_recvEvents ? m_prod->getSpace(entry.size) : MBM_REQ_CANCEL; if ( MBM_NORMAL == ret ) { MBM::EventDesc& e = m_prod->event(); - EventHeader* h = (EventHeader*)e.data; - auto m = h->subHeader().H1->triggerMask(); - e.type = EVENT_TYPE_EVENT; - e.len = entry.size; + e.len = entry.size; + e.type = EVENT_TYPE_EVENT; ret = copyEventData(e.data, entry.buffer, entry.size); if ( DF_SUCCESS == ret ) { - e.mask[0] = m[0]; - e.mask[1] = m[1]; - e.mask[2] = m[2]; + EventHeader* h = (EventHeader*)e.data; + e.mask[3] = ~0x0; + if ( h->size0() == h->size1() && entry.size > h->size0() ) { + auto m = h->subHeader().H1->triggerMask(); + e.mask[0] = m[0]; + e.mask[1] = m[1]; + e.mask[2] = m[2]; + e.type = EVENT_TYPE_BURST; + } + else if ( h->size0() == h->size1() && entry.size == h->size0() ) { + auto m = h->subHeader().H1->triggerMask(); + e.mask[0] = m[0]; + e.mask[1] = m[1]; + e.mask[2] = m[2]; + e.type = EVENT_TYPE_EVENT; + } + else if ( *(unsigned short*)e.data == 0xcefa ) { + e.mask[0] = ~0x0; + e.mask[1] = ~0x0; + e.mask[2] = ~0x0; + e.type = EVENT_TYPE_MEP; + } if ( 0 != m_routingMask ) { e.mask[3] &= ~m_vetoMask; e.mask[3] |= m_routingMask; diff --git a/Online/GaudiOnline/GaudiOnline/EventAccess.h b/Online/GaudiOnline/GaudiOnline/EventAccess.h index a69a6fc07..999d8d278 100644 --- a/Online/GaudiOnline/GaudiOnline/EventAccess.h +++ b/Online/GaudiOnline/GaudiOnline/EventAccess.h @@ -79,9 +79,9 @@ namespace Online { /// Default destructor virtual ~guard_t() = default; /// Careful: unprotected information access - bool empty() const { return consumed == length; } - std::size_t size() const { return length; } - long type() const { return typ; } + bool empty() const { return consumed == length; } + std::size_t size() const { return length; } + long type() const { return typ; } virtual record_t at(size_t which) const = 0; }; @@ -199,9 +199,15 @@ namespace Online { /// Dequeue burst if one is present with lock protection shared_guard_t dequeueBurst(); + /// Access thread safe number of queued bursts + size_t numQueuedBurst(); + /// Dequeue burst if present, otherwise wait for producer to emplace one. shared_guard_t waitBurst(); + /// Clear burst queue and update counters + size_t clear(); + /// Dequeue event if one is present with lock protection event_t dequeueEvent(shared_guard_t& context); diff --git a/Online/GaudiOnline/GaudiOnline/EventTraits.h b/Online/GaudiOnline/GaudiOnline/EventTraits.h index b21adbe15..86f5fe93d 100644 --- a/Online/GaudiOnline/GaudiOnline/EventTraits.h +++ b/Online/GaudiOnline/GaudiOnline/EventTraits.h @@ -42,7 +42,8 @@ namespace Online { enum BurstType { NO_DATA_TYPE = 0, TELL1_DATA_TYPE = 1, - TELL40_DATA_TYPE = 2 + TELL40_DATA_TYPE = 2, + TELL1_BURST_TYPE = 3 }; typedef unsigned char data_t; @@ -77,7 +78,8 @@ namespace Online { typedef EventHeader event_type; typedef record_t record_type; typedef RunInfo odin_type; - static constexpr int data_type = TELL1_DATA_TYPE; + static constexpr int data_type = TELL1_DATA_TYPE; + static constexpr int burst_type = TELL1_BURST_TYPE; static event_type& get_event(record_t r) { return *(r.tell1_event); } diff --git a/Online/GaudiOnline/components/InputAlg.cpp b/Online/GaudiOnline/components/InputAlg.cpp index 3c7d5e3a4..04ca7fdeb 100644 --- a/Online/GaudiOnline/components/InputAlg.cpp +++ b/Online/GaudiOnline/components/InputAlg.cpp @@ -25,7 +25,7 @@ DECLARE_COMPONENT( Online::InputAlg ) void Online::InputAlg::halt() const { if ( m_enableHalt ) { m_halt = 1; - m_logger->warning("Algorithm halted by input data handling problem."); + m_logger->warning("+++ Algorithm halted by input data handling problem."); while ( m_halt ) ::lib_rtl_sleep(100); } } @@ -74,9 +74,9 @@ StatusCode Online::InputAlg::process(EventContext const& /* ctxt */) const { return StatusCode::SUCCESS; } else if ( m_io->isCancelled() ) { - m_logger->error("Algorithm failed [Invalid event after CANCEL]."); + m_logger->error("+++ Algorithm failed [Invalid event after CANCEL]."); return StatusCode::FAILURE; } - m_logger->error("Algorithm failed [Invalid data]."); + m_logger->error("+++ Algorithm failed [Invalid data]."); return StatusCode::FAILURE; } diff --git a/Online/GaudiOnline/components/OnlineEventApp.cpp b/Online/GaudiOnline/components/OnlineEventApp.cpp index f6a99dcd5..67f495c91 100644 --- a/Online/GaudiOnline/components/OnlineEventApp.cpp +++ b/Online/GaudiOnline/components/OnlineEventApp.cpp @@ -21,13 +21,10 @@ #include <Gaudi/PluginService.h> #include <Gaudi/Interfaces/IQueueingEventProcessor.h> -#include <GaudiKernel/IMessageSvc.h> -#include <GaudiKernel/IAppMgrUI.h> #include <GaudiKernel/IProperty.h> #include <GaudiKernel/ISvcLocator.h> #include <GaudiKernel/AppReturnCode.h> #include <Gaudi/Property.h> -#include <GaudiKernel/SmartIF.h> #include <CPP/Event.h> #include <RTL/strdef.h> #include <RTL/rtl.h> @@ -35,9 +32,6 @@ // tbb #include "tbb/task_arena.h" -#if defined(USE_OLD_TBB_INTERFACE) -#include "tbb/task_scheduler_init.h" -#endif /// C/C++ include files #include <iostream> @@ -47,6 +41,15 @@ using namespace std; using namespace Online; +namespace { + template <typename T> void stop_thread(T& t) { + if ( t ) { + t->join(); + t.reset(); + } + } +} + /// Easier command line: only supply --application=OnlineEvents class OnlineEvents : public Online::OnlineEventApp { public: @@ -314,28 +317,20 @@ OnlineEventApp::access_t OnlineEventApp::start_file_access() { /// Pause the application (RUNNING -> READY) int OnlineEventApp::pauseProcessing() { - m_logger->debug("Pause the application."); + m_logger->debug("+++ Pause the application."); m_halt = EVENTLOOP_PAUSE; return OnlineApplication::pauseProcessing(); } /// Continue the application (PAUSED -> RUNNING ) int OnlineEventApp::continueProcessing() { - m_logger->debug("Resume application processing."); + m_logger->debug("+++ Resume application processing."); m_halt = EVENTLOOP_EXECUTE; return OnlineApplication::continueProcessing(); } -template <typename T> static void stop_thread(T& t) { - if ( t ) { - t->join(); - t.reset(); - } -} - /// Stop the application (RUNNING -> READY) int OnlineEventApp::stop() { - m_events->queueCancel(true); if ( m_events ) { m_events->cancel(); } @@ -363,12 +358,7 @@ int OnlineEventApp::stop() { void OnlineEventApp::async_queued_event_loop() { size_t num_threads = m_parallel_threads; -#if defined(USE_OLD_TBB_INTERFACE) - tbb::task_scheduler_init tbb_scheduler( num_threads + 1); -#else - //tbb::task_scheduler_init tbb_scheduler_default; std::unique_ptr<tbb::task_arena> tbb_arena = std::make_unique<tbb::task_arena>(num_threads + 1, 1); -#endif bool tbb_inited = true; EventAccess::shared_guard_t loop_context; SmartIF<IQueued> ev_que{app}; @@ -383,22 +373,16 @@ void OnlineEventApp::async_queued_event_loop() { loop_context = m_events->waitBurst(); if ( loop_context ) { m_monitor.burstsOut = m_events->monitor.burstsOut; - size_t num_event = m_ioSvc->push(move(loop_context)); - while ( num_event > 0 ) { + for(size_t nevt=m_ioSvc->push(move(loop_context)); nevt > 0; --nevt) { EventContext ctx(ev_que->createEventContext()); ev_que->push(move(ctx)); - m_monitor.eventsBuffered = m_events->eventsBuffered()+num_event; + m_monitor.eventsBuffered = m_events->eventsBuffered()+nevt; ++m_monitor.eventsSubmitted; - --num_event; } } - else if ( m_events->isCancelled() ) { - m_logger->info("+++ async_loop: Event requests were cancelled."); - break; - } - if ( num_threads == m_parallel_threads && !m_events->isCancelled() ) + else if ( num_threads == m_parallel_threads && !m_events->isCancelled() ) { continue; - //m_logger->info("+++ async_loop: sleep 10"); + } this_thread::sleep_for(chrono::milliseconds(10)); } catch(const exception& e) { @@ -409,7 +393,6 @@ void OnlineEventApp::async_queued_event_loop() { } } else { - //m_logger->info("+++ async_loop: sleep 1000"); this_thread::sleep_for(chrono::milliseconds(1000)); } /// Check if the scenario changed and we have to adapt the number of threads executing @@ -419,42 +402,44 @@ void OnlineEventApp::async_queued_event_loop() { this_thread::sleep_for(chrono::milliseconds(10)); /// Now stop the scheduler if ( tbb_inited ) { -#if defined(USE_OLD_TBB_INTERFACE) - tbb_scheduler.terminate(); // non blocking -#else tbb_arena->terminate(); // non blocking tbb_arena.reset(); -#endif tbb_inited = false; } num_threads = m_parallel_threads; if ( num_threads > 0 ) { m_logger->info("+++ async_loop: Restart event processing with %ld threads.",num_threads); /// Finally restart the scheduler -#if defined(USE_OLD_TBB_INTERFACE) - tbb_scheduler.initialize(num_threads+1); -#else tbb_arena.reset(new tbb::task_arena(num_threads + 1, 1)); -#endif tbb_inited = true; } } - } while ( (m_halt != EVENTLOOP_STOP) || !(ev_que->empty() && m_events->empty()) ); + if ( m_events->eventsBuffered() > 0 ) { + continue; + } + this_thread::sleep_for(chrono::milliseconds(1000)); + // Check if we got some late frame(s) + if ( m_events->eventsBuffered() > 0 ) { + continue; + } + if ( m_events->isCancelled() ) { + m_logger->info("+++ async_loop: Event requests were cancelled."); + break; + } + } while ( !(m_halt == EVENTLOOP_STOP) || + !m_events->empty() || + !ev_que->empty() ); while ( !ev_que->empty() ) // this is our "threads.join()" alternative this_thread::sleep_for( chrono::milliseconds( 100 ) ); -#if defined(USE_OLD_TBB_INTERFACE) - tbb_scheduler.terminate(); // non blocking -#else tbb_arena->terminate(); // non blocking# tbb_arena.reset(); -#endif if ( m_events->isCancelled() ) { loop_context.reset(); - m_logger->info("async_loop: Event loop exiting [CANCELLED]."); + m_logger->info("+++ async_loop: Event loop exiting [CANCELLED]."); } else { - m_logger->info("async_loop: Event loop exiting."); + m_logger->info("+++ async_loop: Event loop exiting."); } autoStop(); } @@ -473,30 +458,28 @@ void OnlineEventApp::sync_queued_event_loop() { loop_context = m_events->waitBurst(); if ( loop_context ) { m_monitor.burstsOut = m_events->monitor.burstsOut; - size_t num_event = m_ioSvc->push(move(loop_context)); - while ( num_event > 0 ) { + for(size_t nevt=m_ioSvc->push(move(loop_context)); nevt > 0; --nevt) { EventContext ctx(ev_que->createEventContext()); ev_que->executeEvent(move(ctx)).ignore(/* AUTOMATICALLY ADDED FOR gaudi/Gaudi!763 */); - m_monitor.eventsBuffered = m_events->eventsBuffered()+num_event; + m_monitor.eventsBuffered = m_events->eventsBuffered()+nevt; ++m_monitor.eventsSubmitted; - --num_event; } continue; } this_thread::sleep_for(chrono::milliseconds(10)); } catch(const exception& e) { - m_logger->error(e, "Event loop failure with exception."); + m_logger->error(e, "+++ Event loop failure with exception."); } catch(...) { m_logger->error("+++ Event loop failure with UNKNOWN exception"); } this_thread::sleep_for(chrono::milliseconds(100)); } while ( (m_halt != EVENTLOOP_STOP) ); - + // if ( m_events->isCancelled() ) { loop_context.reset(); - m_logger->info("Event loop exiting [CANCELLED]."); + m_logger->info("+++ Event loop exiting [CANCELLED]."); } } diff --git a/Online/GaudiOnline/components/OutputAlg.cpp b/Online/GaudiOnline/components/OutputAlg.cpp index 68fa187ee..120e10fc7 100644 --- a/Online/GaudiOnline/components/OutputAlg.cpp +++ b/Online/GaudiOnline/components/OutputAlg.cpp @@ -119,11 +119,11 @@ OutputAlg::OutputAlg(const string& nam, ISvcLocator* loc) void OutputAlg::handle(const Incident& incident) { if ( incident.type() == "DAQ_END_EVENT" ) { if ( m_output.get() ) { - m_logger->info("Flush all pending data."); + m_logger->info("+++ Flush all pending data."); m_output->commit_all(); return; } - m_logger->error("Failed to commit all transactions."); + m_logger->error("+++ Failed to commit all transactions."); } else if ( incident.type() == "DAQ_STOPPED" ) { stop().ignore(/* AUTOMATICALLY ADDED FOR gaudi/Gaudi!763 */); @@ -140,13 +140,13 @@ StatusCode OutputAlg::initialize() { EventProcessor::initialize().ignore(/* AUTOMATICALLY ADDED FOR gaudi/Gaudi!763 */); m_incident = service<IIncidentSvc>("IncidentSvc"); if ( !m_incident ) { - m_logger->error("Failed to access Incident service."); + m_logger->error("+++ Failed to access Incident service."); return StatusCode::FAILURE; } m_incident->addListener(this,"DAQ_END_EVENT"); m_incident->addListener(this,"DAQ_STOPPED"); m_incident->addListener(this,"DAQ_CANCEL"); - m_logger->info("Successfully initialized output algorithm."); + m_logger->info("+++ Successfully initialized output algorithm."); return StatusCode::SUCCESS; } @@ -196,9 +196,11 @@ StatusCode OutputAlg::process(EventContext const& /* ctxt */) const { const LHCb::RawEvent* evt = m_rawEvent.get(); if ( event->second->type() == event_traits::tell1::data_type ) output_mdf<event_traits::tell1>(*evt, m_routingMask.value(), *m_output); + else if ( event->second->type() == event_traits::tell1::burst_type ) + output_mdf<event_traits::tell1>(*evt, m_routingMask.value(), *m_output); else if ( event->second->type() == event_traits::tell40::data_type ) output_mdf<event_traits::tell40>(*evt, m_routingMask.value(), *m_output); else - m_logger->except("Invalid burst data type encountered: %ld",event->second->type()); + m_logger->except("+++ Invalid burst data type encountered: %ld",event->second->type()); return StatusCode::SUCCESS; } diff --git a/Online/GaudiOnline/components/Passthrough.cpp b/Online/GaudiOnline/components/Passthrough.cpp index 8c468e623..32387b86e 100644 --- a/Online/GaudiOnline/components/Passthrough.cpp +++ b/Online/GaudiOnline/components/Passthrough.cpp @@ -135,6 +135,7 @@ StatusCode Passthrough::start() { } StatusCode Passthrough::execute(EventContext const& ctxt) const { + double delay = m_delay*1000+m_muDelay; auto raw = m_rawEvent.get(); if ( !raw ) { MsgStream log(msgSvc(), name()); @@ -143,7 +144,17 @@ StatusCode Passthrough::execute(EventContext const& ctxt) const { return StatusCode::FAILURE; } - double delay = m_delay*1000+m_muDelay; + /// If mothing is specified everything will be downscaled + if ( m_downScale.empty() && m_rate < 1.0 ) { + double frac = double(::rand()) / double(RAND_MAX); + if ( m_rate < frac ) { + execState(ctxt).setFilterPassed(false); + return _halt(delay); + } + execState(ctxt).setFilterPassed(true); + return _halt(delay); + } + const auto odinBanks = raw->banks(LHCb::RawBank::ODIN); if ( odinBanks.empty() ) { execState(ctxt).setFilterPassed(false); @@ -162,13 +173,6 @@ StatusCode Passthrough::execute(EventContext const& ctxt) const { } } } - else if ( m_rate < 1.0 ) { /// If mothing is specified everything will be downscaled - double frac = double(::rand()) / double(RAND_MAX); - if ( m_rate < frac ) { - execState(ctxt).setFilterPassed(false); - return _halt(delay); - } - } // Check trigger type selection (may be ANDed with calibration type selection if( !m_triggerTypesToPass.empty() ) { diff --git a/Online/GaudiOnline/src/EventAccess.cpp b/Online/GaudiOnline/src/EventAccess.cpp index 8cc329182..7ed12c4ef 100644 --- a/Online/GaudiOnline/src/EventAccess.cpp +++ b/Online/GaudiOnline/src/EventAccess.cpp @@ -124,6 +124,23 @@ EventAccess::shared_guard_t EventAccess::dequeueBurst() {{ return shared_guard_t(); } +/// Access thread safe number of queued bursts +size_t EventAccess::numQueuedBurst() { + lock_t lock(m_burstLock); + return m_bursts.size(); +} + +/// Clear burst queue and update counters +size_t EventAccess::clear() { + size_t count = 0; + for(shared_guard_t burst=dequeueBurst(); burst.get(); burst=dequeueBurst()) { + lock_t lock(m_eventLock); + monitor.eventsBuffered -= burst->length; + count += burst->length; + } + return count; +} + /// Dequeue event if present, otherwise wait for producer to emplace one. EventAccess::shared_guard_t EventAccess::waitBurst() { shared_guard_t burst(dequeueBurst()); @@ -234,7 +251,7 @@ EventAccess::convertPCIE40MEP(datapointer_t start, size_t len) const { pcie40::decoder_t decoder; static constexpr size_t max_bank = 20000; static constexpr size_t max_source = 2000; - static constexpr size_t max_packing = 5000; + static constexpr size_t max_packing = 35000; const auto* pmep_hdr = reinterpret_cast<const pcie40::mep_header_t*>(start); if ( 0 == pmep_hdr->size ) { m_logger->error("MEP with invalid size encountered: %d", pmep_hdr->size); diff --git a/Online/GaudiOnline/src/MBMEventAccess.cpp b/Online/GaudiOnline/src/MBMEventAccess.cpp index e021262b6..84ea60b74 100644 --- a/Online/GaudiOnline/src/MBMEventAccess.cpp +++ b/Online/GaudiOnline/src/MBMEventAccess.cpp @@ -22,7 +22,6 @@ using namespace std; using namespace Online; - namespace { template<typename INT> std::string to_hex(INT i) { @@ -90,9 +89,8 @@ string MBMEventAccess::bufferName(const string& nam) const { /// Cancel all pending I/O requests to the buffer manager int MBMEventAccess::cancel() { - int count = 0; + int count = int(m_busyPlugs.size()); m_inhibit = true; - count = int(m_busyPlugs.size()); while( m_busyPlugs.size() > 0 ) { { lock_t lock(m_consumerLock); @@ -103,7 +101,18 @@ int MBMEventAccess::cancel() { lock_t lock(m_consumerLock); if ( m_busyPlugs.empty() ) break; } - ::lib_rtl_sleep(10); + // We try to wait a bit and give the event processing thread some time + // to work down the event queue of possibly pending entries. + // We assume here a maximum time window of 20*100 msec + for( size_t i=0; i<20; ++i ) { + ::lib_rtl_sleep(100); + size_t buffered = eventsBuffered(); + m_logger->info("+++ MBMEventAccess: Current event stack size: %ld events", buffered); + if ( i > 3 && 0 == buffered ) break; + } + if( size_t dropped=clear(); dropped > 0 ) { + m_logger->warning("+++ MBMEventAccess: Dropped %ld events on cancel.", dropped); + } { lock_t lock(m_consumerLock); if ( m_busyPlugs.empty() ) break; diff --git a/Online/GaudiOnline/src/MBMEventOutput.cpp b/Online/GaudiOnline/src/MBMEventOutput.cpp index 0ef67da9c..e50dd2575 100644 --- a/Online/GaudiOnline/src/MBMEventOutput.cpp +++ b/Online/GaudiOnline/src/MBMEventOutput.cpp @@ -122,7 +122,7 @@ bool MBMEventOutput::mbm_transaction_t::sync() { length = 0; events = 0; if ( sc == MBM_REQ_CANCEL ) { - output->m_logger->error("Failed to declare event for MBM buffer %s. " + output->m_logger->error("+++ Failed to declare event for MBM buffer %s. " "Request was cancelled.", mbm_buffer_name(bmid)); return false; @@ -130,7 +130,7 @@ bool MBMEventOutput::mbm_transaction_t::sync() { else if ( sc == MBM_NORMAL ) { sc = ::mbm_send_space(bmid); if ( sc == MBM_REQ_CANCEL ) { - output->m_logger->error("Failed to declare event for MBM buffer %s. " + output->m_logger->error("+++ Failed to declare event for MBM buffer %s. " "Request was cancelled.", mbm_buffer_name(bmid)); return false; @@ -144,7 +144,7 @@ bool MBMEventOutput::mbm_transaction_t::sync() { /// Shutdown the I/O medium bool MBMEventOutput::mbm_transaction_t::close() { if ( bmid != MBM_INV_DESC ) { - output->m_logger->info("Exclude from buffer: %s", mbm_buffer_name(bmid)); + output->m_logger->info("+++ Exclude from buffer: %s", mbm_buffer_name(bmid)); ::mbm_exclude(bmid); bmid = MBM_INV_DESC; return true; @@ -206,7 +206,7 @@ int MBMEventOutput::connect(const string& output, size_t instance) { stringstream nam; string bm_name = bufferName(output); nam << RTL::processName() << '.' << instance; - m_logger->info("MBM: Connecting to MBM Buffer %s.", bm_name.c_str()); + m_logger->info("+++ MBM: Connecting to MBM Buffer %s.", bm_name.c_str()); { lock_t lock(m_transactionLock); BMID bmid = MBM_INV_DESC; @@ -218,7 +218,7 @@ int MBMEventOutput::connect(const string& output, size_t instance) { bmid = ::mbm_connect(tr->bmid, nam.str().c_str(), config.partitionID); } if ( bmid == MBM_INV_DESC ) { - return m_logger->error("MBM: Failed to connect to MBM buffer %s!", + return m_logger->error("+++ MBM: Failed to connect to MBM buffer %s!", bm_name.c_str()); } /// Insert producer into idle plug container @@ -228,7 +228,7 @@ int MBMEventOutput::connect(const string& output, size_t instance) { if ( config.max_consumer_wait ) { m_mbmInfo = make_unique<MBM::BufferInfo>(); m_mbmInfo->attach(bm_name.c_str()); - m_logger->info("Enabled MBM consumer checks. Max Wait time: %d seconds", + m_logger->info("+++ Enabled MBM consumer checks. Max Wait time: %d seconds", config.max_consumer_wait); } return ONLINE_OK; @@ -240,7 +240,7 @@ void MBMEventOutput::checkConsumers(int ev_type, unsigned int pid) { int max_cons_wait = config.max_consumer_wait; while ( --max_cons_wait >= 0 ) { if ( m_mbmInfo->num_consumers_partid_evtype(ev_type, pid) != 0 ) { - m_logger->debug("Waiting for consumers subscribed to event type: %d " + m_logger->debug("+++ Waiting for consumers subscribed to event type: %d " "partition:%08X time left: %d seconds", ev_type, pid, max_cons_wait); ::lib_rtl_sleep(1000); @@ -249,8 +249,8 @@ void MBMEventOutput::checkConsumers(int ev_type, unsigned int pid) { break; } if ( max_cons_wait < 0 ) { - m_logger->except("FAILED: No suitable consumers subscribed to event type: %d partition:%08X", - ev_type, pid, max_cons_wait); + m_logger->except("+++ FAILED: No suitable consumers subscribed to event type:" + " %d partition:%08X", ev_type, pid, max_cons_wait); } } } diff --git a/Online/OnlineBase/src/MBM/mbm_extract.cpp b/Online/OnlineBase/src/MBM/mbm_extract.cpp index 34ad66b46..2007ff943 100644 --- a/Online/OnlineBase/src/MBM/mbm_extract.cpp +++ b/Online/OnlineBase/src/MBM/mbm_extract.cpp @@ -52,8 +52,9 @@ extern "C" int mbm_extract(int argc,char **argv) { ::lib_rtl_output(LIB_RTL_ALWAYS," -t(cp) Use boost::asio TCP connections for communication\n"); ::lib_rtl_output(LIB_RTL_ALWAYS," -ipc Use boost::asio ICP connections for communication\n"); ::lib_rtl_output(LIB_RTL_ALWAYS," -out(put)=<file-name> File name to dump event data \n"); - ::lib_rtl_output(LIB_RTL_ALWAYS," -mep Request event type MEP (default: Event) \n"); - ::lib_rtl_output(LIB_RTL_ALWAYS," -burst Request event type Burst (default: Event) \n"); + ::lib_rtl_output(LIB_RTL_ALWAYS," -event Request event type EVENT \n"); + ::lib_rtl_output(LIB_RTL_ALWAYS," -mep Request event type MEP \n"); + ::lib_rtl_output(LIB_RTL_ALWAYS," -burst Request event type Burst \n"); exit(0); }); std::string name = "consumer", buffer="0", output = "mbm.dat", log; @@ -99,25 +100,35 @@ extern "C" int mbm_extract(int argc,char **argv) { ::exit(errno); } int event_type = EVENT_TYPE_EVENT; - if ( mep ) - event_type = EVENT_TYPE_MEP; - else if ( brst ) - event_type = EVENT_TYPE_BURST; - else if ( evt ) - event_type = EVENT_TYPE_EVENT; if ( one ) { - ::mbm_add_req(bmid,event_type,trmask,vetomask,BM_MASK_ANY,(inc_opt<<8)+BM_REQ_ONE,BM_FREQ_PERC,100.); + if ( mep ) + ::mbm_add_req(bmid,EVENT_TYPE_MEP,trmask,vetomask,BM_MASK_ANY,(inc_opt<<8)+BM_REQ_ONE,BM_FREQ_PERC,100.); + if ( brst ) + ::mbm_add_req(bmid,EVENT_TYPE_BURST,trmask,vetomask,BM_MASK_ANY,(inc_opt<<8)+BM_REQ_ONE,BM_FREQ_PERC,100.); + if ( evt ) + ::mbm_add_req(bmid,EVENT_TYPE_EVENT,trmask,vetomask,BM_MASK_ANY,(inc_opt<<8)+BM_REQ_ONE,BM_FREQ_PERC,100.); ::lib_rtl_output(LIB_RTL_ALWAYS,"+++ Event request type is BM_REQ_ONE [%d].",inc_opt); } else if ( user ) { - ::mbm_add_req(bmid,event_type,trmask,vetomask,BM_MASK_ANY,BM_REQ_USER,BM_FREQ_PERC,100.); + if ( mep ) + ::mbm_add_req(bmid,EVENT_TYPE_MEP,trmask,vetomask,BM_MASK_ANY,BM_REQ_USER,BM_FREQ_PERC,100.); + if ( brst ) + ::mbm_add_req(bmid,EVENT_TYPE_BURST,trmask,vetomask,BM_MASK_ANY,BM_REQ_USER,BM_FREQ_PERC,100.); + if ( evt ) + ::mbm_add_req(bmid,EVENT_TYPE_EVENT,trmask,vetomask,BM_MASK_ANY,BM_REQ_USER,BM_FREQ_PERC,100.); ::lib_rtl_output(LIB_RTL_ALWAYS,"+++ Event request type is BM_REQ_USER."); } else { - ::mbm_add_req(bmid,event_type,trmask,vetomask,BM_MASK_ANY,BM_REQ_ALL,BM_FREQ_PERC,100.); + if ( mep ) + ::mbm_add_req(bmid,EVENT_TYPE_MEP,trmask,vetomask,BM_MASK_ANY,BM_REQ_ALL,BM_FREQ_PERC,100.); + if ( brst ) + ::mbm_add_req(bmid,EVENT_TYPE_BURST,trmask,vetomask,BM_MASK_ANY,BM_REQ_ALL,BM_FREQ_PERC,100.); + if ( evt ) + ::mbm_add_req(bmid,EVENT_TYPE_EVENT,trmask,vetomask,BM_MASK_ANY,BM_REQ_ALL,BM_FREQ_PERC,100.); ::lib_rtl_output(LIB_RTL_ALWAYS,"+++ Event request type is BM_REQ_ALL."); } + long tr_number, len; for(tr_number = 1; tr_number <= count; ++tr_number) { unsigned int mask[4]; diff --git a/Online/PCIE40Data/PCIE40Data/RawBank40.h b/Online/PCIE40Data/PCIE40Data/RawBank40.h index 2a0ac295e..17f0d49f1 100644 --- a/Online/PCIE40Data/PCIE40Data/RawBank40.h +++ b/Online/PCIE40Data/PCIE40Data/RawBank40.h @@ -69,6 +69,7 @@ namespace Online { RawBank40() = default; enum BankType { + Other=1, L0Calo=0, // 0 L0DU, // 1 PrsE, // 2 diff --git a/Online/PCIE40Data/PCIE40Data/pcie40.h b/Online/PCIE40Data/PCIE40Data/pcie40.h index a24b66b05..3cb69695d 100644 --- a/Online/PCIE40Data/PCIE40Data/pcie40.h +++ b/Online/PCIE40Data/PCIE40Data/pcie40.h @@ -19,6 +19,7 @@ // C/C++ include files #include <stdexcept> +#include <cstring> #include "PCIE40Data/Pack.h" @@ -248,6 +249,8 @@ namespace Online { const bank_t* next(const bank_t* e) const; const bank_t* at(size_t which) const; const bank_t* operator[](size_t i) const; + size_t total_length() const; + void* copy_data(void* ptr) const; }; /// A full pcie40 event with multiple bank collection sorted by bank type @@ -287,10 +290,12 @@ namespace Online { event_t& operator=(const event_t& copy) = delete; public: - static std::pair<bool,size_t> collection_offset(unsigned char type); - std::pair<size_t, const bank_t*> get(unsigned char type) const; - size_t num_bank_collections() const; - const bank_collection_t* bank_collection(size_t i) const; + static std::pair<bool,size_t> collection_offset(unsigned char type); + std::pair<size_t, const bank_t*> get(unsigned char type) const; + size_t num_bank_collections() const; + const bank_collection_t* bank_collection(size_t i) const; + size_t total_length() const; + void* copy_data(void* pointer) const; }; /// Collection of mep events as they arrive from the event builder @@ -325,11 +330,12 @@ namespace Online { size_t size() const { return length; } size_t num_events() const { return length; } size_t max_events() const { return capacity; } - const event_t* begin() const; - const event_t* end() const; + const event_t* begin() const; + const event_t* end() const; const event_t* next(const event_t* e) const; const event_t* at(size_t which) const; const event_t* operator[](size_t which) const; + size_t total_length() const; }; /// Constant parameters to access the various records @@ -368,36 +374,40 @@ namespace Online { static constexpr size_t maxTell40Hcal = 8; static constexpr size_t maxTell40Muon = 10; static constexpr size_t maxTell40Rich = 350; + static constexpr size_t maxTell40Other = 100; static constexpr size_t maxTell40 = maxTell40ODIN + maxTell40VP + maxTell40UT - + maxTell40FT + maxTell40Ecal + maxTell40Hcal + maxTell40Muon + maxTell40Rich; - - static constexpr size_t collectionCapacity[] = {maxTell40ODIN, - maxTell40VP, - maxTell40UT, - maxTell40FT, - maxTell40Ecal, - maxTell40Hcal, - maxTell40Muon, - maxTell40Rich}; - - static constexpr size_t collectionSizeODIN = sizeof(bank_collection_t); - static constexpr size_t collectionSizeVP = sizeof(bank_collection_t)+(maxTell40VP-1)*sizeof(bank_t); - static constexpr size_t collectionSizeUT = sizeof(bank_collection_t)+(maxTell40UT-1)*sizeof(bank_t); - static constexpr size_t collectionSizeFT = sizeof(bank_collection_t)+(maxTell40FT-1)*sizeof(bank_t); - static constexpr size_t collectionSizeEcal = sizeof(bank_collection_t)+(maxTell40Ecal-1)*sizeof(bank_t); - static constexpr size_t collectionSizeHcal = sizeof(bank_collection_t)+(maxTell40Hcal-1)*sizeof(bank_t); - static constexpr size_t collectionSizeMuon = sizeof(bank_collection_t)+(maxTell40Muon-1)*sizeof(bank_t); - static constexpr size_t collectionSizeRich = sizeof(bank_collection_t)+(maxTell40Rich-1)*sizeof(bank_t); - - static constexpr size_t collectionOffset = event_t::data_offset; - static constexpr size_t collectionOffsetODIN = collectionOffset; - static constexpr size_t collectionOffsetVP = collectionOffsetODIN + collectionSizeODIN; - static constexpr size_t collectionOffsetUT = collectionOffsetVP + collectionSizeVP; - static constexpr size_t collectionOffsetFT = collectionOffsetUT + collectionSizeUT; - static constexpr size_t collectionOffsetEcal = collectionOffsetFT + collectionSizeFT; - static constexpr size_t collectionOffsetHcal = collectionOffsetEcal + collectionSizeEcal; - static constexpr size_t collectionOffsetMuon = collectionOffsetHcal + collectionSizeHcal; - static constexpr size_t collectionOffsetRich = collectionOffsetMuon + collectionSizeMuon; + + maxTell40FT + maxTell40Ecal + maxTell40Hcal + maxTell40Muon + maxTell40Rich + maxTell40Other; + + static constexpr size_t collectionCapacity[] = {maxTell40ODIN, + maxTell40VP, + maxTell40UT, + maxTell40FT, + maxTell40Ecal, + maxTell40Hcal, + maxTell40Muon, + maxTell40Rich, + maxTell40Other + }; + static constexpr size_t collectionSizeODIN = sizeof(bank_collection_t); + static constexpr size_t collectionSizeVP = sizeof(bank_collection_t)+(maxTell40VP-1)*sizeof(bank_t); + static constexpr size_t collectionSizeUT = sizeof(bank_collection_t)+(maxTell40UT-1)*sizeof(bank_t); + static constexpr size_t collectionSizeFT = sizeof(bank_collection_t)+(maxTell40FT-1)*sizeof(bank_t); + static constexpr size_t collectionSizeEcal = sizeof(bank_collection_t)+(maxTell40Ecal-1)*sizeof(bank_t); + static constexpr size_t collectionSizeHcal = sizeof(bank_collection_t)+(maxTell40Hcal-1)*sizeof(bank_t); + static constexpr size_t collectionSizeMuon = sizeof(bank_collection_t)+(maxTell40Muon-1)*sizeof(bank_t); + static constexpr size_t collectionSizeRich = sizeof(bank_collection_t)+(maxTell40Rich-1)*sizeof(bank_t); + static constexpr size_t collectionSizeOther = sizeof(bank_collection_t)+(maxTell40Other-1)*sizeof(bank_t); + + static constexpr size_t collectionOffset = event_t::data_offset; + static constexpr size_t collectionOffsetODIN = collectionOffset; + static constexpr size_t collectionOffsetVP = collectionOffsetODIN + collectionSizeODIN; + static constexpr size_t collectionOffsetUT = collectionOffsetVP + collectionSizeVP; + static constexpr size_t collectionOffsetFT = collectionOffsetUT + collectionSizeUT; + static constexpr size_t collectionOffsetEcal = collectionOffsetFT + collectionSizeFT; + static constexpr size_t collectionOffsetHcal = collectionOffsetEcal + collectionSizeEcal; + static constexpr size_t collectionOffsetMuon = collectionOffsetHcal + collectionSizeHcal; + static constexpr size_t collectionOffsetRich = collectionOffsetMuon + collectionSizeMuon; + static constexpr size_t collectionOffsetOther = collectionOffsetRich + collectionSizeRich; static constexpr size_t collectionOffsets[] = {collectionOffsetODIN, collectionOffsetVP, @@ -406,10 +416,11 @@ namespace Online { collectionOffsetEcal, collectionOffsetHcal, collectionOffsetMuon, - collectionOffsetRich - }; + collectionOffsetRich, + collectionOffsetOther }; + static constexpr size_t num_bank_collection = (sizeof(collectionOffsets)/sizeof(collectionOffsets[0])); - static constexpr size_t eventRecordLength = collectionOffsetRich + collectionSizeRich; + static constexpr size_t eventRecordLength = collectionOffsetOther + collectionSizeOther; } /// Access to data block @@ -492,19 +503,19 @@ namespace Online { } inline const bank_t* bank_collection_t::begin() const { - return banks; + return this->banks; } inline const bank_t* bank_collection_t::special_begin() const { - return add_ptr<bank_t>(banks,sizeof(bank_t) * (capacity-specials)); + return add_ptr<bank_t>(this->banks,sizeof(bank_t) * (capacity-specials)); } inline const bank_t* bank_collection_t::special_end() const { - return add_ptr<bank_t>(banks,sizeof(bank_t) * capacity); + return add_ptr<bank_t>(this->banks,sizeof(bank_t) * capacity); } inline const bank_t* bank_collection_t::end() const { - return add_ptr<bank_t>(banks,sizeof(bank_t) * length); + return add_ptr<bank_t>(this->banks,sizeof(bank_t) * length); } inline const bank_t* bank_collection_t::next(const bank_t* e) const { @@ -512,11 +523,37 @@ namespace Online { } inline const bank_t* bank_collection_t::at(size_t i) const { - return add_ptr<bank_t>(banks,sizeof(bank_t)*i); + return add_ptr<bank_t>(this->banks,sizeof(bank_t)*i); } inline const bank_t* bank_collection_t::operator[](size_t i) const { - return add_ptr<bank_t>(banks,sizeof(bank_t)*i); + return add_ptr<bank_t>(this->banks,sizeof(bank_t)*i); + } + + inline size_t bank_collection_t::total_length() const { + size_t len = 0; + for( const auto* b=this->begin(); b != this->end(); b=this->next(b) ) + len += b->totalSize(); + for( const auto* b=this->special_begin(); b != this->special_end(); b=this->next(b) ) + len += b->totalSize(); + return len; + } + + inline void* bank_collection_t::copy_data(void* ptr) const { + unsigned char* p = (unsigned char*)ptr; + for( const auto* b=this->begin(); b != this->end(); b=this->next(b) ) { + size_t len = b->hdrSize(); + ::memcpy(p, b, len); + ::memcpy(p + len, b->data(), b->totalSize()-len); + p += b->totalSize(); + } + for( const auto* b=this->special_begin(); b != this->special_end(); b=this->next(b) ) { + size_t len = b->hdrSize(); + ::memcpy(p, b, len); + ::memcpy(p + len, b->data(), b->totalSize()-len); + p += b->totalSize(); + } + return p; } inline std::pair<bool,size_t> event_t::collection_offset(unsigned char type) { @@ -541,6 +578,8 @@ namespace Online { return std::make_pair(true,params::collectionOffsetHcal); case bank_t::Muon: return std::make_pair(true,params::collectionOffsetMuon); + case bank_t::Other: + return std::make_pair(true,params::collectionOffsetOther); default: throw std::runtime_error("Unknown bank type!"); } @@ -563,6 +602,19 @@ namespace Online { return add_ptr<bank_collection_t>(this,params::collectionOffsets[which]); } + inline size_t event_t::total_length() const { + size_t len = 0; + for( size_t i=0, n=this->num_bank_collections(); i<n; ++i) + len += this->bank_collection(i)->total_length(); + return len; + } + + inline void* event_t::copy_data(void* ptr) const { + for( size_t i=0, n=this->num_bank_collections(); i<n; ++i) + ptr = this->bank_collection(i)->copy_data(ptr); + return ptr; + } + inline const event_t* event_collection_t::begin() const { return events; } @@ -582,6 +634,14 @@ namespace Online { inline const event_t* event_collection_t::at(size_t i) const { return add_ptr<event_t>(events,params::eventRecordLength*i); } + + inline size_t event_collection_t::total_length() const { + size_t len = 0; + for( const event_t* e=this->begin(); e != this->end(); e=this->next(e) ) + len += e->total_length(); + return len; + } + } // namespace pcie40 } // namepace Online diff --git a/Online/PCIE40Data/main/pcie40_decode_file.cpp b/Online/PCIE40Data/main/pcie40_decode_file.cpp index 7b07e392f..54f19e0af 100644 --- a/Online/PCIE40Data/main/pcie40_decode_file.cpp +++ b/Online/PCIE40Data/main/pcie40_decode_file.cpp @@ -16,6 +16,7 @@ #include <unistd.h> #include <fcntl.h> #include <cstdlib> +#include <cstring> #include <cstdio> #include <vector> #include <list> @@ -47,10 +48,18 @@ int main(int argc, char *argv[]) { const auto *mep_end = pcie40::add_ptr<pcie40::mep_header_t>(mep_start, sb.st_size-4); ::close(fd); + if ( mep_start == (pcie40::mep_header_t*)MAP_FAILED ) { + ::printf("+++ FAILED mapping mep file %s [%s]\n", argv[1], ::strerror(errno)); + ::exit(errno); + } + start = clock(); ::printf("+++ mapping mep file %p %ld %p\n", (void*)mep_start, sb.st_size, (void*)((char*)mep_start + sb.st_size)); int n_mep = 0; for(const pcie40::mep_header_t* mep = mep_start; mep < mep_end; ) { + unsigned char* p = (unsigned char*)mep; + p += sizeof(long); + mep = (pcie40::mep_header_t*)p; if ( !mep->is_valid() ) break; ::printf("+++ Reading mep[%2d] at %p %6d %p\n", n_mep, (void*)mep, mep->size, (void*)((char*)mep + mep->size)); diff --git a/Online/SmiController/options/TestMEPProd.opts b/Online/SmiController/options/TestMEPProd.opts index f349f3dab..53cf3dcc6 100644 --- a/Online/SmiController/options/TestMEPProd.opts +++ b/Online/SmiController/options/TestMEPProd.opts @@ -30,6 +30,7 @@ Reader.RequireConsumers = 0; Reader.MMapFiles = 0; Reader.ReuseFile = 1; Reader.PackingFactor = 1000; +Reader.HeaderPrefixSize = 8; //Reader.AllocationSizekB = 4096; //Reader.PackingFactor = 4000; //Reader.AllocationSizekB = 50000; diff --git a/Online/SmiController/scripts/EventProcessor.py b/Online/SmiController/scripts/EventProcessor.py index 9ebee4354..a8e3bbb57 100644 --- a/Online/SmiController/scripts/EventProcessor.py +++ b/Online/SmiController/scripts/EventProcessor.py @@ -33,9 +33,9 @@ application.setup_hive(FlowManager("EventLoop"), 40) application.setup_algorithms(writer, 0.05) application.setup_monitoring() application.monSvc.DimUpdateInterval = 1 -application.config.numEventThreads = 2 -application.config.MBM_numConnections = 2 -application.config.MBM_numEventThreads = 2 +application.config.numEventThreads = 1 +application.config.MBM_numConnections = 1 +application.config.MBM_numEventThreads = 1 application.config.MBM_requests = [ 'EvType=3;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0', 'EvType=2;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0', diff --git a/Online/SmiController/scripts/TestMEPProd.sh b/Online/SmiController/scripts/TestMEPProd.sh new file mode 100644 index 000000000..48b19a903 --- /dev/null +++ b/Online/SmiController/scripts/TestMEPProd.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# +`dataflow_task Class2` `dataflow_default_options ${TASK_TYPE}` diff --git a/Online/SmiController/scripts/TestStorageWriter.sh b/Online/SmiController/scripts/TestStorageWriter.sh index ab01e46e6..a12662bb2 100755 --- a/Online/SmiController/scripts/TestStorageWriter.sh +++ b/Online/SmiController/scripts/TestStorageWriter.sh @@ -9,5 +9,6 @@ # # ========================================================================= export INFO_OPTIONS=/group/online/dataflow/options/LHCb/MONITORING/OnlineEnv.opts; -`dataflow_task Class1` `dataflow_default_options ${TASK_TYPE}` +echo "ERROR `dataflow_task Class1` `dataflow_default_options ${TASK_TYPE}`"; +`dataflow_task Class1` `dataflow_default_options ${TASK_TYPE}`; diff --git a/Online/Storage/src/server/fdb_SQLite.cpp b/Online/Storage/src/server/fdb_SQLite.cpp index da2c14614..733995260 100644 --- a/Online/Storage/src/server/fdb_SQLite.cpp +++ b/Online/Storage/src/server/fdb_SQLite.cpp @@ -59,6 +59,8 @@ fdb_SQLite::handler_t::query(std::string& object, access = file.access; length = ::atol(file.size.c_str()); date = file.date; + ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, + "Query '%s'", object.c_str()); return ec; } ::lib_rtl_output(LIB_RTL_ALWAYS,"%s: Query: file '%s' NOT FOUND!", @@ -81,6 +83,8 @@ fdb_SQLite::handler_t::next( std::string& object, date = file.date; ec = this->lock(file.name, handler_t::STATE_READ); if ( ec == system::errc::success ) { + ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, + "Next '%s'", object.c_str()); return system::error_code(system::errc::success, system::system_category()); } return system::error_code(system::errc::protocol_error, system::system_category()); @@ -106,7 +110,7 @@ system::error_code fdb_SQLite::query_object(const std::string& object, access = this->network_file(access); if ( ec == system::errc::success ) { ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, - "Successfully lookup object: '%s'", object.c_str()); + "Lookup '%s'", object.c_str()); return ec; } ::lib_rtl_output(LIB_RTL_ALWAYS,"FAILED to lookup object: '%s' [%s]", @@ -125,7 +129,7 @@ system::error_code fdb_SQLite::query_next( const std::string& object, access = this->network_file(access); if ( ec == system::errc::success ) { ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, - "Successfully lookup object: '%s'", object.c_str()); + "Lookup '%s'", object.c_str()); return ec; } ::lib_rtl_output(LIB_RTL_ALWAYS,"FAILED to lookup object: '%s' [%s]", @@ -146,7 +150,7 @@ system::error_code fdb_SQLite::delete_next( const std::string& object, access = this->network_file(access); if ( ec == system::errc::success ) { ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, - "Successfully lookup sequence object: '%s'", object.c_str()); + "Lookup '%s'", object.c_str()); return ec; } } @@ -168,7 +172,7 @@ system::error_code fdb_SQLite::delete_object(const std::string& object, ec = this->engine->del(object); if ( ec == system::errc::success ) { ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, - "Removed db record: '%s'", object.c_str()); + "Remove '%s'", object.c_str()); return ec; } } @@ -200,7 +204,9 @@ fdb_SQLite::add_object(const std::string& object, auto ec = this->engine->add(object, date, length, acc); if ( ec == system::errc::success ) { ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, - "Added file: '%s' access:%s.", object.c_str(), acc.c_str()); + "Add '%s' %s %s", object.c_str(), + this->debug > 1 ? "access:" : "", + this->debug > 1 ? acc.c_str() : ""); access = this->network_file(acc); return ec; } diff --git a/Online/Storage/src/server/fdb_SQLite.h b/Online/Storage/src/server/fdb_SQLite.h index 1ef04bf78..3ddb34347 100644 --- a/Online/Storage/src/server/fdb_SQLite.h +++ b/Online/Storage/src/server/fdb_SQLite.h @@ -51,7 +51,7 @@ namespace Online { static constexpr int STATE_READ = 1; sqlite::database database; - bool debug { false }; + int debug { 0 }; struct File { std::string name, state, size, date, access; }; @@ -60,7 +60,7 @@ namespace Online { query_file(const std::string& object_name, File& file, int state) = 0; public: - handler_t(bool dbg) + handler_t(int dbg) : debug(dbg) { } diff --git a/Online/Storage/src/server/fdb_db_server.cpp b/Online/Storage/src/server/fdb_db_server.cpp index 4a85a28e1..67c94adc7 100644 --- a/Online/Storage/src/server/fdb_db_server.cpp +++ b/Online/Storage/src/server/fdb_db_server.cpp @@ -191,8 +191,8 @@ http::basic_http_server<db>::handler_t::handle_request(const request_t& req, rep if ( 0 == rep.bytes_sent ) { auto ret = this->HttpRequestHandler::handle_request(req, rep); if ( this->debug > 0 || !(rep.status == reply_t::temp_redirect || rep.status == reply_t::ok) ) { - ::lib_rtl_output(LIB_RTL_INFO,"basic_http_server<db>::handle: Serving %s %s status: %d", - req.method.c_str(), req.uri.c_str(), rep.status); + ::lib_rtl_output(LIB_RTL_INFO,"basic_http_server<db>::handle: Serving %s %s status: %d [%s]", + req.method.c_str(), req.uri.c_str(), rep.status, reply_t::stock_status(rep.status)); for ( const auto& h : rep.headers ) ::lib_rtl_output(LIB_RTL_INFO,"basic_http_server<db>::handle: %-20s = %s", h.name.c_str(), h.value.c_str()); diff --git a/Online/Tell1Data/Tell1Data/RawFile.h b/Online/Tell1Data/Tell1Data/RawFile.h index 051e6aabc..a69ac80c9 100644 --- a/Online/Tell1Data/Tell1Data/RawFile.h +++ b/Online/Tell1Data/Tell1Data/RawFile.h @@ -86,7 +86,7 @@ namespace Online { bool isOpen() const { return m_fd > 0; } bool isMapped() const { return m_begin != m_end; } size_t mapped_size() const { return m_end - m_begin; } - size_t data_size() const { return m_end - m_ptr; } + size_t data_size() const; const unsigned char* begin() const { return m_begin; } const unsigned char* end() const { return m_end; } const unsigned char* pointer() const { return m_ptr; } diff --git a/Online/Tell1Data/src/RawFile.cpp b/Online/Tell1Data/src/RawFile.cpp index 657cbb761..b66d28a68 100644 --- a/Online/Tell1Data/src/RawFile.cpp +++ b/Online/Tell1Data/src/RawFile.cpp @@ -77,6 +77,19 @@ long RawFile::seek(long offset, int whence) { return -1; } +/// Return file size +size_t RawFile::data_size() const { + if ( isOpen() && !m_begin ) { + struct stat sb; + ::fstat(m_fd, &sb); /* To obtain file size */ + return sb.st_size; + } + else if ( m_begin ) { + return m_end - m_begin; + } + return 0; +} + /// Return file position long RawFile::position() const { if ( isOpen() && !m_begin ) { -- GitLab From b65397674df4fb35c5c9cbd626ecd5d6566652ba Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Fri, 5 Mar 2021 10:23:42 +0100 Subject: [PATCH 02/26] Enable conditional build for EventBuilding package if no PCIE40 is found --- Online/EventBuilding/CMakeLists.txt | 12 +++++++++++- Online/FarmConfig/job/BU.sh | 12 ++++++++++++ Online/FarmConfig/job/EBMBM.sh | 12 ++++++++++++ Online/FarmConfig/job/RU.sh | 12 ++++++++++++ Online/FarmConfig/options/EBMBM.opts | 2 ++ 5 files changed, 49 insertions(+), 1 deletion(-) create mode 100755 Online/FarmConfig/job/BU.sh create mode 100755 Online/FarmConfig/job/EBMBM.sh create mode 100755 Online/FarmConfig/job/RU.sh create mode 100644 Online/FarmConfig/options/EBMBM.opts diff --git a/Online/EventBuilding/CMakeLists.txt b/Online/EventBuilding/CMakeLists.txt index fba083de3..255209d10 100755 --- a/Online/EventBuilding/CMakeLists.txt +++ b/Online/EventBuilding/CMakeLists.txt @@ -22,10 +22,20 @@ gaudi_depends_on_subdirs(Online/DIM Online/Dataflow Online/OnlineBase Online/PCIE40Data) -include(Numa) + +find_path(PCIE40_DAQ_INC /usr/include/lhcb/daq40/daq40.hpp) +if ( "${PCIE40_DAQ_INC}" STREQUAL "PCIE40_DAQ_INC-NOTFOUND" ) + message(STATUS "+======================================================================+") + message(STATUS "| PCIE40 DAQ not present. Will not build EventBuilding libraries. |") + message(STATUS "+======================================================================+") + return() +endif() + # fpisani fixed find of external libraries find_package(MPI REQUIRED) +include(Numa) find_package(PkgConfig REQUIRED) + set(OLD_CONFIG_PATH $ENV{PKG_CONFIG_PATH}) # TODO check if there is a better way to identify this path set(ENV{PKG_CONFIG_PATH} /usr/lib64/pkgconfig) diff --git a/Online/FarmConfig/job/BU.sh b/Online/FarmConfig/job/BU.sh new file mode 100755 index 000000000..beca57daf --- /dev/null +++ b/Online/FarmConfig/job/BU.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# ========================================================================= +# +# Default script to start the buffer manager on the HLT farm worker node +# +# Author M.Frank +# Version: 1.0 +# Date: 05/03/2021 +# +# ========================================================================= +# +exec -a ${UTGID} ${DATAFLOW_task} -class=Class1 -opts=../../EventBuilding/options/EB_BU.opts $(AUTO_STARTUP} ${DEBUG_STARTUP} diff --git a/Online/FarmConfig/job/EBMBM.sh b/Online/FarmConfig/job/EBMBM.sh new file mode 100755 index 000000000..30097ae9f --- /dev/null +++ b/Online/FarmConfig/job/EBMBM.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# ========================================================================= +# +# Default script to start the buffer manager on the HLT farm worker node +# +# Author M.Frank +# Version: 1.0 +# Date: 05/03/2021 +# +# ========================================================================= +# +exec -a ${UTGID} ${DATAFLOW_task} -class=Class0 -opts=../options/${TASK_TYPE}.opts; diff --git a/Online/FarmConfig/job/RU.sh b/Online/FarmConfig/job/RU.sh new file mode 100755 index 000000000..c44774a95 --- /dev/null +++ b/Online/FarmConfig/job/RU.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# ========================================================================= +# +# Default script to start the buffer manager on the HLT farm worker node +# +# Author M.Frank +# Version: 1.0 +# Date: 05/03/2021 +# +# ========================================================================= +# +exec -a ${UTGID} ${DATAFLOW_task} -class=Class1 -opts=../../EventBuilding/options/EB_RU.opts $(AUTO_STARTUP} ${DEBUG_STARTUP} diff --git a/Online/FarmConfig/options/EBMBM.opts b/Online/FarmConfig/options/EBMBM.opts new file mode 100644 index 000000000..2ce8dabe5 --- /dev/null +++ b/Online/FarmConfig/options/EBMBM.opts @@ -0,0 +1,2 @@ +#include "$FARMCONFIG_OPTIONS/MEPInit.opts" +MEPManager.InitFlags = "-s=200000 -e=500 -u=20 -b=17 -f -i=Events_0 -c -s=200000 -e=500 -u=20 -b=17 -f -i=Events_1 -c"; -- GitLab From 2ed54c9241438fd8f6590868933d0bc627209d62 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Mon, 8 Mar 2021 15:31:49 +0100 Subject: [PATCH 03/26] Commit first results from EventBuilding setup --- Online/Dataflow/CMakeLists.txt | 4 +- Online/Dataflow/src/Gaucho/DIMMonitoring.cpp | 26 +++++++---- Online/EventBuilding/CMakeLists.txt | 15 +++--- Online/EventBuilding/options/BU_0.opts | 3 ++ Online/EventBuilding/options/BU_1.opts | 3 ++ Online/EventBuilding/options/EB_BU.opts | 14 ++---- Online/EventBuilding/options/EB_BU_1.opts | 0 Online/EventBuilding/options/EB_BU_algo.opts | 7 +++ Online/EventBuilding/options/EB_RU.opts | 14 ++---- Online/EventBuilding/options/EB_RU_1.opts | 0 Online/EventBuilding/options/EB_RU_algo.opts | 10 ++++ .../options/EB_Transport_properties.opts | 8 ++++ .../EventBuilding/options/EB_transport.opts | 10 +--- Online/EventBuilding/options/Logging.opts | 0 Online/EventBuilding/options/MBMSetup.opts | 0 Online/EventBuilding/options/MBM_server.opts | 0 Online/EventBuilding/options/MEPInit.opts | 0 .../EventBuilding/options/MFP_generator.opts | 10 +--- .../options/MFP_generator_params.opts | 7 +++ Online/EventBuilding/options/OnlineEnv.opts | 0 .../options/events_dispatch/BU_dummy.opts | 0 .../options/events_dispatch/FU.opts | 0 .../events_dispatch/MBMSetup_Input.opts | 0 .../events_dispatch/MBMSetup_Input_2.opts | 0 .../events_dispatch/MBM_reader_dummy.opts | 0 .../events_dispatch/MEPInit_Input.opts | 0 .../events_dispatch/MEPInit_Input_2.opts | 0 .../options/events_dispatch/MU.opts | 0 .../options/events_dispatch/OU.opts | 0 .../EventBuilding/options/hosts_example.json | 0 Online/FarmConfig/job/BU.sh | 5 +- Online/FarmConfig/job/Controller.sh | 2 +- Online/FarmConfig/job/RU.sh | 2 +- Online/FarmConfig/job/preamble.sh | 7 ++- Online/FarmConfig/job/runTask.sh | 14 ++---- Online/FarmConfig/options/EBMBM.opts | 11 ++++- Online/SmiController/src/smi_controller.cpp | 46 ++++++++++++------- Online/Storage/src/server/fdb_db_server.cpp | 8 ++-- 38 files changed, 138 insertions(+), 88 deletions(-) create mode 100644 Online/EventBuilding/options/BU_0.opts create mode 100644 Online/EventBuilding/options/BU_1.opts mode change 100644 => 100755 Online/EventBuilding/options/EB_BU.opts mode change 100644 => 100755 Online/EventBuilding/options/EB_BU_1.opts create mode 100755 Online/EventBuilding/options/EB_BU_algo.opts mode change 100644 => 100755 Online/EventBuilding/options/EB_RU.opts mode change 100644 => 100755 Online/EventBuilding/options/EB_RU_1.opts create mode 100755 Online/EventBuilding/options/EB_RU_algo.opts create mode 100755 Online/EventBuilding/options/EB_Transport_properties.opts mode change 100644 => 100755 Online/EventBuilding/options/EB_transport.opts mode change 100644 => 100755 Online/EventBuilding/options/Logging.opts mode change 100644 => 100755 Online/EventBuilding/options/MBMSetup.opts mode change 100644 => 100755 Online/EventBuilding/options/MBM_server.opts mode change 100644 => 100755 Online/EventBuilding/options/MEPInit.opts mode change 100644 => 100755 Online/EventBuilding/options/MFP_generator.opts create mode 100755 Online/EventBuilding/options/MFP_generator_params.opts mode change 100644 => 100755 Online/EventBuilding/options/OnlineEnv.opts mode change 100644 => 100755 Online/EventBuilding/options/events_dispatch/BU_dummy.opts mode change 100644 => 100755 Online/EventBuilding/options/events_dispatch/FU.opts mode change 100644 => 100755 Online/EventBuilding/options/events_dispatch/MBMSetup_Input.opts mode change 100644 => 100755 Online/EventBuilding/options/events_dispatch/MBMSetup_Input_2.opts mode change 100644 => 100755 Online/EventBuilding/options/events_dispatch/MBM_reader_dummy.opts mode change 100644 => 100755 Online/EventBuilding/options/events_dispatch/MEPInit_Input.opts mode change 100644 => 100755 Online/EventBuilding/options/events_dispatch/MEPInit_Input_2.opts mode change 100644 => 100755 Online/EventBuilding/options/events_dispatch/MU.opts mode change 100644 => 100755 Online/EventBuilding/options/events_dispatch/OU.opts mode change 100644 => 100755 Online/EventBuilding/options/hosts_example.json diff --git a/Online/Dataflow/CMakeLists.txt b/Online/Dataflow/CMakeLists.txt index bf780fad6..ade206dfe 100755 --- a/Online/Dataflow/CMakeLists.txt +++ b/Online/Dataflow/CMakeLists.txt @@ -29,7 +29,7 @@ gaudi_depends_on_subdirs(Online/dim Online/PCIE40Data) # include(Numa) -include(PCIE40) +###include(PCIE40) # find_package(Boost REQUIRED COMPONENTS system filesystem regex) find_package(ROOT REQUIRED COMPONENTS Hist RIO) @@ -239,7 +239,7 @@ IF (PCIE40_FOUND) target_include_directories(DataflowPCIE40 BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) gaudi_generate_componentslist(DataflowPCIE40) ELSE() - message(STATUS "PCIE40 event builder library shall not be built!") + message(STATUS "+++ Dataflow: PCIE40 event builder library shall not be built!") ENDIF() # # --------------------------------------------------------------------------------------- diff --git a/Online/Dataflow/src/Gaucho/DIMMonitoring.cpp b/Online/Dataflow/src/Gaucho/DIMMonitoring.cpp index 4d8da8f47..8923ca603 100755 --- a/Online/Dataflow/src/Gaucho/DIMMonitoring.cpp +++ b/Online/Dataflow/src/Gaucho/DIMMonitoring.cpp @@ -147,8 +147,10 @@ void DIMMonitoring::handle(const DataflowIncident& inc) { /// Monitoring interface: Undeclare single monitoring item int DIMMonitoring::undeclare(const string& owner, const string& nam) { - string oname = !owner.empty() ? owner : string(""); - m_MonIntf->undeclare(oname+"/"+nam); + if ( m_MonIntf ) { + string oname = !owner.empty() ? owner : string(""); + m_MonIntf->undeclare(oname+"/"+nam); + } return DF_SUCCESS; } @@ -156,8 +158,10 @@ int DIMMonitoring::undeclare(const string& owner, const string& nam) { @param owner Owner identifier of the monitoring information */ int DIMMonitoring::undeclare(const string& owner) { - string oname = !owner.empty() ? owner : string(""); - m_MonIntf->undeclareAll(oname); + if ( m_MonIntf ) { + string oname = !owner.empty() ? owner : string(""); + m_MonIntf->undeclareAll(oname); + } return DF_SUCCESS; } @@ -166,7 +170,9 @@ int DIMMonitoring::i_startMonitoring() { debug("Starting publishing monitor information using DIM...."); // setProperties(); // dis_set_debug_on(); - m_MonIntf->start(); + if ( m_MonIntf ) { + m_MonIntf->start(); + } setRunNo(m_runno); m_started = true; return DF_SUCCESS; @@ -179,8 +185,10 @@ int DIMMonitoring::i_stopMonitoring() { return DF_SUCCESS; } void DIMMonitoring::stopUpdate() { - m_MonIntf->stop(); - stopSaving(); + if ( m_MonIntf ) { + m_MonIntf->stop(); + stopSaving(); + } } int DIMMonitoring::finalize() { @@ -188,7 +196,9 @@ int DIMMonitoring::finalize() { dim_lock(); if (m_started) { m_started = false; - m_MonIntf->stop(); + if ( m_MonIntf ) { + m_MonIntf->stop(); + } } dim_unlock(); detail::deletePtr(m_MonIntf); diff --git a/Online/EventBuilding/CMakeLists.txt b/Online/EventBuilding/CMakeLists.txt index 255209d10..79e205d7c 100755 --- a/Online/EventBuilding/CMakeLists.txt +++ b/Online/EventBuilding/CMakeLists.txt @@ -17,18 +17,21 @@ gaudi_subdir(EventBuilding v0r1) # -gaudi_depends_on_subdirs(Online/DIM - Online/GauchoBase - Online/Dataflow - Online/OnlineBase - Online/PCIE40Data) +## Online/dim Online/GauchoBase Online/PCIE40Data +gaudi_depends_on_subdirs(Online/Dataflow + Online/OnlineBase) -find_path(PCIE40_DAQ_INC /usr/include/lhcb/daq40/daq40.hpp) +# Check for local PCIE40. If not present do not build event builder libraries +find_path(PCIE40_DAQ_INC NAMES "daq40.hpp" PATHS /usr/include/lhcb/daq40) if ( "${PCIE40_DAQ_INC}" STREQUAL "PCIE40_DAQ_INC-NOTFOUND" ) message(STATUS "+======================================================================+") message(STATUS "| PCIE40 DAQ not present. Will not build EventBuilding libraries. |") message(STATUS "+======================================================================+") return() +else() + message(STATUS "+======================================================================+") + message(STATUS "| PCIE40 DAQ found. EventBuilding libraries shall be built. |") + message(STATUS "+======================================================================+") endif() # fpisani fixed find of external libraries diff --git a/Online/EventBuilding/options/BU_0.opts b/Online/EventBuilding/options/BU_0.opts new file mode 100644 index 000000000..e210c041e --- /dev/null +++ b/Online/EventBuilding/options/BU_0.opts @@ -0,0 +1,3 @@ +#pragma print off +#include "$EVENTBUILDINGROOT/options/EB_BU.opts" +MEPManager.Buffers = { "Events_0" }; diff --git a/Online/EventBuilding/options/BU_1.opts b/Online/EventBuilding/options/BU_1.opts new file mode 100644 index 000000000..c0b8d80a4 --- /dev/null +++ b/Online/EventBuilding/options/BU_1.opts @@ -0,0 +1,3 @@ +#pragma print off +#include "$EVENTBUILDINGROOT/options/EB_BU.opts" +MEPManager.Buffers = { "Events_1" }; diff --git a/Online/EventBuilding/options/EB_BU.opts b/Online/EventBuilding/options/EB_BU.opts old mode 100644 new mode 100755 index 205d7a315..1f72a15c4 --- a/Online/EventBuilding/options/EB_BU.opts +++ b/Online/EventBuilding/options/EB_BU.opts @@ -1,11 +1,10 @@ -#pragma print on +#pragma print off #include "$INFO_OPTIONS" #include "$MBM_SETUP_OPTIONS" -#include "$EVENTBUILDINGROOT/options/Logging.opts" -#include "$EVENTBUILDINGROOT/options/Monitoring.opts" +#include "$FARMCONFIG_OPTIONS/Logging.opts" +#include "$FARMCONFIG_OPTIONS/Monitoring.opts" #include "$EVENTBUILDINGROOT/options/EB_transport.opts" - Manager.Services = { //"Dataflow_MBMClient/MEPManager", "EB_BU/BU", @@ -15,12 +14,7 @@ Manager.Services = { Manager.Runable = "Wrap"; Wrap.Callable = "BU"; // algorithm properties -// BU.bu_rank = 4; -// BU.buffer_type = 0; -// BU.MBM_name = @OnlineEnv.Input_Buffer; -// BU.buffer_size = 1073741824; -BU.out_file_prefix = "rand_size_BU"; -// BU.write_to_file = false; +#include "$EVENTBUILDINGROOT/options/EB_BU_algo.opts" // properties of the Transport_unit class set them from the EB_transport.opts file BU.RUs_per_nic = @EB_transport.RUs_per_nic; diff --git a/Online/EventBuilding/options/EB_BU_1.opts b/Online/EventBuilding/options/EB_BU_1.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/EB_BU_algo.opts b/Online/EventBuilding/options/EB_BU_algo.opts new file mode 100755 index 000000000..4f407d2f0 --- /dev/null +++ b/Online/EventBuilding/options/EB_BU_algo.opts @@ -0,0 +1,7 @@ +BU.MBM_name = {}; +BU.buffer_size = {}; +BU.buffer_type = {}; +BU.out_file_prefix = ""; +BU.shmem_prefix = ""; +BU.stop_timeout = 0; +BU.write_to_file = { FALSE, FALSE, FALSE, FALSE, FALSE, FALSE }; diff --git a/Online/EventBuilding/options/EB_RU.opts b/Online/EventBuilding/options/EB_RU.opts old mode 100644 new mode 100755 index 443d3cf75..7a9ef1729 --- a/Online/EventBuilding/options/EB_RU.opts +++ b/Online/EventBuilding/options/EB_RU.opts @@ -1,7 +1,8 @@ -#pragma print on +#pragma print off #include "$INFO_OPTIONS" #include "$EVENTBUILDINGROOT/options/EB_transport.opts" -#include "$EVENTBUILDINGROOT/options/Logging.opts" +#include "$FARMCONFIG_OPTIONS/Monitoring.opts" +#include "$FARMCONFIG_OPTIONS/Logging.opts" Manager.Services = {"EB_RU/RU", "Dataflow_RunableWrapper/Wrap" @@ -9,14 +10,7 @@ Manager.Services = {"EB_RU/RU", Manager.Runable = "Wrap"; Wrap.Callable = "RU"; // algorithm properties -RU.buffer_type = {}; -RU.PCIe40_ids = {}; -RU.MDF_filename = "/home/fpisani/public/eb_test_file.mdf"; -RU.n_MFPs = 100; -RU.n_fragment = 3000; -RU.MPI_errors_return = true; -RU.PCIe40_timeout = 5; -RU.shmem_prefix = "$SHMEM_PREFIX"; +#include "$EVENTBUILDINGROOT/options/EB_RU_algo.opts" // properties of the Transport_unit class set them from the EB_transport.opts file RU.RUs_per_nic = @EB_transport.RUs_per_nic; diff --git a/Online/EventBuilding/options/EB_RU_1.opts b/Online/EventBuilding/options/EB_RU_1.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/EB_RU_algo.opts b/Online/EventBuilding/options/EB_RU_algo.opts new file mode 100755 index 000000000..3b6f57e29 --- /dev/null +++ b/Online/EventBuilding/options/EB_RU_algo.opts @@ -0,0 +1,10 @@ +RU.MDF_filename = "/home/fpisani/mc_data/biger_file.mdf"; +RU.PCIe40_ids = {}; +RU.PCIe40_names = { 'tdtel022_0', 'tdtel022_1', 'tdtel042_0', 'tdtel042_1', 'tdtel051_1', 'tdtel052_0', 'tdtel052_1', 'tdtel053_0', 'tdtel053_1' }; +RU.PCIe40_timeout = 30; +RU.buffer_sizes = {}; +RU.buffer_type = { 3, 3, 3, 3, 3, 3 }; +RU.n_MFPs = 100; +RU.n_fragment = 3000; +RU.shmem_prefix = "$SHMEM_PREFIX"; +RU.stop_timeout = 0; diff --git a/Online/EventBuilding/options/EB_Transport_properties.opts b/Online/EventBuilding/options/EB_Transport_properties.opts new file mode 100755 index 000000000..85150ab08 --- /dev/null +++ b/Online/EventBuilding/options/EB_Transport_properties.opts @@ -0,0 +1,8 @@ +EB_transport.BU_ranks = {}; +EB_transport.MPI_errors_return = FALSE; +EB_transport.RU_ranks = {}; +EB_transport.RUs_per_nic = {}; +EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.t25565.json"; +EB_transport.n_par_mess = 0; +EB_transport.n_sources_per_ru = { 1, 1, 1, 1, 3, 2 }; +EB_transport.src_ids = {}; diff --git a/Online/EventBuilding/options/EB_transport.opts b/Online/EventBuilding/options/EB_transport.opts old mode 100644 new mode 100755 index 9731e2514..e2980fc46 --- a/Online/EventBuilding/options/EB_transport.opts +++ b/Online/EventBuilding/options/EB_transport.opts @@ -1,11 +1,5 @@ -#pragma print on -#include "$INFO_OPTIONS" -#include "$EVENTBUILDINGROOT/options/Logging.opts" - // EB transport_unit properties -EB_transport.RUs_per_nic = {}; -EB_transport.n_par_mess = 3; -EB_transport.n_sources_per_ru = {3,3,3,3,3,3,3,3,3,3}; +#include "$EVENTBUILDINGROOT/options/EB_Transport_properties.opts" EB_transport.MPI_errors_return = true; EB_transport.OutputLevel = 3; @@ -14,4 +8,4 @@ EB_transport.src_ids = {}; EB_transport.RU_ranks = {}; EB_transport.BU_ranks = {}; EB_transport.ghost_nodes_topology = {}; -EB_transport.ib_config_file = "/home/aperro/Online/hosts.json"; +//EB_transport.ib_config_file = "/home/aperro/Online/hosts.json"; diff --git a/Online/EventBuilding/options/Logging.opts b/Online/EventBuilding/options/Logging.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/MBMSetup.opts b/Online/EventBuilding/options/MBMSetup.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/MBM_server.opts b/Online/EventBuilding/options/MBM_server.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/MEPInit.opts b/Online/EventBuilding/options/MEPInit.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/MFP_generator.opts b/Online/EventBuilding/options/MFP_generator.opts old mode 100644 new mode 100755 index a9b83ce19..5dd523382 --- a/Online/EventBuilding/options/MFP_generator.opts +++ b/Online/EventBuilding/options/MFP_generator.opts @@ -1,17 +1,11 @@ #pragma print on #include "$INFO_OPTIONS" #include "$EVENTBUILDINGROOT/options/EB_transport.opts" -#include "$EVENTBUILDINGROOT/options/Logging.opts" +#include "$FARMCONFIG_OPTIONS/Logging.opts" Manager.Services = {"MFP_generator/MFP_generator", "Dataflow_RunableWrapper/Wrap" }; Manager.Runable = "Wrap"; Wrap.Callable = "MFP_generator"; -MFP_generator.shmem_name = "$SHMEM_NAME"; -MFP_generator.OutputLevel = 3; -MFP_generator.event_rate = 30e7; -MFP_generator.n_banks = 30000; -MFP_generator.bank_size = 100; -MFP_generator.random_sizes = false; -MFP_generator.reconnect = false; +#include "$EVENTBUILDINGROOT/options/MFP_generator_params.opts" diff --git a/Online/EventBuilding/options/MFP_generator_params.opts b/Online/EventBuilding/options/MFP_generator_params.opts new file mode 100755 index 000000000..3f133088e --- /dev/null +++ b/Online/EventBuilding/options/MFP_generator_params.opts @@ -0,0 +1,7 @@ +MFP_generator.OutputLevel = 3; +MFP_generator.bank_size = 100; +MFP_generator.event_rate = 300000000; +MFP_generator.n_banks = 30000; +MFP_generator.random_sizes = FALSE; +MFP_generator.reconnect = FALSE; +MFP_generator.shmem_name = "$SHMEM_NAME"; diff --git a/Online/EventBuilding/options/OnlineEnv.opts b/Online/EventBuilding/options/OnlineEnv.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/events_dispatch/BU_dummy.opts b/Online/EventBuilding/options/events_dispatch/BU_dummy.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/events_dispatch/FU.opts b/Online/EventBuilding/options/events_dispatch/FU.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/events_dispatch/MBMSetup_Input.opts b/Online/EventBuilding/options/events_dispatch/MBMSetup_Input.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/events_dispatch/MBMSetup_Input_2.opts b/Online/EventBuilding/options/events_dispatch/MBMSetup_Input_2.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/events_dispatch/MBM_reader_dummy.opts b/Online/EventBuilding/options/events_dispatch/MBM_reader_dummy.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/events_dispatch/MEPInit_Input.opts b/Online/EventBuilding/options/events_dispatch/MEPInit_Input.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/events_dispatch/MEPInit_Input_2.opts b/Online/EventBuilding/options/events_dispatch/MEPInit_Input_2.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/events_dispatch/MU.opts b/Online/EventBuilding/options/events_dispatch/MU.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/events_dispatch/OU.opts b/Online/EventBuilding/options/events_dispatch/OU.opts old mode 100644 new mode 100755 diff --git a/Online/EventBuilding/options/hosts_example.json b/Online/EventBuilding/options/hosts_example.json old mode 100644 new mode 100755 diff --git a/Online/FarmConfig/job/BU.sh b/Online/FarmConfig/job/BU.sh index beca57daf..d16ee5834 100755 --- a/Online/FarmConfig/job/BU.sh +++ b/Online/FarmConfig/job/BU.sh @@ -9,4 +9,7 @@ # # ========================================================================= # -exec -a ${UTGID} ${DATAFLOW_task} -class=Class1 -opts=../../EventBuilding/options/EB_BU.opts $(AUTO_STARTUP} ${DEBUG_STARTUP} +eval `python -c "import os;s=os.environ['UTGID'];print 'export BU_OPTIONS='+s[s.find('BU'):]+'.opts'"`; +# +exec -a ${UTGID} ${DATAFLOW_task} -class=Class1 -opts=../../EventBuilding/options/${BU_OPTIONS} \ + ${AUTO_STARTUP} ${DEBUG_STARTUP}; diff --git a/Online/FarmConfig/job/Controller.sh b/Online/FarmConfig/job/Controller.sh index ca86cf369..da768bfa6 100755 --- a/Online/FarmConfig/job/Controller.sh +++ b/Online/FarmConfig/job/Controller.sh @@ -37,7 +37,7 @@ if test -z "${DIM_DNS_NODE}"; then fi; # exec -a ${UTGID} `which gentest.exe` libSmiController.so smi_controller \ - -print=1 \ + -print=3 \ -logger=fifo \ -part=${PARTITION_NAME} \ -dns=${DIM_DNS_NODE} \ diff --git a/Online/FarmConfig/job/RU.sh b/Online/FarmConfig/job/RU.sh index c44774a95..cb15638ba 100755 --- a/Online/FarmConfig/job/RU.sh +++ b/Online/FarmConfig/job/RU.sh @@ -9,4 +9,4 @@ # # ========================================================================= # -exec -a ${UTGID} ${DATAFLOW_task} -class=Class1 -opts=../../EventBuilding/options/EB_RU.opts $(AUTO_STARTUP} ${DEBUG_STARTUP} +exec -a ${UTGID} ${DATAFLOW_task} -class=Class1 -opts=../../EventBuilding/options/EB_RU.opts ${AUTO_STARTUP} ${DEBUG_STARTUP} diff --git a/Online/FarmConfig/job/preamble.sh b/Online/FarmConfig/job/preamble.sh index aaf1ae7b5..c199d1077 100755 --- a/Online/FarmConfig/job/preamble.sh +++ b/Online/FarmConfig/job/preamble.sh @@ -9,7 +9,12 @@ export Class0_task="$gaudi_task -tasktype=LHCb::Class0Task " export Class1_task="$gaudi_task -tasktype=LHCb::Class1Task -main=$OPTS/Main.opts " export Class2_task="$gaudi_task -tasktype=LHCb::Class2Task -main=$OPTS/Main.opts " export Checkpoint_task="GaudiCheckpoint.exe libGaudiOnline.so OnlineTask -msgsvc=$msg_svc -tasktype=LHCb::Class1Task -main=$OPTS/Main.opts " -export DATAFLOW_task="gentest.exe libDataflow.so dataflow_run_task -msg=Dataflow_FmcLogger -mon=Dataflow_DIMMonitoring"; +# +if test -p "${LOGFIFO}"; then + export DATAFLOW_task="gentest.exe libDataflow.so dataflow_run_task -msg=fifo -mon=Dataflow_DIMMonitoring"; +else + export DATAFLOW_task="gentest.exe libDataflow.so dataflow_run_task -msg=Dataflow_FmcLogger -mon=Dataflow_DIMMonitoring"; +fi; # # # diff --git a/Online/FarmConfig/job/runTask.sh b/Online/FarmConfig/job/runTask.sh index a0f94c0ef..c095c52ee 100755 --- a/Online/FarmConfig/job/runTask.sh +++ b/Online/FarmConfig/job/runTask.sh @@ -26,11 +26,7 @@ cd $FARMCONFIGROOT/job; export SUBFARM=`echo ${DIM_DNS_NODE} | tr a-z A-Z`; export STATIC_OPTS=${FARMCONFIGROOT}/options; export NON_DYNAMIC_OPTS=/group/online/dataflow/options/${PARTITION_NAME}; -if test "${SUBFARM}" != "CALD07"; then - export DYNAMIC_OPTS=/run/options/${PARTITION_NAME}; -else - export DYNAMIC_OPTS=${NON_DYNAMIC_OPTS}; -fi; +export DYNAMIC_OPTS=/group/online/dataflow/options/${PARTITION_NAME}; if test -z "${DATAINTERFACE}"; then DATAINTERFACE=`python /group/online/dataflow/scripts/getDataInterface.py`; fi; @@ -39,14 +35,12 @@ export ONLINETASKS=/group/online/dataflow/templates; export INFO_OPTIONS=${DYNAMIC_OPTS}/${PARTITION_NAME}_Info.opts; export STORAGE_INFO_OPTIONS=${DYNAMIC_OPTS}/StorageEnv.opts; export SUBFARM_OPTIONS=${DYNAMIC_OPTS}/${PARTITION_NAME}_${SUBFARM}_HLT.opts; -##echo "[ERROR] SUBFARM_OPTIONS=${DYNAMIC_OPTS}/${PARTITION_NAME}_${DIM_DNS_NODE}_HLT.opts"; +# +# echo "[ERROR] INFO_OPTIONS=${INFO_OPTIONS}"; +# echo "[ERROR] SUBFARM_OPTIONS=${DYNAMIC_OPTS}/${PARTITION_NAME}_${DIM_DNS_NODE}_HLT.opts"; # . ./preamble.sh; # -# Test setup: -##if test "${PARTITION_NAME}" = "LHCb"; then -## export LD_LIBRARY_PATH=/home/frankm/cmtuser/OnlineDev_v6r5/InstallArea/${CMTCONFIG}/lib:${LD_LIBRARY_PATH}; -##fi; if test -f ./${TASK_TYPE}.sh; then # echo "RunTask: TASK_TYPE: ${TASK_TYPE} ./${TASK_TYPE}.sh" diff --git a/Online/FarmConfig/options/EBMBM.opts b/Online/FarmConfig/options/EBMBM.opts index 2ce8dabe5..6297f8c50 100644 --- a/Online/FarmConfig/options/EBMBM.opts +++ b/Online/FarmConfig/options/EBMBM.opts @@ -1,2 +1,9 @@ -#include "$FARMCONFIG_OPTIONS/MEPInit.opts" -MEPManager.InitFlags = "-s=200000 -e=500 -u=20 -b=17 -f -i=Events_0 -c -s=200000 -e=500 -u=20 -b=17 -f -i=Events_1 -c"; +#pragma print off +#include "$INFO_OPTIONS" +#include "$MBM_SETUP_OPTIONS" +#include "$FARMCONFIG_OPTIONS/Logging.opts" +Manager.Setup = {"Dataflow_MBMServer/MEPManager"}; +Manager.Services = {"Dataflow_UI/UI"}; +MEPManager.PartitionBuffers = @OnlineEnv.PartitionBuffers; +MEPManager.PartitionName = @OnlineEnv.PartitionName; +MEPManager.InitFlags = "-s=1000000 -e=20 -u=30 -b=12 -t=2 -f -i=Events_0 -c -s=1000000 -e=20 -u=30 -b=12 -f -i=Events_1 -c"; diff --git a/Online/SmiController/src/smi_controller.cpp b/Online/SmiController/src/smi_controller.cpp index 5f29a317c..6bf1b7af8 100644 --- a/Online/SmiController/src/smi_controller.cpp +++ b/Online/SmiController/src/smi_controller.cpp @@ -26,6 +26,7 @@ #include <cerrno> #include <climits> #include <unistd.h> +#include <sys/stat.h> using namespace std; using namespace FiniteStateMachine; @@ -89,27 +90,40 @@ extern "C" int smi_controller(int argc, char** argv) { auto logger = std::make_shared<RTL::Logger::LogDevice>(); RTL::CLI cli(argc, argv, help_ctrl); string logger_type = "term"; + char log_path[PATH_MAX]; int secs_sleep=0; bool dbg = false; - char log_path[PATH_MAX]; - ssize_t len = ::readlink("/proc/self/fd/1", log_path, sizeof(log_path)); - if ( -1 == len ) { - ::strerror_r(errno, log_path, sizeof(log_path)); - ::fprintf(stderr, "Failed to determine stdout file name. [%s]\n", log_path); - return EINVAL; + if ( ::getenv("LOGFIFO") ) { + struct stat stat; + string path = ::getenv("LOGFIFO"); + if ( 0 == ::stat(path.c_str(), &stat) ) { + if ( S_ISFIFO(stat.st_mode) ) { + config.stderr_file = path; + config.stdout_file = path; + } + } } - log_path[len] = 0; - config.stdout_file = log_path; - - len = ::readlink("/proc/self/fd/2", log_path, sizeof(log_path)); - if ( -1 == len ) { - ::strerror_r(errno, log_path, sizeof(log_path)); - ::fprintf(stderr, "Failed to determine stderr file name. [%s]\n", log_path); - return EINVAL; + + if ( config.stderr_file.empty() || config.stdout_file.empty() ) { + ssize_t len = ::readlink("/proc/self/fd/1", log_path, sizeof(log_path)); + if ( -1 == len ) { + ::strerror_r(errno, log_path, sizeof(log_path)); + ::fprintf(stderr, "Failed to determine stdout file name. [%s]\n", log_path); + return EINVAL; + } + log_path[len] = 0; + config.stdout_file = log_path; + + len = ::readlink("/proc/self/fd/2", log_path, sizeof(log_path)); + if ( -1 == len ) { + ::strerror_r(errno, log_path, sizeof(log_path)); + ::fprintf(stderr, "Failed to determine stderr file name. [%s]\n", log_path); + return EINVAL; + } + log_path[len] = 0; + config.stderr_file = log_path; } - log_path[len] = 0; - config.stderr_file = log_path; config.name = RTL::processName(); config.dns = ::getenv("DIM_DNS_NODE") ? ::getenv("DIM_DNS_NODE") : ""; diff --git a/Online/Storage/src/server/fdb_db_server.cpp b/Online/Storage/src/server/fdb_db_server.cpp index 67c94adc7..512d4128e 100644 --- a/Online/Storage/src/server/fdb_db_server.cpp +++ b/Online/Storage/src/server/fdb_db_server.cpp @@ -29,7 +29,7 @@ public: using namespace std; using namespace Online::storage; -static const std::string PATTERN_NEXT = "/next?prefix="; +static const string PATTERN_NEXT = "/next?prefix="; template <> http::HttpRequestHandler::continue_action http::basic_http_server<db>::handler_t::handle_get(const request_t& req, reply_t& rep) { @@ -51,8 +51,7 @@ http::basic_http_server<db>::handler_t::handle_get(const request_t& req, reply_t rep = reply_t::stock_reply(reply_t::not_found); if ( !error_code_ok(ec) ) { - header_t h(http::constants::error_cause, - "Failed to get "+req.uri+" ["+ec.message()+"]"); + header_t h(http::constants::error_cause,"Failed to get "+req.uri+" ["+ec.message()+"]"); ::lib_rtl_output(LIB_RTL_ERROR,"DELETE: %s %-20s = %s", reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); @@ -191,8 +190,9 @@ http::basic_http_server<db>::handler_t::handle_request(const request_t& req, rep if ( 0 == rep.bytes_sent ) { auto ret = this->HttpRequestHandler::handle_request(req, rep); if ( this->debug > 0 || !(rep.status == reply_t::temp_redirect || rep.status == reply_t::ok) ) { + string status = reply_t::stock_status(rep.status); ::lib_rtl_output(LIB_RTL_INFO,"basic_http_server<db>::handle: Serving %s %s status: %d [%s]", - req.method.c_str(), req.uri.c_str(), rep.status, reply_t::stock_status(rep.status)); + req.method.c_str(), req.uri.c_str(), rep.status, status.c_str()); for ( const auto& h : rep.headers ) ::lib_rtl_output(LIB_RTL_INFO,"basic_http_server<db>::handle: %-20s = %s", h.name.c_str(), h.value.c_str()); -- GitLab From dce3e3ae302bc513d2d42d44f60a07fab4528ebb Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Tue, 9 Mar 2021 17:52:52 +0100 Subject: [PATCH 04/26] Simplify options handling --- Online/EventBuilding/src/BU.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Online/EventBuilding/src/BU.cpp b/Online/EventBuilding/src/BU.cpp index 81a1b4ef8..61be24ba5 100644 --- a/Online/EventBuilding/src/BU.cpp +++ b/Online/EventBuilding/src/BU.cpp @@ -185,6 +185,11 @@ int EB::BU::check_buffer() logger.warning << __FUNCTION__ << " no buffer types provided setting to default value " << EB::default_BU_buffer_type << std::flush; } + else if (_buffer_type.size() == 1) { + _buffer_type.resize(_bu_ranks.size(), _buffer_type[0]); + logger.warning << __FUNCTION__ << " Single buffer type provided: " + << _buffer_type[0] << std::flush; + } if (_buffer_type.size() != _bu_ranks.size()) { logger.error << __FUNCTION__ << " Configuration error: not enough buffer types provided " << _buffer_type.size() @@ -198,6 +203,11 @@ int EB::BU::check_buffer() logger.warning << __FUNCTION__ << " no buffer sizes provided setting to default value " << EB::default_BU_buffer_size << std::flush; } + else if (_buffer_sizes.size() == 1) { + _buffer_sizes.resize(_bu_ranks.size(), _buffer_sizes[0]); + logger.warning << __FUNCTION__ << " Single buffer size provided: " + << _buffer_sizes[0] << std::flush; + } if (_buffer_sizes.size() != _bu_ranks.size()) { logger.error << __FUNCTION__ << " Configuration error: not enough buffer sizes provided " << _buffer_sizes.size() @@ -224,6 +234,11 @@ int EB::BU::check_buffer() logger.warning << __FUNCTION__ << " no MBM names provided setting to default value " << EB::default_MBM_name << std::flush; } + else if (_mbm_name.size() == 1) { + _mbm_name.resize(_bu_ranks.size(), _mbm_name[0]); + logger.warning << __FUNCTION__ << " single MBM name provided: " << _mbm_name[0] + << std::flush; + } if (_mbm_name.size() != _bu_ranks.size()) { logger.error << __FUNCTION__ << " Configuration error: not enough MBM names provided " << _mbm_name.size() << "/" -- GitLab From de652d8bda36f53b2b2d58f6ef92f74f944bddd5 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Mon, 15 Mar 2021 10:27:22 +0100 Subject: [PATCH 05/26] Fix storage library and servers --- Online/Dataflow/Dataflow/DiskReader.h | 14 +- Online/Dataflow/options/StorageReader.opts | 34 ++ Online/Dataflow/options/StorageWriter.opts | 2 +- Online/Dataflow/src/Storage/StorageReader.cpp | 42 ++- Online/Dataflow/src/Storage/StorageWriter.cpp | 46 ++- Online/Dataflow/src/Storage/StorageWriter.h | 10 +- Online/Dataflow/src/framework/DiskReader.cpp | 295 ++++++++++++------ Online/GaudiOnline/scripts/readDatafile | 1 + Online/GaudiOnline/src/EventAccess.cpp | 10 +- Online/GaudiOnline/src/MBMEventAccess.cpp | 52 +-- .../GaudiOnlineTests/options/CreateMEP.opts | 2 +- .../scripts/go_test_CreateMEP.sh | 4 +- .../tests/refs/go_create_mep.ref | 2 +- .../tests/refs/go_create_tae.ref | 2 +- Online/HTTP/HTTP/HttpServer.h | 2 + Online/HTTP/src/HttpServer.cpp | 68 ++-- Online/OnlineBase/src/LOG/FifoLog.cpp | 16 +- Online/PCIE40Data/PCIE40Data/RawBank40.h | 4 +- Online/PCIE40Data/PCIE40Data/pcie40.h | 5 +- .../scripts/TestStorageReader.sh | 14 + Online/Storage/Storage/client.h | 4 +- Online/Storage/Storage/client_async.h | 2 +- Online/Storage/Storage/client_sync.h | 2 +- Online/Storage/Storage/communication.h | 2 +- Online/Storage/Storage/fdb_client.h | 18 +- Online/Storage/Storage/fdb_server.h | 7 +- Online/Storage/src/client/client.cpp | 4 +- Online/Storage/src/client/client_async.cpp | 9 +- Online/Storage/src/client/client_sync.cpp | 9 +- Online/Storage/src/client/communication.cpp | 12 +- Online/Storage/src/client/fdb_client.cpp | 31 +- Online/Storage/src/server/fdb_SQLite.cpp | 44 ++- Online/Storage/src/server/fdb_SQLite_bind.cpp | 2 - Online/Storage/src/server/fdb_db_server.cpp | 23 +- Online/Storage/src/server/fdb_fs_server.cpp | 111 ++++--- Online/Storage/src/server/sqlite_test.cpp | 11 +- Online/Storage/tests/src/Setup.cpp | 4 +- Online/Storage/tests/src/Setup.h | 1 + Online/Storage/tests/src/fdb_cli.cpp | 36 ++- Online/Storage/tests/src/fdb_client_test.cpp | 4 +- Online/Tell1Data/Tell1Data/EventHeader.h | 81 +++-- Online/Tell1Data/Tell1Data/RawFile.h | 3 +- Online/Tell1Data/src/RawFile.cpp | 25 +- Online/Tell1Data/src/Tell1Decoder.cpp | 4 +- 44 files changed, 704 insertions(+), 370 deletions(-) create mode 100644 Online/Dataflow/options/StorageReader.opts create mode 100755 Online/SmiController/scripts/TestStorageReader.sh diff --git a/Online/Dataflow/Dataflow/DiskReader.h b/Online/Dataflow/Dataflow/DiskReader.h index 65c99017f..4da4be04c 100644 --- a/Online/Dataflow/Dataflow/DiskReader.h +++ b/Online/Dataflow/Dataflow/DiskReader.h @@ -228,16 +228,22 @@ namespace Online { /// Update run number service void updateRunNumber(int new_run); /// Wait until event buffers are empty before finishing.... - virtual void waitForPendingEvents(int seconds); + virtual void waitForPendingEvents(int seconds); + /// Patch the string array with allowed runs + virtual void checkAllowedRuns(); struct LoadResult { size_t length; int packing; RawFile::EventType type; }; - LoadResult load_data_in_mbm(RawFile::Allocator& allocator, RawFile& file); - int declare_mbm_data(MBM::BufferInfo* info, RawFile& file); - int handle_mbm_request(MBM::BufferInfo* info, RawFile& file); + + /// Load event data from file into the buffer manager + virtual LoadResult load_event_data(RawFile::Allocator& allocator, RawFile& file); + /// Declare event frame to the buffer manager to trigger clients + virtual int declare_mbm_data(MBM::BufferInfo* info, const char* fname, const LoadResult& load); + /// Handle MBM request: load event data and declare to MBM + virtual int handle_mbm_request(MBM::BufferInfo* info, RawFile& file); void save_file_rest(RawFile& file); diff --git a/Online/Dataflow/options/StorageReader.opts b/Online/Dataflow/options/StorageReader.opts new file mode 100644 index 000000000..1ab0bcc91 --- /dev/null +++ b/Online/Dataflow/options/StorageReader.opts @@ -0,0 +1,34 @@ +#pragma print off +#include "$INFO_OPTIONS" +#include "$MBM_SETUP_OPTIONS" +#include "$FARMCONFIGROOT/options/Logging.opts" +#include "$FARMCONFIGROOT/options/Monitoring.opts" +Manager.Services = {"Dataflow_MBMClient/MBM", + "Dataflow_StorageReader/Reader", + "Dataflow_RunableWrapper/Wrap" + }; +Manager.Runable = "Wrap"; +Wrap.Callable = "Reader"; + +Reader.Directories = {"/daqarea/lhcb/data/2016/RAW/FULL/LHCb/COLLISION16/188171"}; +Reader.AllowedRuns = {"0"}; +Reader.DeleteFiles = false; +Reader.SaveRest = false; +Reader.FilePrefix = ""; +Reader.PauseSleep = 1; +Reader.InitialSleep = 1; +Reader.MaxPauseWait = 1; +Reader.RequireConsumers = 0; +Reader.PackingFactor = 1000; +Reader.Buffer = "Events"; +Reader.Server = "XXEB09.lbdaq.cern.ch:8000"; +Reader.Mount = "/objects/data/testing"; + +MBM.PartitionBuffers = @OnlineEnv.PartitionBuffers; +MBM.PartitionName = @OnlineEnv.PartitionName; +MBM.PartitionID = @OnlineEnv.PartitionID; +MBM.Buffers = {"Events"}; + +#include "../options/Monitoring.opts" +Logger.OutputLevel = @OnlineEnv.OutputLevel; +// Logger.OutputLevel = 0; diff --git a/Online/Dataflow/options/StorageWriter.opts b/Online/Dataflow/options/StorageWriter.opts index b48a52f45..cd7003a10 100644 --- a/Online/Dataflow/options/StorageWriter.opts +++ b/Online/Dataflow/options/StorageWriter.opts @@ -28,7 +28,7 @@ Manager.Algorithms = {"Dataflow_StorageWriter/Writer"}; Writer.Server = "XXEB09.lbdaq.cern.ch:8000"; Writer.Mount = "/objects/data/testing"; Writer.FileSizeMB = 800; -Writer.FileSizeMB = 100; +Writer.FileSizeMB = 40; Writer.MinFileSizeMB = 10; Writer.WriteErrorRetry = 100000; Writer.WriteErrorSleep = 3000; diff --git a/Online/Dataflow/src/Storage/StorageReader.cpp b/Online/Dataflow/src/Storage/StorageReader.cpp index 20ab4eb82..8fa94d4ba 100644 --- a/Online/Dataflow/src/Storage/StorageReader.cpp +++ b/Online/Dataflow/src/Storage/StorageReader.cpp @@ -41,14 +41,21 @@ namespace Online { protected: friend class MBMAllocator; std::string m_server; + std::string m_mount; + /// Property: Debug FDB client + int m_debugClient; /// Runable implementation : Run the class implementation virtual int i_run() override; + /// Patch the string array with allowed runs + virtual void checkAllowedRuns() override {} public: /// Standard algorithm constructor StorageReader(const std::string& nam, Context& ctxt); - + /// Default destructor + virtual ~StorageReader() = default; + /// Callback to open new storage file int openFile(int& runno, std::vector<unsigned char>& buffer, RawFile& input); }; } // End namespace Online @@ -87,7 +94,9 @@ DECLARE_DATAFLOW_NAMED_COMPONENT_NS(Online,Dataflow_StorageReader,StorageReader) StorageReader::StorageReader(const string& nam, Context& ctxt) : DiskReader(nam, ctxt) { - declareProperty("Server", m_server); + declareProperty("Server", m_server); + declareProperty("Mount", m_mount = "/objects"); + declareProperty("DebugClient", m_debugClient = 0); } int StorageReader::openFile(int& runno, std::vector<unsigned char>& buffer, RawFile& input) { @@ -100,14 +109,26 @@ int StorageReader::openFile(int& runno, std::vector<unsigned char>& buffer, RawF runno = -1; input.reset(); buffer.clear(); - client.fdbclient = client::create<client::sync>(url.host, url.port, 10000); + client.fdbclient = client::create<client::sync>(url.host, url.port, 10000, m_debugClient); client.create_client = std::function(client::create<client::sync>); for(const auto& run : m_allowedRuns ) { - string obj = "/objects/Run_"+run; - auto reply = client.next_object_delete(obj, date, length); + char fname[PATH_MAX]; + const char* crun = run.c_str(); + const char* cmnt = m_mount.c_str(); + const char* pref = m_filePrefix.c_str(); + if ( run[0] == '*' ) + ::snprintf(fname, sizeof(fname), "%s/%s", cmnt, pref); + else if ( !::isdigit(run[0]) ) + ::snprintf(fname, sizeof(fname), "%s/%s%s", cmnt, pref, crun); + else if ( ::isdigit(crun[0]) ) { + long runno = ::strtol(crun, 0, 10); + ::snprintf(fname, sizeof(fname), "%s/%s%010ld", cmnt, pref, runno); + } + + auto reply = client.next_object_delete(fname, date, length); if ( reply.status == reply.ok ) { buffer = move(reply.content); - input.map(&buffer.at(0), buffer.size()); + input.map_memory(&buffer.at(0), buffer.size()); runno = ::atol(run.c_str()); return DF_SUCCESS; } @@ -135,7 +156,11 @@ int StorageReader::i_run() { ::lib_rtl_sleep(100); continue; } - while ( m_receiveEvts && GO_PROCESS == m_goValue ) { + else if ( !m_receiveEvts ) { + break; + } + while ( (m_receiveEvts && GO_PROCESS == m_goValue) || + (0 != current_input.data_size()) ) { if ( m_muDelay > 0 ) { ::usleep(m_muDelay); } @@ -181,9 +206,10 @@ int StorageReader::i_run() { } } } - current_input.reset(); + current_input.reset(true); // Bad file: Cannot read input (m_eventsIN==0) updateRunNumber(0); + } /// Before actually declaring PAUSED, we wait until no events are pending anymore. waitForPendingEvents(m_maxPauseWait); diff --git a/Online/Dataflow/src/Storage/StorageWriter.cpp b/Online/Dataflow/src/Storage/StorageWriter.cpp index 33b056d44..c8b61a392 100644 --- a/Online/Dataflow/src/Storage/StorageWriter.cpp +++ b/Online/Dataflow/src/Storage/StorageWriter.cpp @@ -53,6 +53,7 @@ StorageWriter::StorageWriter(const string& nam, Context& ctxt) declareProperty("NumThreads", m_num_threads = 1); declareProperty("MinFileSizeMB", m_minFileSizeMB = 3000); declareProperty("ForceMDF", m_force_mdf = 0); + declareProperty("DebugClient", m_debugClient = 0); } /// Default destructor @@ -67,7 +68,8 @@ int StorageWriter::initialize() { m_bufferSize *= 1024*1024; m_curr_run = 0; m_cancelled = 0; - // m_client.fdbclient = storage::client::create<storage::client::sync>(url.host, url.port, 10000); + // m_client.fdbclient = + // storage::client::create<storage::client::sync>(url.host, url.port, 10000, m_debugClient); // m_client.create_client = std::function(storage::client::create<storage::client::sync>); for(Buffer& b : m_free) ::free(b.buffer); @@ -167,7 +169,7 @@ StorageWriter::Buffer& StorageWriter::get_buffer() { void StorageWriter::flush_buffer(Buffer& buff) { if ( buff.buffer != nullptr ) { lock_guard<mutex> bufferLock(m_mutex); - m_todo.push_back(buff); + m_todo.emplace_back(buff); buff = {nullptr, nullptr}; } buff = m_shutdown ? Buffer() : get_buffer(); @@ -214,20 +216,22 @@ int StorageWriter::save_buffer(Buffer& buff, const void* data, long length) { int StorageWriter::save_pcie40_as_mdf(Buffer& buff, const uint8_t* start, long len) { auto* mep_start = (pcie40::mep_header_t*)start; auto* mep_end = pcie40::add_ptr<pcie40::mep_header_t>(start, len); - std::unique_ptr<pcie40::event_collection_t> ev; pcie40::decoder_t decoder; long nev = 0; for(const pcie40::mep_header_t *m, *mep = mep_start; mep < mep_end; ) { if ( mep->is_valid() ) { const pcie40::multi_fragment_t *mfp = mep->multi_fragment(0); + std::unique_ptr<pcie40::event_collection_t> ev; uint16_t packing = mfp->header.packing; nev += packing; decoder.decode(ev, mep); /// Save the events one by one to the buffer for( auto* e=ev->begin(); e != ev->end(); e=ev->next(e) ) { - size_t length = e->total_length(); - int sc = ensure_buffer(buff, length); + uint32_t hdrvsn = 3; + size_t hdrlen = EventHeader::sizeOf(hdrvsn); + size_t length = e->total_length(); + int sc = ensure_buffer(buff, length + hdrlen); if ( sc == DF_SUCCESS ) { if ( 0 == m_curr_run ) { const auto* odin = e->bank_collection(0)->at(0); @@ -238,9 +242,28 @@ int StorageWriter::save_pcie40_as_mdf(Buffer& buff, const uint8_t* start, long l m_curr_bunch = sodin->bunch_id(); } } + const uint8_t* b_beg = buff.pointer; + EventHeader* hdr = (EventHeader*)buff.pointer; + hdr->setChecksum(0); + hdr->setCompression(0); + hdr->setHeaderVersion(hdrvsn); + hdr->setSpare(0); + hdr->setDataType(EventHeader::BODY_TYPE_BANKS); + hdr->setSubheaderLength(sizeof(EventHeader::Header1)); + hdr->setSize(length); + EventHeader::SubHeader h = hdr->subHeader(); + h.H1->setTriggerMask(~0u,~0u,~0u,~0u); + h.H1->setRunNumber(m_curr_run); + h.H1->setOrbitNumber(m_curr_orbit); + h.H1->setBunchID(m_curr_bunch); + buff.pointer += hdrlen; for( size_t i=0, n=e->num_bank_collections(); i<n; ++i) { buff.pointer = (uint8_t*)e->bank_collection(i)->copy_data(buff.pointer); } + if ( buff.pointer-b_beg != hdr->recordSize() ) { + error("++ Event length inconsistency: %ld <> %ld %ld %ld", + buff.pointer-b_beg, hdr->recordSize(), length, hdrlen); + } } } /// Move to the next MEP if any @@ -309,18 +332,19 @@ int StorageWriter::execute(const Context::EventData& event) { m_shutdown ? "Shutdown requested" : "??????????"); return status; } + /// Auto detect data type: first check for PCIE40 MEP format auto* mep_hdr = (pcie40::mep_header_t*)start; - if( mep_hdr->magic == mep_hdr->MagicPattern ) { - status = m_force_mdf + if( mep_hdr->is_valid() ) { + status = this->m_force_mdf ? save_pcie40_as_mdf(buff, start, mep_hdr->size*sizeof(uint32_t)) : save_buffer (buff, start, mep_hdr->size*sizeof(uint32_t)); return status; } + /// Auto detect data type: first check for MDF/BURST format auto* mdf_hdr = (EventHeader*)start; - if ( mdf_hdr->size0() == mdf_hdr->size1() && - mdf_hdr->size0() == mdf_hdr->size2() ) { + if ( mdf_hdr->is_mdf() ) { status = save_mdf_buffer(buff, start, len); return status; } @@ -423,8 +447,8 @@ int StorageWriter::writeBuffer(const Buffer& buff) { string url; client_t cl; storage::uri_t srv(m_server); - cl.fdbclient = storage::client::create<storage::client::sync>(srv.host, srv.port, 10000); - cl.create_client = std::function(storage::client::create<storage::client::sync>); + cl.fdbclient = + storage::client::create<storage::client::sync>(srv.host, srv.port, 10000, m_debugClient); auto reply = cl.save_object_record(fname, url, len); if ( reply.status == reply.temp_redirect ) { /// OK. the registration was now successful. Send the data. diff --git a/Online/Dataflow/src/Storage/StorageWriter.h b/Online/Dataflow/src/Storage/StorageWriter.h index bffe85a36..e5322a67a 100644 --- a/Online/Dataflow/src/Storage/StorageWriter.h +++ b/Online/Dataflow/src/Storage/StorageWriter.h @@ -73,12 +73,14 @@ namespace Online { /// Property: Number of retries when write connection fails int m_write_error_retry; /// Property: Poll timeout to detect transfer buffers [microseconds] - int m_poll_tmo {100}; + int m_poll_tmo {100}; /// Property: Cancel timeout to empty pending buffers - int m_cancel_tmo {100}; + int m_cancel_tmo {100}; /// Property: Force output in MDF format - int m_force_mdf {0}; - + int m_force_mdf {0}; + /// Property: Debug FDB client + int m_debugClient {0}; + /// Buffer handling thread std::vector<thread_t> m_threads; /// Mutex to lock the event buffer when filling/saving diff --git a/Online/Dataflow/src/framework/DiskReader.cpp b/Online/Dataflow/src/framework/DiskReader.cpp index cd12af3ad..cb1295159 100644 --- a/Online/Dataflow/src/framework/DiskReader.cpp +++ b/Online/Dataflow/src/framework/DiskReader.cpp @@ -16,21 +16,22 @@ //========================================================================== // Framework includes -#include "Dataflow/DiskReader.h" -#include "Dataflow/DataflowTask.h" -#include "Dataflow/ControlPlug.h" -#include "Dataflow/MBMClient.h" -#include "Dataflow/Incidents.h" -#include "Dataflow/Plugins.h" -#include "Tell1Data/Tell1Decoder.h" -#include "RTL/rtl.h" -#include "RTL/readdir.h" +#include <Dataflow/DiskReader.h> +#include <Dataflow/DataflowTask.h> +#include <Dataflow/ControlPlug.h> +#include <Dataflow/MBMClient.h> +#include <Dataflow/Incidents.h> +#include <Dataflow/Plugins.h> +#include <Tell1Data/Tell1Decoder.h> +#include <PCIE40Data/pcie40.h> +#include <RTL/rtl.h> +#include <RTL/readdir.h> #include <fcntl.h> #include <cerrno> #include <cstring> #include <sstream> -#include "dim/dic.h" -#include "dim/dis.h" +#include <dim/dic.h> +#include <dim/dis.h> #ifdef _WIN32 #include <io.h> #else @@ -199,17 +200,7 @@ int DiskReader::initialize() { m_mbmInfo[nam] = &m_mbmInfos[i]; m_mbmInfos[i].attach(bm_name.c_str()); } - // Patch the string array with allowed runs - for(size_t i=0; i<m_allowedRuns.size(); ++i) { - const char* crun = m_allowedRuns[i].c_str(); - if ( ::isdigit(crun[0]) && crun[0] != '*' ) { - char text[PATH_MAX]; - long runno = ::strtol(crun,0,10); - ::snprintf(text,sizeof(text),"%s%07ld_",m_filePrefix.c_str(),runno); - m_allowedRuns[i] = text; - info("Add run %s to allowed run-list.",m_allowedRuns[i].c_str()); - } - } + checkAllowedRuns(); return sc; } @@ -328,6 +319,21 @@ int DiskReader::cancel() { return DF_SUCCESS; } + +/// Patch the string array with allowed runs +void DiskReader::checkAllowedRuns() { + for(size_t i=0; i<m_allowedRuns.size(); ++i) { + const char* crun = m_allowedRuns[i].c_str(); + if ( ::isdigit(crun[0]) && crun[0] != '*' ) { + char text[PATH_MAX]; + long runno = ::strtol(crun,0,10); + ::snprintf(text,sizeof(text),"%s%07ld_",m_filePrefix.c_str(),runno); + m_allowedRuns[i] = text; + info("Add run %s to allowed run-list.",m_allowedRuns[i].c_str()); + } + } +} + void DiskReader::waitForPendingEvents(int seconds) { size_t count; do { @@ -411,58 +417,151 @@ void DiskReader::updateRunNumber(int new_run) { void DiskReader::save_file_rest(RawFile& file) { if ( file.isOpen() ) { if ( m_deleteFiles && m_saveRest ) file.saveRestOfFile(); - file.reset(); + file.reset(true); } } +/// Load event data from file into the buffer manager DiskReader::LoadResult -DiskReader::load_data_in_mbm(RawFile::Allocator& allocator, RawFile& file) { - off_t file_position = file.position(); - RawFile::EventType type_found; - RawFile::EventType type_in = RawFile::EventType(m_inputType); - int packing_factor = -1; - long length = 0; - if ( m_packingFactor == 1 ) { - length = file.read_event(type_in, type_found, allocator, 0); - packing_factor = 1; - } - else { - unsigned char* data = allocator(m_preAllocSize); - length = 0; - if ( data ) { - auto ret = file.read_multiple_events(m_packingFactor, type_in, - type_found, data, m_preAllocSize); - length = ret.first > 0 ? ret.second : 0; - packing_factor = ret.first; - } - } - if ( length < 0 ) { - file.reset(); +DiskReader::load_event_data(RawFile::Allocator& allocator, RawFile& file) { + int size[3]; + uint8_t *data_ptr, *alloc_ptr; + off_t position = file.position(); + static constexpr int peek = int(3*sizeof(int)); + int status = file.read(size, peek); + + if ( status < peek ) { + file.reset(true); return {0, -1, RawFile::NO_INPUT_TYPE}; } - else if ( length == 0 ) { // MBM Error - // Set back the file position to the beginning of the event and continue. - // If there was a cancel in between, the file shall be saved. - file.position(file_position); - return {0, -1, RawFile::NO_INPUT_TYPE}; + // + // Auto detect data type: first check for PCIE40 MEP format + auto* mep_hdr = (pcie40::mep_header_t*)size; + if ( mep_hdr->magic == mep_hdr->MagicPattern ) { + int data_size = mep_hdr->size*sizeof(uint32_t); + data_ptr = allocator(data_size); + ::memcpy(data_ptr, size, peek); + status = file.read(data_ptr+peek, data_size-peek); + if ( status < int(data_size-peek) ) { + file.reset(true); + return {0, -1, RawFile::NO_INPUT_TYPE}; + } + return {size_t(data_size), 1, RawFile::MEP_INPUT_TYPE}; + } + // + // Now check for MDF format: 3 times same size + auto* mdf_hdr = (EventHeader*)size; + if ( mdf_hdr->is_mdf() ) { + int num_evts = 1; + int data_size = size[0]; + int alloc_size = m_packingFactor == 1 ? mdf_hdr->recordSize() : m_preAllocSize; + + if ( data_size > alloc_size ) { + file.position(position); + return {0, -1, RawFile::NO_INPUT_TYPE}; + } + alloc_ptr = data_ptr = allocator(alloc_size); + ::memcpy(data_ptr, mdf_hdr, peek); + status = file.read(data_ptr + peek, mdf_hdr->recordSize() - peek); + if ( status < mdf_hdr->recordSize() - peek ) { + file.reset(true); + return {0, -1, RawFile::NO_INPUT_TYPE}; + } + data_ptr += size[0]; + for ( ; num_evts < m_packingFactor; ) { + if ( peek + data_size > alloc_size ) { + file.reset(true); + return {size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE}; + } + position = file.position(); + status = file.read(data_ptr, peek); + if ( status < peek ) { + file.reset(true); + return {size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE}; + } + mep_hdr = (pcie40::mep_header_t*)size; + if ( mep_hdr->magic == mep_hdr->MagicPattern ) { + file.position(position); + return {size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE}; + } + mdf_hdr = (EventHeader*)data_ptr; + if ( mdf_hdr->recordSize() + data_size > alloc_size ) { + file.position(position); + return {size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE}; + } + status = file.read(data_ptr + peek, mdf_hdr->recordSize() - peek); + if ( status < mdf_hdr->recordSize() - peek ) { + file.reset(true); + return {size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE}; + } + data_size += mdf_hdr->recordSize(); + data_ptr += mdf_hdr->recordSize(); + ++num_evts; + } + return {size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE}; } - return {size_t(length), packing_factor, type_found}; + // + // Now the only thing left is Tell1 MEP format + { + static int id = -1; + int data_size = sizeof(MEPEVENT)+sizeof(Tell1MEP)+size[0]; + MEPEVENT* e = (MEPEVENT*)(alloc_ptr=allocator(data_size)); + Tell1MEP* me = (Tell1MEP*)e->data; + me->setSize(size[0]); + e->refCount = 1; + e->evID = ++id; + e->begin = 0; + e->packing = -1; + e->valid = 1; + e->magic = mep_magic_pattern(); + for (size_t j = 0; j < MEP_MAX_PACKING; ++j) { + e->events[j].begin = 0; + e->events[j].evID = 0; + e->events[j].status = EVENT_TYPE_OK; + e->events[j].signal = 0; + } + data_ptr = (unsigned char*)me->start(); + ::memcpy(data_ptr, size+1, 2*sizeof(size[0])); + status = file.read(data_ptr + 2*sizeof(size[0]), me->size() - 2*sizeof(int)); + if ( status < int(me->size() - 2*sizeof(int)) ) { // End-of file, continue with next file + file.reset(true); + return {0, -1, RawFile::NO_INPUT_TYPE}; + } + data_ptr += me->size(); + return {size_t(data_ptr-alloc_ptr), 1, RawFile::MEP_INPUT_TYPE}; + } + // + // Last chance: we have a corrrupted MDF record. Try to sweep to the next MDF frame + long skip = file.scanToNextMDF(); + if ( skip > 0 ) { // Just move on to next record! + warning("Corrupted file found: %s at position %ld. " + "Skip %ld bytes after sweep to next MDF record.", + file.cname(), position, skip); + return this->load_event_data(allocator, file); + } + error("Corrupted file found: %s at position %ld. Skip rest of file!", + file.cname(), position); + file.reset(true); + return {0, -1, RawFile::NO_INPUT_TYPE}; } -int DiskReader::declare_mbm_data(MBM::BufferInfo* mbm_info, RawFile& file) { - int status, partid = context.mbm->partitionID(); - MBM::EventDesc& dsc = m_producer->event(); +/// Declare event frame to the buffer manager to trigger clients +int DiskReader::declare_mbm_data(MBM::BufferInfo* mbm_info, const char* fname, const LoadResult& load) { + int pid = context.mbm->partitionID(); + MBM::EventDesc& dsc = this->m_producer->event(); EventHeader* hdr = (EventHeader*)dsc.data; - if ( hdr->size0() == hdr->size1() && hdr->size0() == hdr->size2() ) { + + dsc.len = load.length; + if ( hdr->is_mdf() ) { // If the first bank is a MDF header, we copy the real trigger mask - ::memcpy(dsc.mask,&hdr->subHeader().H1->triggerMask()[0],sizeof(dsc.mask)); context.manager.setRunNumber(hdr->subHeader().H1->runNumber()); - dsc.type = (m_packingFactor == 1) ? EVENT_TYPE_EVENT : EVENT_TYPE_BURST; + ::memcpy(dsc.mask, &hdr->subHeader().H1->triggerMask()[0], sizeof(dsc.mask)); + dsc.type = (load.packing == 1 && m_packingFactor) ? EVENT_TYPE_EVENT : EVENT_TYPE_BURST; } else { // If MEP, we emulate a trigger mask with the partition ID dsc.type = EVENT_TYPE_MEP; - dsc.mask[0] = partid; + dsc.mask[0] = pid; dsc.mask[1] = ~0x0; dsc.mask[2] = ~0x0; dsc.mask[3] = ~0x0; @@ -471,30 +570,32 @@ int DiskReader::declare_mbm_data(MBM::BufferInfo* mbm_info, RawFile& file) { // This should be a rare case, since there ARE (were?) consumers. // Though: In this case the event is really lost! // But what can I do... - int cons_wait = waitForConsumers(mbm_info); + int cons_wait = this->waitForConsumers(mbm_info); if ( cons_wait != DF_SUCCESS ) { - error("No consumers (partition:%d event type:%d) present", partid, dsc.type); - error("Save rest of file and finish. Skipping rest of file: %s", file.cname()); + error("No consumers (partition:%d event type:%d) present", pid, dsc.type); + error("Save rest of file and finish. Skipping rest of file: %s", fname); // If we did not see an MDF header, the file may also be corrupted: if ( dsc.type == EVENT_TYPE_MEP ) { - error("If Moores are alive, this file may be corrupted: %s", file.cname()); + error("If Moores are alive, this file may be corrupted: %s", fname); } return DF_ERROR; } + debug("++ Declared MBM data %p -> %p [%ld bytes]", dsc.data, ((char*)dsc.data)+dsc.len, dsc.len); + int status = MBM_ERROR; { lock_guard<mutex> lck(m_prodLock); // // Declare the event to the buffer manager // try { - status = m_producer->declareEvent(); + status = this->m_producer->declareEvent(); } catch (const exception& e) { - info("Exception while delareEvent: %s File:%s.", e.what(), file.cname()); + info("Exception while delareEvent: %s File:%s.", e.what(), fname); status = MBM_ERROR; } catch(...) { - info("UNKNOWN Exception while delareEvent: File:%s.", file.cname()); + info("UNKNOWN Exception while delareEvent: File:%s.", fname); status = MBM_ERROR; } if (status != MBM_NORMAL) { @@ -504,14 +605,14 @@ int DiskReader::declare_mbm_data(MBM::BufferInfo* mbm_info, RawFile& file) { // Now send space // try { - status = m_producer->sendSpace(); + status = this->m_producer->sendSpace(); } catch (const exception& e) { - info("Exception while sendSpace: %s File:%s.", e.what(), file.cname()); + info("Exception while sendSpace: %s File:%s.", e.what(), fname); status = MBM_ERROR; } catch(...) { - info("UNKNOWN Exception while sendSpace: File:%s.", file.cname()); + info("UNKNOWN Exception while sendSpace: File:%s.", fname); status = MBM_ERROR; } } @@ -521,45 +622,41 @@ int DiskReader::declare_mbm_data(MBM::BufferInfo* mbm_info, RawFile& file) { return DF_SUCCESS; } +/// Handle MBM request: load event data and declare to MBM int DiskReader::handle_mbm_request(MBM::BufferInfo* mbmInfo, RawFile& input) { - MBMAllocator allocator(this); - MBM::EventDesc& dsc = m_producer->event(); - off_t file_pos = input.position(); - auto load = load_data_in_mbm(allocator, input); - auto type_found = load.type; - auto packing = load.packing; - dsc.len = load.length; - if ( packing <= 0 && m_reuse_file ) { - debug("Re-use input file: %s",input.cname()); - input.seek(0, SEEK_SET); - return DF_CONTINUE;; - } - else if ( packing <= 0 ) { - input.reset(); + MBMAllocator allocator(this); + off_t pos = input.position(); + auto load = this->load_event_data(allocator, input); + + if ( load.packing <= 0 ) { + ( m_reuse_file ) ? input.seek(0, SEEK_SET) : input.reset(true); return DF_CONTINUE; } - m_eventsIN += packing; - m_inputType = type_found; + m_eventsIN += load.packing; + m_inputType = load.type; + + int status = this->declare_mbm_data(mbmInfo, input.cname(), load); + switch( status ) { + case DF_CONTINUE: + return DF_CONTINUE; - int status = declare_mbm_data(mbmInfo, input); - if ( status == DF_SUCCESS ) { - m_eventsOUT += packing; - m_evtCountFile += packing; + case DF_SUCCESS: + m_eventsOUT += load.packing; + m_evtCountFile += load.packing; // If we have exceeded the total number of events per file, close it! - if ( m_max_events_per_file>0 && m_evtCountFile>m_max_events_per_file ) { - input.reset(); - } + if ( m_max_events_per_file > 0 && m_evtCountFile > m_max_events_per_file ) + input.reset(true); return DF_SUCCESS; - } - else if ( status == DF_CONTINUE ) { - return DF_CONTINUE;; - } - else if ( status == DF_ERROR ) { - input.position(file_pos); - save_file_rest(input); + + case DF_ERROR: + input.position(pos); + this->save_file_rest(input); /// Before actually declaring ERROR, we wait until no events are pending anymore. - waitForPendingEvents(m_maxConsWait); + this->waitForPendingEvents(m_maxConsWait); return DF_ERROR; + + default: + break; } return DF_SUCCESS; } diff --git a/Online/GaudiOnline/scripts/readDatafile b/Online/GaudiOnline/scripts/readDatafile index 334910292..af1eb9f7d 100755 --- a/Online/GaudiOnline/scripts/readDatafile +++ b/Online/GaudiOnline/scripts/readDatafile @@ -9,6 +9,7 @@ import GaudiOnline GaudiOnline.readDatafile(['${1}']) END-OF-OPTS # +# gdb --args `which python` `which gaudirun.py` /tmp/${USER}_ReadMDF.py --application=Online::OnlineEventApp rm /tmp/${USER}_ReadMDF.py; } diff --git a/Online/GaudiOnline/src/EventAccess.cpp b/Online/GaudiOnline/src/EventAccess.cpp index 7ed12c4ef..4d095c1b5 100644 --- a/Online/GaudiOnline/src/EventAccess.cpp +++ b/Online/GaudiOnline/src/EventAccess.cpp @@ -58,10 +58,12 @@ namespace { } #endif template <typename BANK_TYPE> inline bool checkBank(const BANK_TYPE* bank) { - return bank->magic() == BANK_TYPE::MagicPattern && - bank->size() >= 0 && - bank->type() >= 0 && - bank->type() < Tell1Bank::LastType; + return + // Check for Tell1 bank magic word or PCIE40 magic word + (bank->magic() == BANK_TYPE::MagicPattern || bank->magic() == 0xFACE) && + (bank->size() >= 0) && + (bank->type() >= 0) && + (bank->type() < Tell1Bank::LastType); } inline bool checkHeader(const EventHeader* header) { diff --git a/Online/GaudiOnline/src/MBMEventAccess.cpp b/Online/GaudiOnline/src/MBMEventAccess.cpp index 84ea60b74..9a1eb323d 100644 --- a/Online/GaudiOnline/src/MBMEventAccess.cpp +++ b/Online/GaudiOnline/src/MBMEventAccess.cpp @@ -147,38 +147,44 @@ int MBMEventAccess::fill_cache() { } sc = ::mbm_get_event(bmid, &ev_data, &ev_len, &ev_type, trmask, config.partitionID); if ( sc == MBM_NORMAL ) { - shared_guard_t burst; + union _data { + int* mbm; + EventHeader* tell1; + pcie40::mep_header_t* tell40; + datapointer_t pointer; + _data(int* p) { mbm = p; } + } data(ev_data); switch(ev_type) { case EVENT_TYPE_EVENT: - case EVENT_TYPE_BURST: { - auto events = make_unique<event_collection_t>(convertMultiMDF(datapointer_t(ev_data), size_t(ev_len))); - burst = make_shared<mbm_guard_t<event_collection_t> >(move(events), event_traits::tell1::data_type, this, bmid); - break; - } + case EVENT_TYPE_BURST: case EVENT_TYPE_MEP: { - auto events = convertPCIE40MEP(datapointer_t(ev_data), size_t(ev_len)); - burst = make_shared<mbm_guard_t<pcie40::event_collection_t> >(move(events), event_traits::tell40::data_type, this, bmid); + shared_guard_t burst; + if( data.tell40->is_valid() ) { + auto evts = convertPCIE40MEP(data.pointer, size_t(ev_len)); + burst = make_shared<mbm_guard_t<pcie40::event_collection_t> >(move(evts), event_traits::tell40::data_type, this, bmid); + } + else if ( data.tell1->is_mdf() ) { + auto evts = make_unique<event_collection_t>(convertMultiMDF(data.pointer, size_t(ev_len))); + burst = make_shared<mbm_guard_t<event_collection_t> >(move(evts), event_traits::tell1::data_type, this, bmid); + } + else { + m_logger->error("Execute: Cannot determine event type. Drop event buffer."); + break; + } + if ( burst.get() && !burst->empty() ) { + queueBurst(move(burst)); + printInfo(); + return ONLINE_OK; + } + m_logger->error("MBM: Empty burst encountered."); break; } default: m_logger->error("MBM: Unknown MBM event type: %d", ev_type); - ::mbm_free_event(bmid); - requeueConsumer(bmid); - { - lock_t lock(m_burstLock); - ++monitor.burstsRelease; - } - return ONLINE_OK; - } - if ( !burst->empty() ) { - queueBurst(move(burst)); - printInfo(); - return ONLINE_OK; + break; } - m_logger->error("MBM: Empty burst encountered."); ::mbm_free_event(bmid); - requeueConsumer(bmid); - { + requeueConsumer(bmid); { lock_t lock(m_burstLock); ++monitor.burstsRelease; } diff --git a/Online/GaudiOnlineTests/options/CreateMEP.opts b/Online/GaudiOnlineTests/options/CreateMEP.opts index 74dd191a2..493752455 100755 --- a/Online/GaudiOnlineTests/options/CreateMEP.opts +++ b/Online/GaudiOnlineTests/options/CreateMEP.opts @@ -1,4 +1,4 @@ -ApplicationMgr.DLLs = { "MDF" }; +ApplicationMgr.Dlls = { "MDF" }; ApplicationMgr.SvcOptMapping += { "Gaudi::MultiFileCatalog/FileCatalog", "Gaudi::IODataManager/IODataManager" }; ApplicationMgr.TopAlg = { "Online::RawEventTestCreator/RawEventGen", "StoreExplorerAlg" }; ApplicationMgr.EvtMax = 2000; diff --git a/Online/GaudiOnlineTests/scripts/go_test_CreateMEP.sh b/Online/GaudiOnlineTests/scripts/go_test_CreateMEP.sh index 1e8f55759..8a5ad8339 100755 --- a/Online/GaudiOnlineTests/scripts/go_test_CreateMEP.sh +++ b/Online/GaudiOnlineTests/scripts/go_test_CreateMEP.sh @@ -41,4 +41,6 @@ END-OF-OPTS # cat ${opts_file}; `which gentest.exe` libGaudiKernel.so GaudiMain ${opts_file}; -rm ${opts_file}; +if test -f ${opts_file}; then + rm ${opts_file}; +fi; diff --git a/Online/GaudiOnlineTests/tests/refs/go_create_mep.ref b/Online/GaudiOnlineTests/tests/refs/go_create_mep.ref index 76529ec8c..08484b5bb 100644 --- a/Online/GaudiOnlineTests/tests/refs/go_create_mep.ref +++ b/Online/GaudiOnlineTests/tests/refs/go_create_mep.ref @@ -11,7 +11,7 @@ JobOptionsSvc INFO //GP:================================================================================ //GP: include "/tmp/CreateMEP.opts" (0,0) //GP: include "/afs/cern.ch/lhcb/software/releases/LHCB/LHCB_v26r2/DAQ/MDF/options/CreateMEP.opts" (1,10) -ApplicationMgr.DLLs = [ "MDF" ] ; //GP: (1,1) +ApplicationMgr.Dlls = [ "MDF" ] ; //GP: (1,1) ApplicationMgr.SvcOptMapping += [ "Gaudi::MultiFileCatalog/FileCatalog" , "Gaudi::IODataManager/IODataManager" ] ;//GP: (1,1) ApplicationMgr.TopAlg = [ "LHCb::RawEventTestCreator/RawEventGen" , "StoreExplorerAlg" ] ;//GP: (1,1) ApplicationMgr.EvtMax = 2000; //GP: (1,1) diff --git a/Online/GaudiOnlineTests/tests/refs/go_create_tae.ref b/Online/GaudiOnlineTests/tests/refs/go_create_tae.ref index 1b37663fd..0d2b59e26 100644 --- a/Online/GaudiOnlineTests/tests/refs/go_create_tae.ref +++ b/Online/GaudiOnlineTests/tests/refs/go_create_tae.ref @@ -12,7 +12,7 @@ JobOptionsSvc INFO //GP:================================================================================ //GP: include "/tmp/CreateMEP.opts" (0,0) //GP: include "/afs/cern.ch/lhcb/software/releases/LHCB/LHCB_v26r2/DAQ/MDF/options/CreateMEP.opts" (1,10) -ApplicationMgr.DLLs = [ "MDF" ] ; //GP: (1,1) +ApplicationMgr.Dlls = [ "MDF" ] ; //GP: (1,1) ApplicationMgr.SvcOptMapping += [ "Gaudi::MultiFileCatalog/FileCatalog" , "Gaudi::IODataManager/IODataManager" ] ;//GP: (1,1) ApplicationMgr.TopAlg = [ "LHCb::RawEventTestCreator/RawEventGen" , "StoreExplorerAlg" ] ;//GP: (1,1) ApplicationMgr.EvtMax = 2000; //GP: (1,1) diff --git a/Online/HTTP/HTTP/HttpServer.h b/Online/HTTP/HTTP/HttpServer.h index 4bca6d92a..196877edf 100644 --- a/Online/HTTP/HTTP/HttpServer.h +++ b/Online/HTTP/HTTP/HttpServer.h @@ -343,6 +343,8 @@ namespace http { virtual continue_action handle_update(const HttpRequest& req, HttpReply& rep); /// Handle a request and produce a reply. virtual continue_action handle_request(const HttpRequest& req, HttpReply& rep); + /// Handle possible statistics summaries. + virtual void handle_stat(const HttpRequest& req, HttpReply& rep); }; /// Basic HTTP server interface implementation diff --git a/Online/HTTP/src/HttpServer.cpp b/Online/HTTP/src/HttpServer.cpp index 366d5e97c..cb5bdd316 100644 --- a/Online/HTTP/src/HttpServer.cpp +++ b/Online/HTTP/src/HttpServer.cpp @@ -96,6 +96,10 @@ void HttpConnection::stop() { if ( ec ) { std::cout << "HttpConnection: STOP: " << ec.message() << std::endl << std::flush; } + auto now = system_clock::now(); + request.netio += std::chrono::duration_cast<milliseconds>(now-request.start); + request.start = now; + handler.handle_stat(request, reply); } if ( handler.debug ) { std::cout << "HttpConnection: STOP." << std::endl << std::flush; @@ -137,7 +141,22 @@ void HttpConnection::do_next(int next_action) { do_write(); } else if ( next_action == HttpRequestHandler::close ) { + // We do not shurdown immediately. + // When the connection gets closed by the client, + // shutdown will be Invoked automatically. do_shutdown(); + auto self(shared_from_this()); + socket.async_read_some(asio::buffer(buffer), + [this, self](system::error_code ec, std::size_t bytes) { + if ( handler.debug ) { + std::cout << "HttpConnection: do_write: STOP " + << bytes << " bytes. error code[" + << ec.value() << "]: " << ec.message().c_str() + << std::endl << std::flush; + } + manager.stop(shared_from_this()); + if ( self.get() ) {} + }); } else { do_shutdown(); @@ -147,8 +166,8 @@ void HttpConnection::do_next(int next_action) { /// Perform an graceful shutdown of the connection void HttpConnection::do_shutdown() { system::error_code ignored_ec; - //socket.shutdown(asio::ip::tcp::socket::shutdown_send,ignored_ec); - socket.shutdown(asio::ip::tcp::socket::shutdown_both,ignored_ec); + socket.shutdown(asio::ip::tcp::socket::shutdown_send,ignored_ec); + //socket.shutdown(asio::ip::tcp::socket::shutdown_both,ignored_ec); if ( handler.debug ) { std::cout << "HttpConnection: SHUTDOWN." << std::endl << std::flush; } @@ -330,32 +349,25 @@ void HttpConnection::do_write() { next = handler.handle_request(request, reply); now = system_clock::now(); request.handling += std::chrono::duration_cast<milliseconds>(now-request.start); - request.start = now; + request.start = now; do_next(next); } - if ( ec != asio::error::operation_aborted ) { - switch(next) { - case HttpRequestHandler::read: - return; - case HttpRequestHandler::write: - return; - default: - // We know that the transaction is over. Trigger an additional receive - // which will fail, but which is used to close the connection. - socket.async_read_some(asio::buffer(buffer), - [this, self](system::error_code ec, std::size_t bytes) { - if ( handler.debug ) { - std::cout << "HttpConnection: do_write: STOP " - << bytes << " bytes. error code[" - << ec.value() << "]: " << ec.message().c_str() - << std::endl << std::flush; - } - manager.stop(shared_from_this()); - if ( self.get() ) {} - }); - if ( self.get() ) {} - return; - } + else /* if ( ec != asio::error::operation_aborted ) */ { + // We know that the transaction is over. Trigger an additional receive + // which will fail, but which is used to close the connection. + socket.async_read_some(asio::buffer(buffer), + [this, self](system::error_code ec, std::size_t bytes) { + if ( handler.debug ) { + std::cout << "HttpConnection: do_write: STOP " + << bytes << " bytes. error code[" + << ec.value() << "]: " << ec.message().c_str() + << std::endl << std::flush; + } + manager.stop(shared_from_this()); + if ( self.get() ) {} + }); + if ( self.get() ) {} + return; } }); } @@ -679,6 +691,10 @@ HttpRequestHandler::handle_update(const HttpRequest& /* req */, HttpReply& rep) return write; } +/// Handle possible statistics summaries. +void HttpRequestHandler::handle_stat(const HttpRequest& /* req */, HttpReply& /* rep */) { +} + /// Handle a request and produce a reply. HttpRequestHandler::continue_action HttpRequestHandler::handle_request(const HttpRequest& req, HttpReply& rep) { diff --git a/Online/OnlineBase/src/LOG/FifoLog.cpp b/Online/OnlineBase/src/LOG/FifoLog.cpp index 511eb0f33..b58b7279c 100644 --- a/Online/OnlineBase/src/LOG/FifoLog.cpp +++ b/Online/OnlineBase/src/LOG/FifoLog.cpp @@ -335,15 +335,17 @@ void Logger::Implementation::read_data(int fd, string& data) { /// Poll data from stdout/stderr pipes void Logger::Implementation::poll() { while ( 1 ) { - struct epoll_event ep[2]; - int nc = ::epoll_wait(this->epoll_fd, ep, sizeof(ep)/sizeof(ep[0]), 100); + struct epoll_event ep; + int nc = ::epoll_wait(this->epoll_fd, &ep, 1, 100); if ( nc > 0 ) { for( int i=0; i < nc; ++i ) { - if ( ep[i].events&EPOLLIN ) { - if ( ep[i].data.fd == this->stdout_fd[0] ) - this->read_data(this->stdout_fd[0], this->stdout_buffer); - else if ( ep[i].data.fd == this->stderr_fd[0] ) - this->read_data(this->stderr_fd[0], this->stderr_buffer); + if ( ep.events&EPOLLIN ) { + if ( ep.data.fd == this->stdout_fd[0] ) + this->read_data(ep.data.fd, this->stdout_buffer); + else if ( ep.data.fd == this->stderr_fd[0] ) + this->read_data(ep.data.fd, this->stderr_buffer); + else + this->read_data(ep.data.fd, this->stdout_buffer); } } } diff --git a/Online/PCIE40Data/PCIE40Data/RawBank40.h b/Online/PCIE40Data/PCIE40Data/RawBank40.h index 17f0d49f1..78d20aaf3 100644 --- a/Online/PCIE40Data/PCIE40Data/RawBank40.h +++ b/Online/PCIE40Data/PCIE40Data/RawBank40.h @@ -167,9 +167,9 @@ namespace Online { } /// Header size - int hdrSize() const + static int hdrSize() { - return sizeof(RawBank40) - sizeof(m_data); + return sizeof(RawBank40) - sizeof(RawBank40::m_data); } /// Return size of the data body part of the bank diff --git a/Online/PCIE40Data/PCIE40Data/pcie40.h b/Online/PCIE40Data/PCIE40Data/pcie40.h index 3cb69695d..0eaaa1dab 100644 --- a/Online/PCIE40Data/PCIE40Data/pcie40.h +++ b/Online/PCIE40Data/PCIE40Data/pcie40.h @@ -195,7 +195,7 @@ namespace Online { public: static constexpr uint16_t MagicPattern = 0xCEFA; /// Magic Word (0xCEFA) - uint16_t magic {MagicPattern}; + uint16_t magic {MagicPattern}; /// Number of Sources (PCIe40s) uint16_t num_source {0}; /// Length of the MEP (without Header) @@ -541,14 +541,13 @@ namespace Online { inline void* bank_collection_t::copy_data(void* ptr) const { unsigned char* p = (unsigned char*)ptr; + size_t len = bank_t::hdrSize(); for( const auto* b=this->begin(); b != this->end(); b=this->next(b) ) { - size_t len = b->hdrSize(); ::memcpy(p, b, len); ::memcpy(p + len, b->data(), b->totalSize()-len); p += b->totalSize(); } for( const auto* b=this->special_begin(); b != this->special_end(); b=this->next(b) ) { - size_t len = b->hdrSize(); ::memcpy(p, b, len); ::memcpy(p + len, b->data(), b->totalSize()-len); p += b->totalSize(); diff --git a/Online/SmiController/scripts/TestStorageReader.sh b/Online/SmiController/scripts/TestStorageReader.sh new file mode 100755 index 000000000..a12662bb2 --- /dev/null +++ b/Online/SmiController/scripts/TestStorageReader.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# ========================================================================= +# +# Generic farm task startup script +# +# Author M.Frank +# Version: 1.0 +# Date: 20/05/2013 +# +# ========================================================================= +export INFO_OPTIONS=/group/online/dataflow/options/LHCb/MONITORING/OnlineEnv.opts; +echo "ERROR `dataflow_task Class1` `dataflow_default_options ${TASK_TYPE}`"; +`dataflow_task Class1` `dataflow_default_options ${TASK_TYPE}`; + diff --git a/Online/Storage/Storage/client.h b/Online/Storage/Storage/client.h index 8a6e7750e..77518766d 100644 --- a/Online/Storage/Storage/client.h +++ b/Online/Storage/Storage/client.h @@ -52,11 +52,11 @@ namespace Online { public: /// Static constructor for concrete implementations template <typename T> - static std::unique_ptr<client> create(const std::string& host, const std::string& port, int tmo); + static std::unique_ptr<client> create(const std::string& host, const std::string& port, int tmo, int dbg); public: /// Initializing constructor - client(const std::string& host, const std::string& port, int tmo); + client(const std::string& host, const std::string& port, int tmo, int dbg); /// Default constructor client() = delete; /// NO Move constructor diff --git a/Online/Storage/Storage/client_async.h b/Online/Storage/Storage/client_async.h index 58272735e..82e3fab82 100644 --- a/Online/Storage/Storage/client_async.h +++ b/Online/Storage/Storage/client_async.h @@ -59,7 +59,7 @@ namespace Online { /// NO assignment operator client_async& operator=(const client_async& copy) = delete; /// Initializing constructor: No need to call open if this constructor is issued - client_async(const std::string& host, const std::string& port, int tmo=10000); + client_async(const std::string& host, const std::string& port, int tmo, int dbg); /// Default destructor virtual ~client_async(); diff --git a/Online/Storage/Storage/client_sync.h b/Online/Storage/Storage/client_sync.h index 01b436b26..8fe6168f7 100644 --- a/Online/Storage/Storage/client_sync.h +++ b/Online/Storage/Storage/client_sync.h @@ -59,7 +59,7 @@ namespace Online { /// NO assignment operator client_sync& operator=(const client_sync& copy) = delete; /// Initializing constructor: No need to call open if this constructor is issued - client_sync(const std::string& host, const std::string& port, int tmo=10000); + client_sync(const std::string& host, const std::string& port, int tmo, int dbg); /// Default destructor virtual ~client_sync(); diff --git a/Online/Storage/Storage/communication.h b/Online/Storage/Storage/communication.h index 046d8ecff..fb343dcd9 100644 --- a/Online/Storage/Storage/communication.h +++ b/Online/Storage/Storage/communication.h @@ -40,7 +40,7 @@ namespace Online { /// Namespace for the storage implementation namespace storage { - bool error_code_ok(const boost::system::error_code& ec); + bool error_code_ok(const boost::system::error_code& ec, bool debug=true); typedef http::Uri uri_t; typedef http::HttpConnection connection_t; diff --git a/Online/Storage/Storage/fdb_client.h b/Online/Storage/Storage/fdb_client.h index 18ad15ef9..9059d94a4 100644 --- a/Online/Storage/Storage/fdb_client.h +++ b/Online/Storage/Storage/fdb_client.h @@ -36,15 +36,16 @@ namespace Online { */ class fdb_client { public: - typedef std::unique_ptr<client>(create_t)(const std::string& host, const std::string& port, int tmo); + typedef std::unique_ptr<client>(create_t)(const std::string& host, const std::string& port, int tmo, int dbg); public: - std::function<create_t> create_client; + std::function<create_t> create_client {client::create<client::sync>}; std::unique_ptr<client> fdbclient; std::string fdbserver; int dataPort; int dataTMO {1000}; - + int debug {0}; + public: /// Default constructor fdb_client() = default; @@ -69,13 +70,16 @@ namespace Online { reply_t save_object(const std::string& location, const void* data, std::size_t length); /// Read object data from the specified location without removing disk object - reply_t get_object(const std::string& location, time_t& date, std::size_t& length); + reply_t get_object(const std::string& location, std::time_t& date, std::size_t& length); /// Read AND Delete object data from disk at the specified location - reply_t delete_object(const std::string& location, time_t& date, std::size_t& length); + reply_t delete_object(const std::string& location, std::time_t& date, std::size_t& length); /// Read object data from the specified location without removing disk object - reply_t next_object_get(const std::string& location, time_t& date, std::size_t& length); + reply_t next_object_get(const std::string& location, std::time_t& date, std::size_t& length); /// Read object data from the specified location with removing disk object - reply_t next_object_delete(const std::string& location, time_t& date, std::size_t& length); + reply_t next_object_delete(const std::string& location, std::time_t& date, std::size_t& length); + + /// Helper: Delete the specified object data from the database + reply_t db_object_delete(const std::string& prefix, std::time_t& date, std::size_t& length); }; } // End namespace storage diff --git a/Online/Storage/Storage/fdb_server.h b/Online/Storage/Storage/fdb_server.h index c6d9aa1bd..9ac1c18ff 100644 --- a/Online/Storage/Storage/fdb_server.h +++ b/Online/Storage/Storage/fdb_server.h @@ -44,7 +44,7 @@ namespace http { /// Standard constructor explicit handler_t() = default; /// Standard destructor - virtual ~handler_t() = default; + virtual ~handler_t(); /// Specific handler for GET requests virtual continue_action handle_get(const request_t& req, reply_t& rep) override { @@ -70,6 +70,11 @@ namespace http { virtual continue_action handle_request(const request_t& req, reply_t& rep) override { return this->HttpRequestHandler::handle_request(req, rep); } + /// Handle possible statistics summaries. + virtual void handle_stat(const request_t& req, reply_t& rep) override { + this->HttpRequestHandler::handle_stat(req, rep); + } + }; } diff --git a/Online/Storage/src/client/client.cpp b/Online/Storage/src/client/client.cpp index 601f830ed..70bde187e 100644 --- a/Online/Storage/src/client/client.cpp +++ b/Online/Storage/src/client/client.cpp @@ -22,8 +22,8 @@ using namespace Online::storage; /// Initializing constructor -client::client(const std::string& h, const std::string& p, int tmo) - : host(h), port(p), timeout(tmo) +client::client(const std::string& h, const std::string& p, int tmo, int dbg) + : host(h), port(p), timeout(tmo), debug(dbg) { } diff --git a/Online/Storage/src/client/client_async.cpp b/Online/Storage/src/client/client_async.cpp index 0ec1c135d..c5f4b7428 100644 --- a/Online/Storage/src/client/client_async.cpp +++ b/Online/Storage/src/client/client_async.cpp @@ -82,13 +82,14 @@ void Online::storage::client_async::io_t::close() { } template <> -std::unique_ptr<client> client::create<client::async>(const string& host, const string& port, int tmo) { - return unique_ptr<client>(new client_async(host,port,tmo)); +std::unique_ptr<client> client::create<client::async>(const string& host, const string& port, int tmo, int dbg) { + auto obj = make_unique<client_async>(host, port, tmo, dbg); + return obj; } /// Initializing constructor -client_async::client_async(const string& h, const string& p, int tmo) - : client(h,p,tmo), io(make_unique<io_t>(this)) +client_async::client_async(const string& h, const string& p, int tmo, int dbg) + : client(h, p, tmo, dbg), io(make_unique<io_t>(this)) { } diff --git a/Online/Storage/src/client/client_sync.cpp b/Online/Storage/src/client/client_sync.cpp index 7d537c582..8694145a0 100644 --- a/Online/Storage/src/client/client_sync.cpp +++ b/Online/Storage/src/client/client_sync.cpp @@ -83,13 +83,14 @@ void Online::storage::client_sync::io_t::close() { } template <> -std::unique_ptr<client> client::create<client::sync>(const string& host, const string& port, int tmo) { - return unique_ptr<client>(new client_sync(host,port,tmo)); +std::unique_ptr<client> client::create<client::sync>(const string& host, const string& port, int tmo, int dbg) { + auto obj = make_unique<client_sync>(host, port, tmo, dbg); + return obj; } /// Initializing constructor -client_sync::client_sync(const string& h, const string& p, int tmo) - : client(h,p,tmo), io(make_unique<io_t>(this)) +client_sync::client_sync(const string& h, const string& p, int tmo, int dbg) + : client(h, p, tmo, dbg), io(make_unique<io_t>(this)) { } diff --git a/Online/Storage/src/client/communication.cpp b/Online/Storage/src/client/communication.cpp index 448f10d8c..88ab8d78d 100644 --- a/Online/Storage/src/client/communication.cpp +++ b/Online/Storage/src/client/communication.cpp @@ -21,13 +21,15 @@ using namespace std; -bool Online::storage::error_code_ok(const boost::system::error_code& ec) { +bool Online::storage::error_code_ok(const boost::system::error_code& ec, bool debug) { if ( !ec ) { return true; } - std::cout << "Bad error code[" - << ec.value() << "] (" - << ec.category().name() << ") " - << ec.message() << std::endl; + if ( debug ) { + std::cout << "Bad error code[" + << ec.value() << "] (" + << ec.category().name() << ") " + << ec.message() << std::endl; + } return false; } diff --git a/Online/Storage/src/client/fdb_client.cpp b/Online/Storage/src/client/fdb_client.cpp index 34a82ed88..3067da8ef 100644 --- a/Online/Storage/src/client/fdb_client.cpp +++ b/Online/Storage/src/client/fdb_client.cpp @@ -32,8 +32,8 @@ reply_t fdb_client::get_object(const string& location, time_t& date, size_t& len for ( const auto& h : reply.headers ) { if ( h.name == http::constants::location ) { uri_t u(h.value); - unique_ptr<client> cl(create_client(u.host, u.port, dataTMO)); - if ( error_code_ok(cl->open()) ) { + unique_ptr<client> cl(create_client(u.host, u.port, dataTMO, this->debug)); + if ( error_code_ok(cl->open(), this->debug) ) { reply = cl->get(u.path); if ( reply.status == reply_t::ok ) { const header_t* hdr = reply.header(http::constants::date); @@ -63,8 +63,8 @@ reply_t fdb_client::delete_object(const string& location, time_t& date, size_t& for ( const auto& h : reply.headers ) { if ( h.name == http::constants::location ) { uri_t u(h.value); - unique_ptr<client> cl(create_client(u.host, u.port, dataTMO)); - if ( error_code_ok(cl->open()) ) { + unique_ptr<client> cl(create_client(u.host, u.port, dataTMO, this->debug)); + if ( error_code_ok(cl->open(), this->debug) ) { reply = cl->del(u.path); if ( reply.status == reply_t::ok ) { const header_t* hdr = reply.header(http::constants::date); @@ -93,8 +93,8 @@ reply_t fdb_client::next_object_get(const string& prefix, time_t& date, size_t& for ( const auto& h : reply.headers ) { if ( h.name == http::constants::location ) { uri_t u(h.value); - unique_ptr<client> cl(create_client(u.host, u.port, dataTMO)); - if ( error_code_ok(cl->open()) ) { + unique_ptr<client> cl(create_client(u.host, u.port, dataTMO, this->debug)); + if ( error_code_ok(cl->open(), this->debug) ) { reply = cl->get(u.path); if ( reply.status == reply_t::ok ) { const header_t* hdr = reply.header(http::constants::date); @@ -123,8 +123,8 @@ reply_t fdb_client::next_object_delete(const string& prefix, time_t& date, size_ for ( const auto& h : reply.headers ) { if ( h.name == http::constants::location ) { uri_t u(h.value); - unique_ptr<client> cl(create_client(u.host, u.port, dataTMO)); - if ( error_code_ok(cl->open()) ) { + unique_ptr<client> cl(create_client(u.host, u.port, dataTMO, this->debug)); + if ( error_code_ok(cl->open(), this->debug) ) { reply = cl->del(u.path); if ( reply.status == reply_t::ok ) { const header_t* hdr = reply.header(http::constants::date); @@ -142,6 +142,17 @@ reply_t fdb_client::next_object_delete(const string& prefix, time_t& date, size_ return reply; } +/// Helper: Delete the specified object data from the database +reply_t fdb_client::db_object_delete(const string& location, time_t& date, size_t& length) { + reply_t reply = fdbclient->request(http::constants::del, location); + length = 0; + date = 0; + if ( reply.status == reply_t::temp_redirect ) { + reply.status = reply_t::ok; + } + return reply; +} + /// Save object record to database and return the location to store the data reply_t fdb_client::save_object_record(const string& location, string& url, size_t len) { struct tm tim; @@ -174,8 +185,8 @@ reply_t fdb_client::save_object_data(const string& location, const void* data, s { http::constants::date, ::gmtime_r(&now, &tim) } }; uri_t u(location); - unique_ptr<client> cl(create_client(u.host, u.port, dataTMO)); - if ( error_code_ok(cl->open()) ) { + unique_ptr<client> cl(create_client(u.host, u.port, dataTMO, this->debug)); + if ( error_code_ok(cl->open(), this->debug) ) { auto reply = cl->put(u.path, data, len, hdrs); if ( reply.status == reply_t::created ) { return reply; diff --git a/Online/Storage/src/server/fdb_SQLite.cpp b/Online/Storage/src/server/fdb_SQLite.cpp index 733995260..661ac40e7 100644 --- a/Online/Storage/src/server/fdb_SQLite.cpp +++ b/Online/Storage/src/server/fdb_SQLite.cpp @@ -22,6 +22,11 @@ using namespace boost; using namespace Online::storage; +namespace { + template<typename T> inline int _prt(const T* o) { + return o->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG; + } +} system::error_code fdb_SQLite::handler_t::initialize(const std::string& db_name) { std::string err; @@ -59,12 +64,10 @@ fdb_SQLite::handler_t::query(std::string& object, access = file.access; length = ::atol(file.size.c_str()); date = file.date; - ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, - "Query '%s'", object.c_str()); + ::lib_rtl_output(_prt(this), "Query '%s'", object.c_str()); return ec; } - ::lib_rtl_output(LIB_RTL_ALWAYS,"%s: Query: file '%s' NOT FOUND!", - this->database.name(), object.c_str()); + ::lib_rtl_output(LIB_RTL_ALWAYS,"Query: file '%s' NOT FOUND!", object.c_str()); return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); } @@ -77,20 +80,18 @@ fdb_SQLite::handler_t::next( std::string& object, File file; system::error_code ec = this->query_file(object, file, handler_t::STATE_WRITTEN); if ( ec == system::errc::success ) { - object = file.name; + object = file.name; access = file.access; - length = ::atol(file.size.c_str()); - date = file.date; + length = ::atol(file.size.c_str()); + date = file.date; ec = this->lock(file.name, handler_t::STATE_READ); if ( ec == system::errc::success ) { - ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, - "Next '%s'", object.c_str()); + ::lib_rtl_output(_prt(this), "Next '%s'", object.c_str()); return system::error_code(system::errc::success, system::system_category()); } return system::error_code(system::errc::protocol_error, system::system_category()); } - ::lib_rtl_output(LIB_RTL_ALWAYS,"%s: Query: file '%s' NOT FOUND!", - this->database.name(), object.c_str()); + ::lib_rtl_output(LIB_RTL_ALWAYS,"Query: file '%s' NOT FOUND!", object.c_str()); return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); } @@ -109,8 +110,7 @@ system::error_code fdb_SQLite::query_object(const std::string& object, auto ec = this->engine->query(obj, access, date, length); access = this->network_file(access); if ( ec == system::errc::success ) { - ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, - "Lookup '%s'", object.c_str()); + ::lib_rtl_output(_prt(this), "Lookup '%s'", object.c_str()); return ec; } ::lib_rtl_output(LIB_RTL_ALWAYS,"FAILED to lookup object: '%s' [%s]", @@ -128,8 +128,7 @@ system::error_code fdb_SQLite::query_next( const std::string& object, auto ec = this->engine->query(obj, access, date, length); access = this->network_file(access); if ( ec == system::errc::success ) { - ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, - "Lookup '%s'", object.c_str()); + ::lib_rtl_output(_prt(this), "Lookup '%s'", object.c_str()); return ec; } ::lib_rtl_output(LIB_RTL_ALWAYS,"FAILED to lookup object: '%s' [%s]", @@ -146,11 +145,10 @@ system::error_code fdb_SQLite::delete_next( const std::string& object, std::string obj = object; auto ec = this->engine->next(obj, access, date, length); if ( ec == system::errc::success ) { - ec = this->engine->del(object); + ec = this->engine->del(_location_prefix+access); access = this->network_file(access); if ( ec == system::errc::success ) { - ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, - "Lookup '%s'", object.c_str()); + ::lib_rtl_output(_prt(this), "Lookup '%s'", object.c_str()); return ec; } } @@ -168,15 +166,14 @@ system::error_code fdb_SQLite::delete_object(const std::string& object, std::string obj = object; auto ec = this->engine->query(obj, access, date, length); if ( ec == system::errc::success ) { - access = this->network_file(access); ec = this->engine->del(object); + access = this->network_file(access); if ( ec == system::errc::success ) { - ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, - "Remove '%s'", object.c_str()); + ::lib_rtl_output(_prt(this), "Remove '%s'", object.c_str()); return ec; } } - ::lib_rtl_output(LIB_RTL_ERROR,"FAILED to remove db record: '%s' [%s]", + ::lib_rtl_output(_prt(this), "FAILED to remove: '%s' [%s]", object.c_str(), ec.message().c_str()); return ec; } @@ -203,8 +200,7 @@ fdb_SQLite::add_object(const std::string& object, std::string acc = object.substr(_location_prefix.length(), std::string::npos); auto ec = this->engine->add(object, date, length, acc); if ( ec == system::errc::success ) { - ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, - "Add '%s' %s %s", object.c_str(), + ::lib_rtl_output(_prt(this), "Add '%s' %s %s", object.c_str(), this->debug > 1 ? "access:" : "", this->debug > 1 ? acc.c_str() : ""); access = this->network_file(acc); diff --git a/Online/Storage/src/server/fdb_SQLite_bind.cpp b/Online/Storage/src/server/fdb_SQLite_bind.cpp index e782f3d8e..bea063ef6 100644 --- a/Online/Storage/src/server/fdb_SQLite_bind.cpp +++ b/Online/Storage/src/server/fdb_SQLite_bind.cpp @@ -89,8 +89,6 @@ namespace { file.access = query_record.get<std::string>(4); return system::error_code(system::errc::success, system::system_category()); } - ::lib_rtl_output(LIB_RTL_ALWAYS,"%s: query: file %s NOT FOUND!", - this->query_record.db().name(), object_name.c_str()); return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); } diff --git a/Online/Storage/src/server/fdb_db_server.cpp b/Online/Storage/src/server/fdb_db_server.cpp index 512d4128e..5f2b63cbb 100644 --- a/Online/Storage/src/server/fdb_db_server.cpp +++ b/Online/Storage/src/server/fdb_db_server.cpp @@ -31,6 +31,12 @@ using namespace Online::storage; static const string PATTERN_NEXT = "/next?prefix="; + +/// Standard destructor +template <> http::basic_http_server<db>::handler_t::~handler_t() { +} + +/// Specific handler for GET requests template <> http::HttpRequestHandler::continue_action http::basic_http_server<db>::handler_t::handle_get(const request_t& req, reply_t& rep) { size_t length = 0; @@ -47,10 +53,10 @@ http::basic_http_server<db>::handler_t::handle_get(const request_t& req, reply_t rep = reply_t::stock_reply(reply_t::not_found); else if ( ec == boost::system::errc::permission_denied ) rep = reply_t::stock_reply(reply_t::unauthorized); - else if ( !error_code_ok(ec) ) + else if ( !error_code_ok(ec, this->debug) ) rep = reply_t::stock_reply(reply_t::not_found); - if ( !error_code_ok(ec) ) { + if ( !error_code_ok(ec, this->debug) ) { header_t h(http::constants::error_cause,"Failed to get "+req.uri+" ["+ec.message()+"]"); ::lib_rtl_output(LIB_RTL_ERROR,"DELETE: %s %-20s = %s", reply_t::stock_status(rep.status).c_str(), @@ -64,6 +70,7 @@ http::basic_http_server<db>::handler_t::handle_get(const request_t& req, reply_t return write; } +/// Specific handler for DELETE requests template <> http::HttpRequestHandler::continue_action http::basic_http_server<db>::handler_t::handle_delete(const request_t& req, reply_t& rep) { size_t length = 0; @@ -72,7 +79,7 @@ http::basic_http_server<db>::handler_t::handle_delete(const request_t& req, repl boost::system::error_code ec; { dbase_t::lock_t lock(dbase.get()); if ( get_next ) - ec = dbase->delete_next(req.uri.substr(PATTERN_NEXT.length()), access_name, date, length); + ec = dbase->delete_next(req.uri.substr(PATTERN_NEXT.length())+'%', access_name, date, length); else ec = dbase->delete_object(req.uri, access_name, date, length); } @@ -83,7 +90,7 @@ http::basic_http_server<db>::handler_t::handle_delete(const request_t& req, repl else if ( !error_code_ok(ec) ) rep = reply_t::stock_reply(reply_t::not_found); - if ( !error_code_ok(ec) ) { + if ( !error_code_ok(ec, this->debug) ) { header_t h(http::constants::error_cause,"Failed to delete "+req.uri+" ["+ec.message()+"]"); ::lib_rtl_output(LIB_RTL_ERROR,"DELETE: %s %-20s = %s", reply_t::stock_status(rep.status).c_str(), @@ -97,6 +104,7 @@ http::basic_http_server<db>::handler_t::handle_delete(const request_t& req, repl return write; } +/// Specific handler for PUT requests template <> http::HttpRequestHandler::continue_action http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t& rep) { const header_t *hdr_date, *hdr_len; @@ -130,7 +138,7 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t h.name.c_str(), h.value.c_str()); rep.headers.emplace_back(move(h)); } - else if ( !error_code_ok(ec) ) { + else if ( !error_code_ok(ec, this->debug) ) { header_t h(http::constants::error_cause, "Failed to add object "+req.uri+" in dbase: "+ec.message()); rep = reply_t::stock_reply(reply_t::bad_request); @@ -151,6 +159,7 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t return write; } +/// Specific handler for POST requests template <> http::HttpRequestHandler::continue_action http::basic_http_server<db>::handler_t::handle_update(const request_t& req, reply_t& rep) { const header_t *hdr_state; @@ -166,7 +175,7 @@ http::basic_http_server<db>::handler_t::handle_update(const request_t& req, repl dbase_t::lock_t lock(dbase.get()); ec = dbase->update_object_state(req.uri, hdr_state->value); } - if ( !error_code_ok(ec) ) { + if ( !error_code_ok(ec, this->debug) ) { header_t h(http::constants::error_cause, "Failed to update object "+req.uri+" in dbase: "+ec.message()); rep = reply_t::stock_reply(reply_t::bad_request); @@ -189,7 +198,7 @@ template <> http::HttpRequestHandler::continue_action http::basic_http_server<db>::handler_t::handle_request(const request_t& req, reply_t& rep) { if ( 0 == rep.bytes_sent ) { auto ret = this->HttpRequestHandler::handle_request(req, rep); - if ( this->debug > 0 || !(rep.status == reply_t::temp_redirect || rep.status == reply_t::ok) ) { + if ( this->debug > 0 ) { string status = reply_t::stock_status(rep.status); ::lib_rtl_output(LIB_RTL_INFO,"basic_http_server<db>::handle: Serving %s %s status: %d [%s]", req.method.c_str(), req.uri.c_str(), rep.status, status.c_str()); diff --git a/Online/Storage/src/server/fdb_fs_server.cpp b/Online/Storage/src/server/fdb_fs_server.cpp index c9d7d72d8..eb5a0e6b4 100644 --- a/Online/Storage/src/server/fdb_fs_server.cpp +++ b/Online/Storage/src/server/fdb_fs_server.cpp @@ -137,6 +137,11 @@ public: using namespace Online::storage; +/// Standard destructor +template <> http::basic_http_server<fs>::handler_t::~handler_t() { +} + +/// Specific handler for GET requests template <> http::HttpRequestHandler::continue_action http::basic_http_server<fs>::handler_t::handle_get(const request_t& req, reply_t& rep) { auto* ctxt = (fs_io*)rep.context.get(); @@ -164,7 +169,7 @@ http::basic_http_server<fs>::handler_t::handle_get(const request_t& req, reply_t // Now open the file, read a chunk and send it. // The rest gets sent by consecutive calls boost::system::error_code ec = ctxt->open_read(fname); - if ( !error_code_ok(ec) ) { + if ( !error_code_ok(ec, this->debug) ) { rep = reply_t::stock_reply(reply_t::not_found); rep.headers.emplace_back(constants::error_cause,"File '"+fname+"' does not exist"); } @@ -183,30 +188,35 @@ http::basic_http_server<fs>::handler_t::handle_get(const request_t& req, reply_t ctxt->st_data += rd; } else { - ::lib_rtl_output(LIB_RTL_ERROR, - "+++ %s: Failed to read data from '%s' [%s] only got %ld out of %ld bytes", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read data '%s' [%s] only got %ld out of %ld bytes", req.method.c_str(), req.uri.c_str(), ::strerror(errno), rd, len); } if ( ctxt->st_data == ctxt->st_size ) { - ::lib_rtl_output(LIB_RTL_INFO, - "+++ %s: Prepared last chunk from '%s' total:%ld / %ld bytes", - req.method.c_str(), req.uri.c_str(), - ctxt->st_data, ctxt->st_size); + ::lib_rtl_output(LIB_RTL_INFO, "+++ %s: Prepared last chunk '%s' %.1f MB", + req.method.c_str(), req.uri.c_str(), double(ctxt->st_size)/(1024e0*1024e0)); } return write; } else if ( ctxt && ctxt->st_size == ctxt->st_data ) { + std::chrono::time_point<std::chrono::system_clock> null; + long net = std::chrono::duration_cast<std::chrono::milliseconds>(req.netio-null).count(); + long wrt = std::chrono::duration_cast<std::chrono::milliseconds>(req.handling-null).count(); + double mb = double(ctxt->st_size)/(1024e0*1024e0); + ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %s: %ld bytes netio: %ld ms %.1f MB/s wrt: %ld ms %.1f MB/s", + req.method.c_str(), ctxt->st_size, + net, mb/std::max(1e-6, double(net)/1e3), + wrt, mb/std::max(1e-6, double(wrt)/1e3)); return none; } - ::lib_rtl_output(LIB_RTL_ERROR, - "+++ %s: Failed to read data from '%s' size:%ld / %ld %s [Inconsistemt context]", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read data '%s' size:%ld / %ld %s [Inconsistemt context]", req.method.c_str(), req.uri.c_str(), rep.bytes_sent, rep.bytes_total, rep.stock_status(rep.status).c_str()); rep.context.reset(); return none; } +/// Specific handler for DELETE requests template <> http::HttpRequestHandler::continue_action http::basic_http_server<fs>::handler_t::handle_delete(const request_t& req, reply_t& rep) { auto* ctxt = (fs_io*)rep.context.get(); @@ -236,7 +246,7 @@ http::basic_http_server<fs>::handler_t::handle_delete(const request_t& req, repl // The rest gets sent by consecutive calls ec = ctxt->open_read(fname); - if ( !error_code_ok(ec) ) { + if ( !error_code_ok(ec, this->debug) ) { rep = reply_t::stock_reply(reply_t::not_found); rep.headers.emplace_back(constants::error_cause,"File '"+fname+"' does not exist"); } @@ -255,35 +265,39 @@ http::basic_http_server<fs>::handler_t::handle_delete(const request_t& req, repl ctxt->st_data += rd; } else { - ::lib_rtl_output(LIB_RTL_ERROR, - "+++ %s: Failed to read data from '%s' [%s] only got %ld out of %ld bytes", - req.method.c_str(), req.uri.c_str(), - ::strerror(errno), rd, len); + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read data '%s' [%s] only got %ld out of %ld bytes", + req.method.c_str(), req.uri.c_str(), ::strerror(errno), rd, len); } if ( ctxt->st_data == ctxt->st_size ) { const std::string fname = dbase->local_file(req.uri); ctxt->close(); ec = ctxt->unlink(fname.c_str()); - if ( !error_code_ok(ec) ) { + if ( !error_code_ok(ec, this->debug) ) { } - ::lib_rtl_output(LIB_RTL_INFO, - "+++ %s: Prepared last chunk from '%s' total:%ld / %ld bytes", - req.method.c_str(), req.uri.c_str(), - ctxt->st_data, ctxt->st_size); + ::lib_rtl_output(LIB_RTL_INFO, "+++ %s: Prepared last chunk '%s' %.1f MB", + req.method.c_str(), req.uri.c_str(), double(ctxt->st_size)/(1024e0*1024e0)); } return write; } else if ( ctxt && ctxt->st_size == ctxt->st_data ) { + std::chrono::time_point<std::chrono::system_clock> null; + long net = std::chrono::duration_cast<std::chrono::milliseconds>(req.netio-null).count(); + long wrt = std::chrono::duration_cast<std::chrono::milliseconds>(req.handling-null).count(); + double mb = double(ctxt->st_size)/(1024e0*1024e0); + ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %s: %ld bytes netio: %ld ms %.1f MB/s wrt: %ld ms %.1f MB/s", + req.method.c_str(), ctxt->st_size, + net, mb/std::max(1e-6, double(net)/1e3), + wrt, mb/std::max(1e-6, double(wrt)/1e3)); return none; } - ::lib_rtl_output(LIB_RTL_ERROR, - "+++ %s: Failed to read data from '%s' size:%ld / %ld %s [Inconsistemt context]", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read data '%s' size:%ld / %ld %s [Inconsistemt context]", req.method.c_str(), req.uri.c_str(), rep.bytes_sent, rep.bytes_total, rep.stock_status(rep.status).c_str()); rep.context.reset(); return none; } +/// Specific handler for PUT requests template <> http::HttpRequestHandler::continue_action http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t& rep) { auto* ctxt = (fs_io*)rep.context.get(); @@ -320,20 +334,22 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t if ( -1 == ::stat(dir.c_str(), &stat) ) { ret_stat = ::mkdir(dir.c_str(), 0777); if ( ret_stat != 0 ) { - char err[1024]; - ::strerror_r(errno, err, sizeof(err)); - ::lib_rtl_output(LIB_RTL_ERROR, - "+++ %s: Failed to create parent directory %s [%s]", - req.method.c_str(), dir.c_str(), err); - if ( errno == EACCES || errno == ENOENT || errno == EPERM || errno == EROFS ) - rep = reply_t::stock_reply(reply_t::unauthorized); - else - rep = reply_t::stock_reply(reply_t::bad_request); - rep.headers.emplace_back(constants::error_cause,err); - return write; + ::lib_rtl_sleep(10); + // Wait and then check if another thread created the directory + if ( -1 == ::stat(dir.c_str(), &stat) ) { + char err[1024]; + ::strerror_r(errno, err, sizeof(err)); + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to create parent directory %s [%s]", + req.method.c_str(), dir.c_str(), err); + if ( errno == EACCES || errno == ENOENT || errno == EPERM || errno == EROFS ) + rep = reply_t::stock_reply(reply_t::unauthorized); + else + rep = reply_t::stock_reply(reply_t::bad_request); + rep.headers.emplace_back(constants::error_cause,err); + return write; + } } - ::lib_rtl_output(LIB_RTL_INFO, - "+++ %s: Created parent directory %s", + ::lib_rtl_output(LIB_RTL_INFO, "+++ %s: Created parent directory %s", req.method.c_str(), dir.c_str()); } } @@ -346,17 +362,15 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t // Now open the file, read a chunk and send it. // The rest gets sent by handle_request_cont_full boost::system::error_code ec = ctxt->open_write(fname); - if ( !error_code_ok(ec) ) { + if ( !error_code_ok(ec, this->debug) ) { rep = reply_t::stock_reply(reply_t::forbidden); - rep.headers.emplace_back(constants::error_cause, - "Failed to open file "+fname+" ["+ec.message()+"]"); + rep.headers.emplace_back(constants::error_cause, "Failed to open file "+fname+" ["+ec.message()+"]"); } if ( req.content.empty() ) { return write; } // This should not happen: The client must first receive the continue! - ::lib_rtl_output(LIB_RTL_ERROR, - "+++ %s: Received non-empty data on request to write " + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Received non-empty data on request to write " "'%s' [Protocol Violation]", req.method.c_str(), fname.c_str()); //req.content.clear(); return write; @@ -370,8 +384,7 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t long wrt = ctxt->write(len, &req.content.at(0)); ctxt->st_data += wrt; if ( wrt != len ) { - ::lib_rtl_output(LIB_RTL_ERROR, - "+++ %s: Failed to write data chunk from '%s' " + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to write data chunk '%s' " "[%s] only wrote %ld out of %ld bytes", req.method.c_str(), req.uri.c_str(), ::strerror(errno), wrt, len); } @@ -386,8 +399,8 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t double mb = double(ctxt->st_size)/(1024e0*1024e0); ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %s: Wrote data '%s' %.0f %cB", req.method.c_str(), req.uri.c_str(), mb > 10e0 ? mb : mb*1024e0, mb > 10e0 ? 'M' : 'k'); - ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %s: netio: %ld ms %.1f MB/s wrt: %ld ms %.1f MB/s", - req.method.c_str(), + ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %s: %ld bytes netio: %ld ms %.1f MB/s wrt: %ld ms %.1f MB/s", + req.method.c_str(), ctxt->st_size, net, mb/std::max(1e-6, double(net)/1e3), wrt, mb/std::max(1e-6, double(wrt)/1e3)); rep.context.reset(); @@ -398,8 +411,8 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t rep.headers.emplace_back(constants::content_location, req.uri); uri_t u(dbase->server); - std::unique_ptr<client> cl(client::create<client::sync>(u.host, u.port, 10000)); - if ( error_code_ok(cl->open()) ) { + std::unique_ptr<client> cl(client::create<client::sync>(u.host, u.port, 10000, this->debug)); + if ( error_code_ok(cl->open(), this->debug) ) { client::reqheaders_t hdrs { { http::constants::state, "WRITTEN" } }; @@ -409,14 +422,12 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t return write; } } - ::lib_rtl_output(LIB_RTL_ERROR, - "+++ %s: FAILED to update '%s' to state WRITTEN!", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: FAILED to update '%s' to state WRITTEN!", req.method.c_str(), req.uri.c_str()); return write; } - ::lib_rtl_output(LIB_RTL_ERROR, - "+++ %s: Finished request handling for '%s' with %ld /%ld bytes", - req.method.c_str(), req.uri.c_str(), ctxt->st_data, ctxt->st_size); + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Finished request '%s' %ld bytes", + req.method.c_str(), req.uri.c_str(), ctxt ? ctxt->st_size : 0); /// The request is handled. Trigger the shutdown of the connection return none; } diff --git a/Online/Storage/src/server/sqlite_test.cpp b/Online/Storage/src/server/sqlite_test.cpp index 1ae89609b..3ebc8f0d4 100644 --- a/Online/Storage/src/server/sqlite_test.cpp +++ b/Online/Storage/src/server/sqlite_test.cpp @@ -21,7 +21,7 @@ gentest.exe libStorageClientTest.so sqlite_populate_database_direct -database=sq gentest.exe libStorageClientTest.so sqlite_update_database_direct -database=sqlite_test.dbase -stream=HLT1 -run=11223346 -count=100 -gentest.exe libStorageClientTest.so sqlite_read_database_direct -database=/home/frankm/storage_files.dbase +gentest.exe libStorageServer.so fdb_cli_dumpdb -database=/home/frankm/storage_files.dbase */ // Framework inclde files #include <CPP/sqlite.h> @@ -37,7 +37,7 @@ using namespace sqlite; namespace { struct Setup { - string name, database, stream, log; + string name, database, stream, log, entry; size_t count = 0; long run = 0; int prt = LIB_RTL_INFO; @@ -51,6 +51,7 @@ namespace { cli.getopt("database", 8, database); cli.getopt("stream", 5, stream); cli.getopt("count", 5, count); + cli.getopt("entry", 5, entry); cli.getopt("run", 3, run); cli.getopt("logger", 4, log); cli.getopt("print", 4, prt); @@ -257,7 +258,7 @@ extern "C" int sqlite_update_database_direct(int argc, char** argv) { } return 0; } - +// // //========================================================================== extern "C" int fdb_cli_dumpdb(int argc, char** argv) { @@ -270,9 +271,9 @@ extern "C" int fdb_cli_dumpdb(int argc, char** argv) { return 0; } }; - Setup setup("fdb_cli_delete", argc, argv, [](int,char**) { + Setup setup("fdb_cli_dumpdb", argc, argv, [](int,char**) { ::fprintf(stderr, - "fdb_cli_delete -opt [-opt] \n" + "fdb_cli_dumpdb -opt [-opt] \n" " -database=<file-path> Path to the SQLite file to dump. \n" " -help Print this help. \n"); }); diff --git a/Online/Storage/tests/src/Setup.cpp b/Online/Storage/tests/src/Setup.cpp index 4914ffbc8..187812f60 100644 --- a/Online/Storage/tests/src/Setup.cpp +++ b/Online/Storage/tests/src/Setup.cpp @@ -41,6 +41,7 @@ Setup::Setup(const std::string& n, int argc, char* argv[], void (*help)(int,char cli.getopt("file", 4, file); cli.getopt("logger", 4, log); cli.getopt("print", 4, prt); + cli.getopt("debug", 4, debug); RTL::Logger::install_fifolog(log.c_str(), RTL::Logger::log_args(prt)); ::lib_rtl_output(LIB_RTL_DEBUG,"%s: Test starting....", name.c_str()); } @@ -52,8 +53,7 @@ Setup::~Setup() { fdb_client Setup::client() { fdb_client client; uri_t url(this->server); - client.fdbclient = client::create<client::sync>(url.host, url.port, 10000); - client.create_client = std::function(client::create<client::sync>); + client.fdbclient = client::create<client::sync>(url.host, url.port, 10000, debug); return client; } diff --git a/Online/Storage/tests/src/Setup.h b/Online/Storage/tests/src/Setup.h index d7fb676f7..570063baa 100644 --- a/Online/Storage/tests/src/Setup.h +++ b/Online/Storage/tests/src/Setup.h @@ -30,6 +30,7 @@ namespace Online { std::string server, log, url, file, name, check_file; int prt = LIB_RTL_INFO; + int debug = 0; public: Setup(const std::string& n, int argc, char* argv[], void (*help)(int,char**)=nullptr); diff --git a/Online/Storage/tests/src/fdb_cli.cpp b/Online/Storage/tests/src/fdb_cli.cpp index 217ca01df..bc3fac4f1 100644 --- a/Online/Storage/tests/src/fdb_cli.cpp +++ b/Online/Storage/tests/src/fdb_cli.cpp @@ -16,7 +16,10 @@ gentest.exe libStorageCli.so fdb_test_client_get -server=${HOST}:8000 -url=/objects/data/072908_0000000072.raw -gentest.exe libStorageCli.so fdb_cli_get -server=${HOST}:8000 -url=/objects/data/072908_0000000072.raw -file=/home/frankm/data/072908_0000000072.raw +gentest.exe libStorageCli.so fdb_cli_save -server=${HOST}:8000 -url=/objects/data/072908_0000000072.raw +gentest.exe libStorageCli.so fdb_cli_put -server=${HOST}:8000 -url=/objects/data/072908_0000000072.raw + +gentest.exe libStorageCli.so fdb_cli_get -server=${HOST}:8000 -url=/objects/data/072908_0000000072.raw gentest.exe libStorageCli.so fdb_test_delete -server=${HOST}:8000 -url=/objects/data/072908_0000000072.raw -file=/home/frankm/data/072908_0000000072.raw @@ -25,6 +28,10 @@ gentest.exe libStorageCli.so fdb_test_put -server=${HOST}:8000 -url=/ob gentest.exe libStorageCli.so fdb_test_client_save -server=${HOST}:8000 -url=/objects/data/072908_0099999999.raw -file=/home/frankm/data/072908_0000000072.raw gentest.exe libStorageCli.so fdb_test_save -server=${HOST}:8000 -url=/objects/data/072908_0099999999.raw -file=/home/frankm/data/072908_0000000072.raw + +gentest.exe libStorageCli.so fdb_cli_db_delete -server=${HOST}:8000 -url=/objects/data/testing/0000000000/2021-03-10__14-34/0000000000_mon0101_14-34-53_21842.raw + + */ // Framework inclde files @@ -170,3 +177,30 @@ extern "C" int fdb_cli_delete(int argc, char** argv) { } return 0; } +// +//========================================================================== +extern "C" int fdb_cli_db_delete(int argc, char** argv) { + Setup setup("fdb_cli_db_delete",argc, argv, [](int,char**) { + ::fprintf(stderr, + "fdb_cli_db_delete -opt [-opt] \n" + " Remove entry from the database only. \n\n" + " -server=<name:port> Server access specification. \n" + " -url=<url-spec> URL of the file to resolve \n" + " -print=<level> Set print level 1=DEBUG, 5=FATAL \n" + " -help Print this help. \n"); + }); + time_t date = 0; + size_t length = 0; + auto client = setup.client(); + auto reply = client.db_object_delete(setup.url, date, length); + if ( reply.status == reply_t::ok ) { + ::lib_rtl_output(LIB_RTL_ALWAYS, "Successfully removed dbase entry: %s", + setup.url.c_str()); + } + else { + ::lib_rtl_output(LIB_RTL_ALWAYS, "FAILED to remove dbase entry: %s", + setup.url.c_str()); + setup.print(reply); + } + return 0; +} diff --git a/Online/Storage/tests/src/fdb_client_test.cpp b/Online/Storage/tests/src/fdb_client_test.cpp index 41de88ee2..e2c1c5c46 100644 --- a/Online/Storage/tests/src/fdb_client_test.cpp +++ b/Online/Storage/tests/src/fdb_client_test.cpp @@ -81,7 +81,7 @@ extern "C" int fdb_test_client_get(int argc, char** argv) { " -print=<level> Set print level 1=DEBUG, 5=FATAL\n"); }); uri_t url(setup.server); - auto client = client::create<client::sync>(url.host, url.port, 10000); + auto client = client::create<client::sync>(url.host, url.port, 10000, setup.debug); auto reply = client->get(setup.url); reply.content.push_back(0); reply.print(cout,"fdb_client_get_test: "); @@ -113,7 +113,7 @@ extern "C" int fdb_test_client_save(int argc, char** argv) { return err; } uri_t url(setup.server); - auto client = client::create<client::sync>(url.host, url.port, 10000); + auto client = client::create<client::sync>(url.host, url.port, 10000, setup.debug); client::reqheaders_t hdrs { { http::constants::content_length, stat.st_size }, { http::constants::content_type, "rawdata/lhcb" }, diff --git a/Online/Tell1Data/Tell1Data/EventHeader.h b/Online/Tell1Data/Tell1Data/EventHeader.h index 665c458b5..9f8c4273b 100644 --- a/Online/Tell1Data/Tell1Data/EventHeader.h +++ b/Online/Tell1Data/Tell1Data/EventHeader.h @@ -72,33 +72,46 @@ namespace Online { /// Spare unsigned char m_spare[1]; - struct PACKED_DATA HeaderTriggerMask { + struct PACKED_DATA HeaderTriggerMask { /// Trigger mask used for event selection unsigned int m_trMask[4]; - HeaderTriggerMask() { + HeaderTriggerMask() { m_trMask[0] = m_trMask[1] = m_trMask[2] = m_trMask[3] = 0; } + /// Accessor: Number of bits in the trigger mask - unsigned int maskBits() const { return sizeof(m_trMask)*8; } + unsigned int maskBits() const { return sizeof(m_trMask)*8; } + /// Accessor: trigger mask - std::array<unsigned int, 4> triggerMask() const { + std::array<unsigned int, 4> triggerMask() const { return {m_trMask[0], m_trMask[1], m_trMask[2], m_trMask[3]}; } + /// Update the trigger mask of the event - void setTriggerMask(const std::array<unsigned int,4>& mask){ + void setTriggerMask(const std::array<unsigned int,4>& mask) { m_trMask[0] = mask[0]; m_trMask[1] = mask[1]; m_trMask[2] = mask[2]; m_trMask[3] = mask[3]; } + /// Update the trigger mask of the event - void setTriggerMask(const unsigned int* mask){ + void setTriggerMask(const unsigned int* mask) { m_trMask[0] = mask[0]; m_trMask[1] = mask[1]; m_trMask[2] = mask[2]; m_trMask[3] = mask[3]; } + + /// Update the trigger mask of the event + void setTriggerMask(unsigned int m0, unsigned int m1, unsigned int m2, unsigned int m3) { + m_trMask[0] = m0; + m_trMask[1] = m1; + m_trMask[2] = m2; + m_trMask[3] = m3; + } }; + struct PACKED_DATA Header0 { typedef long long int int_64_t; /// Event type identifier @@ -130,14 +143,14 @@ namespace Online { return {m_trMask[0], m_trMask[1], m_trMask[2], m_trMask[3]}; } /// Update the trigger mask of the event - void setTriggerMask(const std::array<unsigned int,4>& mask){ + void setTriggerMask(const std::array<unsigned int,4>& mask) { m_trMask[0] = mask[0]; m_trMask[1] = mask[1]; m_trMask[2] = mask[2]; m_trMask[3] = mask[3]; } /// Update the trigger mask of the event - void setTriggerMask(const unsigned int* mask){ + void setTriggerMask(const unsigned int* mask) { m_trMask[0] = mask[0]; m_trMask[1] = mask[1]; m_trMask[2] = mask[2]; @@ -203,60 +216,62 @@ namespace Online { } /// Default destructor ~EventHeader() {} + /// Check if this is really an MDF header + bool is_mdf() const + { return m_size[0] > 0 && m_size[0]==m_size[1] && m_size[0]==m_size[2]; } /// Access record size - unsigned int recordSize() const { return m_size[0]; } + int recordSize() const { return int(m_size[0]); } /// Accessor: event size - unsigned int size() const - { return m_size[0]-sizeOf(headerVersion()); } + unsigned int size() const { return m_size[0]-sizeOf(headerVersion()); } /// Update event size void setSize(unsigned int val) - { m_size[0]=m_size[1]=m_size[2]=val+sizeOf(headerVersion()); } + { m_size[0]=m_size[1]=m_size[2]=val+sizeOf(headerVersion()); } /// For checks: return 0th. size word - unsigned int size0() const { return m_size[0]; } + unsigned int size0() const { return m_size[0]; } /// For checks: return 1rst. size word - unsigned int size1() const { return m_size[1]; } + unsigned int size1() const { return m_size[1]; } /// For checks: return 2nd. size word - unsigned int size2() const { return m_size[2]; } + unsigned int size2() const { return m_size[2]; } /// For special stuff: modify 3rd. size word by hand - void setSize2(unsigned int val) { m_size[2] = val; } + void setSize2(unsigned int val) { m_size[2] = val; } /// Accessor: checksum of the event data - unsigned int checkSum() const { return m_checkSum; } + unsigned int checkSum() const { return m_checkSum; } /// Update checksum of the event data - void setChecksum(unsigned int val) { m_checkSum = val; } - /// Accessor: Identifier of the compression method - unsigned char compression() const { return m_compression; } + void setChecksum(unsigned int val) { m_checkSum = val; } + /// Accessor: Identifier of the compression method + unsigned char compression() const { return m_compression; } /// Update the identifier of the compression method - void setCompression(unsigned int val) { m_compression=(unsigned char)val; } + void setCompression(unsigned int val) { m_compression=(unsigned char)val; } /// Accessor: length of the event header - unsigned int subheaderLength() const { return (m_hdr&0x0F)*sizeof(int); } + unsigned int subheaderLength() const { return (m_hdr&0x0F)*sizeof(int); } /// Update the length of the event header void setSubheaderLength(unsigned int l) { l = l%sizeof(int) ? (l/sizeof(int)) + 1 : l/sizeof(int); m_hdr = (unsigned char)((0xF0&m_hdr) + (0x0F&l)); } /// Accessor: version of the event header - unsigned int headerVersion() const { return m_hdr>>4; } + unsigned int headerVersion() const { return m_hdr>>4; } /// Update the version of the event header void setHeaderVersion(unsigned int vsn) - { m_hdr = (unsigned char)(((vsn<<4)+(m_hdr&0xF))&0xFF); } + { m_hdr = (unsigned char)(((vsn<<4)+(m_hdr&0xF))&0xFF); } /// Accessor: hdr field - unsigned char hdr() const { return m_hdr; } + unsigned char hdr() const { return m_hdr; } /// Update hdr field - void setHdr(unsigned char val) { m_hdr = val; } + void setHdr(unsigned char val) { m_hdr = val; } /// Accessor: event type identifier - unsigned char dataType() const { return m_dataType; } + unsigned char dataType() const { return m_dataType; } /// Update the event type - void setDataType(unsigned char val) { m_dataType = val; } + void setDataType(unsigned char val) { m_dataType = val; } /// Set spare word - void setSpare(unsigned char val) { m_spare[0] = val; } + void setSpare(unsigned char val) { m_spare[0] = val; } /// Access to data payload (Header MUST be initialized) - char* data() { return ((char*)this)+sizeOf(headerVersion()); } + char* data() { return ((char*)this)+sizeOf(headerVersion()); } /// Access to data payload (Header MUST be initialized) - const char* data() const { return ((char*)this)+sizeOf(headerVersion()); } + const char* data() const { return ((char*)this)+sizeOf(headerVersion()); } /// Access to sub-headers - SubHeader subHeader0() { return SubHeader(m_spare-1); } - SubHeader subHeader() const { return SubHeader(m_spare+1); } + SubHeader subHeader0() { return SubHeader(m_spare-1); } + SubHeader subHeader() const { return SubHeader(m_spare+1); } //const SubHeader subHeader() const { return SubHeader(m_spare+1); } }; } // End namespace LHCb diff --git a/Online/Tell1Data/Tell1Data/RawFile.h b/Online/Tell1Data/Tell1Data/RawFile.h index a69ac80c9..0a60c9e75 100644 --- a/Online/Tell1Data/Tell1Data/RawFile.h +++ b/Online/Tell1Data/Tell1Data/RawFile.h @@ -97,7 +97,8 @@ namespace Online { long write(const void* data, int len); long write(const void* data, size_t len); int open(); - int map(const void* data, size_t len); + int map_memory(const void* data, size_t len); + int unmap_memory(); int openMapped(); int openWrite(); int unlink(); diff --git a/Online/Tell1Data/src/RawFile.cpp b/Online/Tell1Data/src/RawFile.cpp index b66d28a68..2327f422a 100644 --- a/Online/Tell1Data/src/RawFile.cpp +++ b/Online/Tell1Data/src/RawFile.cpp @@ -183,14 +183,22 @@ int RawFile::openMapped() { return fd; } -int RawFile::map(const void* ptr, size_t len) { - if ( m_fd > 0 ) close(); +int RawFile::map_memory(const void* ptr, size_t len) { + m_begin = m_ptr = m_end = nullptr; + this->close(); m_fd = -1; m_ptr = m_begin = (unsigned char*)ptr; m_end = m_begin + len; return 1; } +int RawFile::unmap_memory() { + m_begin = m_ptr = m_end = nullptr; + this->close(); + m_fd = -1; + return 1; +} + long RawFile::write(const void* data, int len) { const unsigned char* p = (const unsigned char*) data; int tmp = len; @@ -223,10 +231,13 @@ long RawFile::read(void* pointer, int len) { m_ptr += len; return len; } - len = m_end - m_ptr; - ::memcpy(p, m_ptr, len); - m_ptr = m_end; - return 0; + else if ( m_ptr < m_end ) { + len = m_end - m_ptr; + ::memcpy(p, m_ptr, len); + m_ptr = m_end; + return len; + } + return -1; } int tmp = 0; while (tmp < len) { @@ -281,7 +292,7 @@ long RawFile::read_event(EventType expected, off_t file_position = this->seek(0, SEEK_CUR); int status = this->read((unsigned char*)size_buf, sizeof(size_buf)); if ( data ) *data = 0; - if (status <= 0) { + if ( status <= 0 ) { this->close(); return -1; } diff --git a/Online/Tell1Data/src/Tell1Decoder.cpp b/Online/Tell1Data/src/Tell1Decoder.cpp index e069825dd..8df5386d6 100644 --- a/Online/Tell1Data/src/Tell1Decoder.cpp +++ b/Online/Tell1Data/src/Tell1Decoder.cpp @@ -241,8 +241,8 @@ static void print_previous_bank(const Tell1Bank* prev) { /// Check sanity of raw bank structure bool Online::checkRawBank(const Tell1Bank* b, bool throw_exc, bool print_cout) { typedef Tell1Printout _P; - // Check bank's magic word - if ( b->magic() == Tell1Bank::MagicPattern ) { + // Check bank's magic word: Either Tell1 magic word or PCIE40 magic word + if ( b->magic() == Tell1Bank::MagicPattern || b->magic() == 0xFACE ) { // Crude check on the bank type if ( b->type() < Tell1Bank::LastType || b->type() >= Tell1Bank::DaqErrorBase ) { // Crude check on the bank length -- GitLab From 6569b5ae3c989e8c0d1d109287b8cec7256ac04d Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Tue, 16 Mar 2021 17:47:56 +0100 Subject: [PATCH 06/26] Fix to event builder options after debugging --- Online/Dataflow/Dataflow/DiskReader.h | 28 ++++------- Online/Dataflow/src/framework/DiskReader.cpp | 47 +++++++++---------- Online/EventBuilding/options/BU_0.opts | 3 +- Online/EventBuilding/options/BU_1.opts | 3 +- Online/EventBuilding/options/EB_BU_algo.opts | 4 +- Online/EventBuilding/options/EB_RU_algo.opts | 2 +- .../options/EB_Transport_properties.opts | 4 +- .../EventBuilding/options/EB_transport.opts | 8 ++-- Online/FarmConfig/job/EBPass.sh | 47 +++++++++++++++++++ Online/FarmConfig/job/EBStorage.sh | 12 +++++ Online/FarmConfig/job/preamble.sh | 26 ++++++++++ Online/FarmConfig/options/EBMBM.opts | 2 +- Online/FarmConfig/options/EBStorage.opts | 42 +++++++++++++++++ Online/GaudiOnline/src/FileEventOutput.cpp | 1 + 14 files changed, 175 insertions(+), 54 deletions(-) create mode 100755 Online/FarmConfig/job/EBPass.sh create mode 100755 Online/FarmConfig/job/EBStorage.sh create mode 100644 Online/FarmConfig/options/EBStorage.opts diff --git a/Online/Dataflow/Dataflow/DiskReader.h b/Online/Dataflow/Dataflow/DiskReader.h index 4da4be04c..09fa3810b 100644 --- a/Online/Dataflow/Dataflow/DiskReader.h +++ b/Online/Dataflow/Dataflow/DiskReader.h @@ -91,9 +91,9 @@ namespace Online { public: /// Scan directory for matching files - size_t scanDirectory(const std::string& dir_name); + size_t scanDirectory(const std::string& dir_name); /// Scan directory for matching files - size_t scanFiles(); + size_t scanFiles(); /// Open a new data file RawFile openFile(); @@ -213,20 +213,12 @@ namespace Online { static bool check_consumers_partid(const MBM::BufferInfo& info, int pid); static bool check_consumers(const MBM::BufferInfo& info, int pid, int evtyp); - /// Scan single directory for matching files - //virtual size_t scanDirectory(const std::string& dir_name); - /// Scan directories for matching files - //virtual size_t scanFiles(); - /// Open a new data file - //virtual RawFile openFile(); - - - /// If the GO service is present, wait until processing is enabled - int waitForGo(); - /// Wait until consumers are ready (if required) - int waitForConsumers(MBM::BufferInfo* mbmInfo); /// Update run number service void updateRunNumber(int new_run); + /// If the GO service is present, wait until processing is enabled + virtual int waitForGo(); + /// Wait until consumers are ready (if required) + virtual int waitForConsumers(MBM::BufferInfo* mbmInfo); /// Wait until event buffers are empty before finishing.... virtual void waitForPendingEvents(int seconds); /// Patch the string array with allowed runs @@ -245,10 +237,10 @@ namespace Online { /// Handle MBM request: load event data and declare to MBM virtual int handle_mbm_request(MBM::BufferInfo* info, RawFile& file); - void save_file_rest(RawFile& file); + virtual void save_file_rest(RawFile& file); /// Runable implementation : Run the class implementation - virtual int i_run() = 0; + virtual int i_run() = 0; public: /// Standard algorithm constructor @@ -280,5 +272,5 @@ namespace Online { /// IRunable implementation : Run the class implementation virtual int run() override; }; -} // End namespace Online -#endif // ONLINE_DATAFLOW_INPUTREADER_H +} // End namespace Online +#endif // ONLINE_DATAFLOW_INPUTREADER_H diff --git a/Online/Dataflow/src/framework/DiskReader.cpp b/Online/Dataflow/src/framework/DiskReader.cpp index cb1295159..8fc1e2685 100644 --- a/Online/Dataflow/src/framework/DiskReader.cpp +++ b/Online/Dataflow/src/framework/DiskReader.cpp @@ -414,6 +414,7 @@ void DiskReader::updateRunNumber(int new_run) { if ( m_runSvcID ) ::dis_update_service(m_runSvcID); } +/// On pause: save rest of unprocessed data void DiskReader::save_file_rest(RawFile& file) { if ( file.isOpen() ) { if ( m_deleteFiles && m_saveRest ) file.saveRestOfFile(); @@ -432,7 +433,7 @@ DiskReader::load_event_data(RawFile::Allocator& allocator, RawFile& file) { if ( status < peek ) { file.reset(true); - return {0, -1, RawFile::NO_INPUT_TYPE}; + return { 0, -1, RawFile::NO_INPUT_TYPE }; } // // Auto detect data type: first check for PCIE40 MEP format @@ -444,66 +445,63 @@ DiskReader::load_event_data(RawFile::Allocator& allocator, RawFile& file) { status = file.read(data_ptr+peek, data_size-peek); if ( status < int(data_size-peek) ) { file.reset(true); - return {0, -1, RawFile::NO_INPUT_TYPE}; + return { 0, -1, RawFile::NO_INPUT_TYPE }; } - return {size_t(data_size), 1, RawFile::MEP_INPUT_TYPE}; + return { size_t(data_size), 1, RawFile::MEP_INPUT_TYPE }; } // // Now check for MDF format: 3 times same size auto* mdf_hdr = (EventHeader*)size; if ( mdf_hdr->is_mdf() ) { int num_evts = 1; - int data_size = size[0]; + int data_size = mdf_hdr->recordSize(); int alloc_size = m_packingFactor == 1 ? mdf_hdr->recordSize() : m_preAllocSize; if ( data_size > alloc_size ) { file.position(position); - return {0, -1, RawFile::NO_INPUT_TYPE}; + return { 0, -1, RawFile::NO_INPUT_TYPE }; } alloc_ptr = data_ptr = allocator(alloc_size); ::memcpy(data_ptr, mdf_hdr, peek); status = file.read(data_ptr + peek, mdf_hdr->recordSize() - peek); if ( status < mdf_hdr->recordSize() - peek ) { file.reset(true); - return {0, -1, RawFile::NO_INPUT_TYPE}; + return { 0, -1, RawFile::NO_INPUT_TYPE }; } data_ptr += size[0]; for ( ; num_evts < m_packingFactor; ) { - if ( peek + data_size > alloc_size ) { - file.reset(true); - return {size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE}; - } position = file.position(); status = file.read(data_ptr, peek); if ( status < peek ) { file.reset(true); - return {size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE}; + return { size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE }; } - mep_hdr = (pcie40::mep_header_t*)size; + mep_hdr = (pcie40::mep_header_t*)data_ptr; if ( mep_hdr->magic == mep_hdr->MagicPattern ) { file.position(position); - return {size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE}; + return { size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE }; } mdf_hdr = (EventHeader*)data_ptr; if ( mdf_hdr->recordSize() + data_size > alloc_size ) { file.position(position); - return {size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE}; + return { size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE }; } status = file.read(data_ptr + peek, mdf_hdr->recordSize() - peek); if ( status < mdf_hdr->recordSize() - peek ) { file.reset(true); - return {size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE}; + return { size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE }; } data_size += mdf_hdr->recordSize(); data_ptr += mdf_hdr->recordSize(); ++num_evts; } - return {size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE}; + return { size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE }; } // // Now the only thing left is Tell1 MEP format { static int id = -1; + static constexpr int peek_mep = peek - sizeof(int); int data_size = sizeof(MEPEVENT)+sizeof(Tell1MEP)+size[0]; MEPEVENT* e = (MEPEVENT*)(alloc_ptr=allocator(data_size)); Tell1MEP* me = (Tell1MEP*)e->data; @@ -520,15 +518,16 @@ DiskReader::load_event_data(RawFile::Allocator& allocator, RawFile& file) { e->events[j].status = EVENT_TYPE_OK; e->events[j].signal = 0; } - data_ptr = (unsigned char*)me->start(); - ::memcpy(data_ptr, size+1, 2*sizeof(size[0])); - status = file.read(data_ptr + 2*sizeof(size[0]), me->size() - 2*sizeof(int)); - if ( status < int(me->size() - 2*sizeof(int)) ) { // End-of file, continue with next file + data_ptr = (uint8_t*)me->start(); + ::memcpy(data_ptr, &size[1], peek_mep); + status = file.read(data_ptr + peek_mep, me->size() - peek_mep); + if ( status < int(me->size() - peek_mep) ) { + // End-of file, continue with next file file.reset(true); - return {0, -1, RawFile::NO_INPUT_TYPE}; + return { 0, -1, RawFile::NO_INPUT_TYPE }; } data_ptr += me->size(); - return {size_t(data_ptr-alloc_ptr), 1, RawFile::MEP_INPUT_TYPE}; + return { size_t(data_ptr-alloc_ptr), 1, RawFile::MEP_INPUT_TYPE }; } // // Last chance: we have a corrrupted MDF record. Try to sweep to the next MDF frame @@ -542,7 +541,7 @@ DiskReader::load_event_data(RawFile::Allocator& allocator, RawFile& file) { error("Corrupted file found: %s at position %ld. Skip rest of file!", file.cname(), position); file.reset(true); - return {0, -1, RawFile::NO_INPUT_TYPE}; + return { 0, -1, RawFile::NO_INPUT_TYPE }; } /// Declare event frame to the buffer manager to trigger clients @@ -556,7 +555,7 @@ int DiskReader::declare_mbm_data(MBM::BufferInfo* mbm_info, const char* fname, c // If the first bank is a MDF header, we copy the real trigger mask context.manager.setRunNumber(hdr->subHeader().H1->runNumber()); ::memcpy(dsc.mask, &hdr->subHeader().H1->triggerMask()[0], sizeof(dsc.mask)); - dsc.type = (load.packing == 1 && m_packingFactor) ? EVENT_TYPE_EVENT : EVENT_TYPE_BURST; + dsc.type = (1 == load.packing && 1 == m_packingFactor) ? EVENT_TYPE_EVENT : EVENT_TYPE_BURST; } else { // If MEP, we emulate a trigger mask with the partition ID diff --git a/Online/EventBuilding/options/BU_0.opts b/Online/EventBuilding/options/BU_0.opts index e210c041e..fc05a9c5c 100644 --- a/Online/EventBuilding/options/BU_0.opts +++ b/Online/EventBuilding/options/BU_0.opts @@ -1,3 +1,4 @@ #pragma print off #include "$EVENTBUILDINGROOT/options/EB_BU.opts" -MEPManager.Buffers = { "Events_0" }; +MEPManager.Buffers = { "Events_0" }; +BU.MBM_name = { "Events_0" }; diff --git a/Online/EventBuilding/options/BU_1.opts b/Online/EventBuilding/options/BU_1.opts index c0b8d80a4..33f12dc5b 100644 --- a/Online/EventBuilding/options/BU_1.opts +++ b/Online/EventBuilding/options/BU_1.opts @@ -1,3 +1,4 @@ #pragma print off #include "$EVENTBUILDINGROOT/options/EB_BU.opts" -MEPManager.Buffers = { "Events_1" }; +MEPManager.Buffers = { "Events_1" }; +BU.MBM_name = { "Events_1" }; diff --git a/Online/EventBuilding/options/EB_BU_algo.opts b/Online/EventBuilding/options/EB_BU_algo.opts index 4f407d2f0..59f599362 100755 --- a/Online/EventBuilding/options/EB_BU_algo.opts +++ b/Online/EventBuilding/options/EB_BU_algo.opts @@ -1,6 +1,6 @@ -BU.MBM_name = {}; +BU.MBM_name = { 'Input', 'Input', 'Input', 'Input', 'Input', 'Input' }; BU.buffer_size = {}; -BU.buffer_type = {}; +BU.buffer_type = { 2, 2, 2, 2, 2, 2 }; BU.out_file_prefix = ""; BU.shmem_prefix = ""; BU.stop_timeout = 0; diff --git a/Online/EventBuilding/options/EB_RU_algo.opts b/Online/EventBuilding/options/EB_RU_algo.opts index 3b6f57e29..ad70f5203 100755 --- a/Online/EventBuilding/options/EB_RU_algo.opts +++ b/Online/EventBuilding/options/EB_RU_algo.opts @@ -1,6 +1,6 @@ RU.MDF_filename = "/home/fpisani/mc_data/biger_file.mdf"; RU.PCIe40_ids = {}; -RU.PCIe40_names = { 'tdtel022_0', 'tdtel022_1', 'tdtel042_0', 'tdtel042_1', 'tdtel051_1', 'tdtel052_0', 'tdtel052_1', 'tdtel053_0', 'tdtel053_1' }; +RU.PCIe40_names = { 'tdtel022_0', 'tdtel022_1', 'tdtel031_0', 'tdtel031_1', 'tdtel032_0', 'tdtel033_0', 'tdtel033_1', 'tdtel042_0', 'tdtel042_1' }; RU.PCIe40_timeout = 30; RU.buffer_sizes = {}; RU.buffer_type = { 3, 3, 3, 3, 3, 3 }; diff --git a/Online/EventBuilding/options/EB_Transport_properties.opts b/Online/EventBuilding/options/EB_Transport_properties.opts index 85150ab08..f44bef70b 100755 --- a/Online/EventBuilding/options/EB_Transport_properties.opts +++ b/Online/EventBuilding/options/EB_Transport_properties.opts @@ -2,7 +2,7 @@ EB_transport.BU_ranks = {}; EB_transport.MPI_errors_return = FALSE; EB_transport.RU_ranks = {}; EB_transport.RUs_per_nic = {}; -EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.t25565.json"; +EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.d25197.json"; EB_transport.n_par_mess = 0; -EB_transport.n_sources_per_ru = { 1, 1, 1, 1, 3, 2 }; +EB_transport.n_sources_per_ru = { 1, 1, 3, 2, 1, 1 }; EB_transport.src_ids = {}; diff --git a/Online/EventBuilding/options/EB_transport.opts b/Online/EventBuilding/options/EB_transport.opts index 593c72269..63f024e4b 100755 --- a/Online/EventBuilding/options/EB_transport.opts +++ b/Online/EventBuilding/options/EB_transport.opts @@ -1,6 +1,3 @@ -// EB transport_unit properties -#include "$EVENTBUILDINGROOT/options/EB_Transport_properties.opts" - EB_transport.MPI_errors_return = true; EB_transport.OutputLevel = 3; // an empty array will trigger the default value in the code @@ -8,4 +5,7 @@ EB_transport.src_ids = {}; EB_transport.RU_ranks = {}; EB_transport.BU_ranks = {}; EB_transport.ghost_nodes_topology = {}; -EB_transport.ib_config_file = "$EVENTBUILDINGROOT/options/hosts_example.json"; +//EB_transport.ib_config_file = "$EVENTBUILDINGROOT/options/hosts_example.json"; + +// EB transport_unit properties +#include "$EVENTBUILDINGROOT/options/EB_Transport_properties.opts" diff --git a/Online/FarmConfig/job/EBPass.sh b/Online/FarmConfig/job/EBPass.sh new file mode 100755 index 000000000..a282a8614 --- /dev/null +++ b/Online/FarmConfig/job/EBPass.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# ========================================================================= +# +# Default script to start the buffer manager on the HLT farm worker node +# +# Author M.Frank +# Version: 1.0 +# Date: 05/03/2021 +# +# ========================================================================= +# +cat > /tmp/${UTGID}.py <<EOF +import Configurables +import GaudiOnline +import OnlineEnvBase +import os + +from Configurables import Online__FlowManager as FlowManager +application = GaudiOnline.Passthrough(outputLevel=OnlineEnvBase.OutputLevel, + partitionName=OnlineEnvBase.PartitionName, + partitionID=OnlineEnvBase.PartitionID, + classType=GaudiOnline.Class1) +items = os.environ['UTGID'].split('_') +id = items[-1] +application.setup_fifolog() +application.log(GaudiOnline.MSG_INFO,'+++ OnlineEnv: %s Level:%d',OnlineEnvBase.__file__,OnlineEnvBase.OutputLevel) +application.setup_mbm_access('Events_'+id, True) +writer = None +writer = application.setup_mbm_output('EventOutput') +writer.MBM_buffer = 'Output' +writer.MBM_maxConsumerWait = 10 +# +application.setup_hive(FlowManager("EventLoop"), 40) +application.setup_algorithms(writer, 0.05) +application.setup_monitoring() +application.monSvc.DimUpdateInterval = 1 +application.config.numEventThreads = 1 +application.config.MBM_numConnections = 1 +application.config.MBM_numEventThreads = 1 +application.config.MBM_requests = [ + 'EvType=3;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0', + 'EvType=2;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0', + 'EvType=1;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0' +] +EOF +export PYTHONPATH=/group/online/dataflow/options/${PARTITION}/HLT:${PYTHONPATH}; +exec -a ${UTGID} python `which gaudirun.py` /tmp/${UTGID}.py --application=OnlineEvents diff --git a/Online/FarmConfig/job/EBStorage.sh b/Online/FarmConfig/job/EBStorage.sh new file mode 100755 index 000000000..98e1eadf0 --- /dev/null +++ b/Online/FarmConfig/job/EBStorage.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# ========================================================================= +# +# Generic farm task startup script +# +# Author M.Frank +# Version: 1.0 +# Date: 20/05/2013 +# +# ========================================================================= +# echo "INFO_OPTIONS=${INFO_OPTIONS}"; +`dataflow_task Class1` -opts=../options/${TASK_TYPE}.opts; diff --git a/Online/FarmConfig/job/preamble.sh b/Online/FarmConfig/job/preamble.sh index c199d1077..966ee4fd7 100755 --- a/Online/FarmConfig/job/preamble.sh +++ b/Online/FarmConfig/job/preamble.sh @@ -10,6 +10,7 @@ export Class1_task="$gaudi_task -tasktype=LHCb::Class1Task -main=$OPTS/Main.opts export Class2_task="$gaudi_task -tasktype=LHCb::Class2Task -main=$OPTS/Main.opts " export Checkpoint_task="GaudiCheckpoint.exe libGaudiOnline.so OnlineTask -msgsvc=$msg_svc -tasktype=LHCb::Class1Task -main=$OPTS/Main.opts " # +# if test -p "${LOGFIFO}"; then export DATAFLOW_task="gentest.exe libDataflow.so dataflow_run_task -msg=fifo -mon=Dataflow_DIMMonitoring"; else @@ -17,6 +18,31 @@ else fi; # # +dataflow_specialized_options() +{ + if test -n "${1}"; then + echo "-opts=/group/online/dataflow/options/${PARTITION_NAME}/MONITORING/${1}.opts"; + else + echo "-opts=/group/online/dataflow/options/${PARTITION_NAME}/MONITORING/${UTGID}.opts"; + fi; +} +# +# +dataflow_default_options() +{ + echo "-opts=/group/online/dataflow/options/${PARTITION_NAME}/MONITORING/${UTGID}.opts"; +} +# +# +dataflow_task() +{ + if test -p "${LOGFIFO}"; then + echo "exec -a ${UTGID} gentest.exe libDataflow.so dataflow_run_task -msg=fifo -mon=Dataflow_DIMMonitoring -class=$1"; + else + echo "exec -a ${UTGID} gentest.exe libDataflow.so dataflow_run_task -msg=Dataflow_FmcLogger -mon=Dataflow_DIMMonitoring -class=$1"; + fi; +} +# # mbmmon() { diff --git a/Online/FarmConfig/options/EBMBM.opts b/Online/FarmConfig/options/EBMBM.opts index 6297f8c50..bbc5f2dae 100644 --- a/Online/FarmConfig/options/EBMBM.opts +++ b/Online/FarmConfig/options/EBMBM.opts @@ -6,4 +6,4 @@ Manager.Setup = {"Dataflow_MBMServer/MEPManager"}; Manager.Services = {"Dataflow_UI/UI"}; MEPManager.PartitionBuffers = @OnlineEnv.PartitionBuffers; MEPManager.PartitionName = @OnlineEnv.PartitionName; -MEPManager.InitFlags = "-s=1000000 -e=20 -u=30 -b=12 -t=2 -f -i=Events_0 -c -s=1000000 -e=20 -u=30 -b=12 -f -i=Events_1 -c"; +MEPManager.InitFlags = "-s=1000000 -e=20 -u=30 -b=12 -t=2 -f -i=Events_0 -c -s=1000000 -e=20 -u=30 -b=12 -f -i=Events_1 -c -s=500000 -e=30 -u=10 -b=12 -f -i=Output -c"; diff --git a/Online/FarmConfig/options/EBStorage.opts b/Online/FarmConfig/options/EBStorage.opts new file mode 100644 index 000000000..80419561c --- /dev/null +++ b/Online/FarmConfig/options/EBStorage.opts @@ -0,0 +1,42 @@ +#pragma print off +#include "$INFO_OPTIONS" +#include "$FARMCONFIGROOT/options/Logging.opts" +#include "$FARMCONFIGROOT/options/Monitoring.opts" +// +Manager.Services = {"Dataflow_MBMClient/MBM", + "Dataflow_MBMSelector/EventProc", + "Dataflow_RunableWrapper/Wrap" + }; +Manager.Algorithms = {"Dataflow_EmptyProcessor/Empty"}; +Manager.Runable = "Wrap"; +Wrap.Callable = "EventProc"; +// +EventProc.REQ1 = "EvType=1;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0"; +EventProc.REQ2 = "EvType=2;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0"; +EventProc.REQ3 = "EvType=3;TriggerMask=0xffffffff,0xffffffff,0xffffffff,0xffffffff;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0"; +EventProc.Input = "Output"; +// +// +OnlineEnv.PartitionBuffers = true; +MBM.PartitionBuffers = @OnlineEnv.PartitionBuffers; +MBM.PartitionName = @OnlineEnv.PartitionName; +MBM.PartitionID = @OnlineEnv.PartitionID; +MBM.Buffers = {"Output"}; +// +// +Manager.Algorithms = {"Dataflow_StorageWriter/Writer"}; +Writer.Server = "XXEB09.lbdaq.cern.ch:8000"; +Writer.Mount = "/objects/data/testing"; +Writer.FileSizeMB = 500; +Writer.MinFileSizeMB = 10; +Writer.WriteErrorRetry = 100000; +Writer.WriteErrorSleep = 3000; +Writer.FileNameFormat = 1; +Writer.PollTimeout = 100000; // micro-seconds +Writer.NumBuffers = 6; +Writer.NumThreads = 4; +Writer.CancelTimeout = 100; // seconds +Writer.ForceMDF = true; +// +Logger.OutputLevel = 0; +// diff --git a/Online/GaudiOnline/src/FileEventOutput.cpp b/Online/GaudiOnline/src/FileEventOutput.cpp index d43b087f3..0b84c2dfb 100644 --- a/Online/GaudiOnline/src/FileEventOutput.cpp +++ b/Online/GaudiOnline/src/FileEventOutput.cpp @@ -42,6 +42,7 @@ public: public: /// Default constructor file_transaction_t() = delete; + /// Initializing constructor file_transaction_t(const string& name, FileEventOutput* out) : file(name), output(out) {} /// Move constructor file_transaction_t(file_transaction_t&& copy) = default; -- GitLab From a6acccee0c7c08fd4c3e46c474a746026f404292 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Tue, 16 Mar 2021 19:57:12 +0100 Subject: [PATCH 07/26] Improve HTTP server functionality: Allow source info access from HttpRequest --- Online/HTTP/HTTP/HttpRequest.h | 11 +++++++++ Online/HTTP/src/HttpRequest.cpp | 27 +++++++++++++++++++++ Online/HTTP/src/HttpServer.cpp | 7 +++--- Online/Storage/src/server/fdb_db_server.cpp | 17 +++++++++++-- 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/Online/HTTP/HTTP/HttpRequest.h b/Online/HTTP/HTTP/HttpRequest.h index ea1c12d58..60ca66c61 100644 --- a/Online/HTTP/HTTP/HttpRequest.h +++ b/Online/HTTP/HTTP/HttpRequest.h @@ -33,6 +33,9 @@ // Framework include files #include <HTTP/HttpHeader.h> +// Boost inlcude files +#include <boost/asio.hpp> + // C/C++ include files #include <vector> #include <chrono> @@ -79,6 +82,8 @@ namespace http { reqheaders_t headers {}; /// The optional content sent with the request. content_t content {}; + /// Socket for the connection. + boost::asio::ip::tcp::socket* socket {nullptr}; /// Copy construction is allowed here HttpRequest(const HttpRequest&) = default; @@ -93,6 +98,12 @@ namespace http { /// Access header by name const HttpHeader* header(const std::string& name) const; + /// Access requester host endpoint + boost::asio::ip::tcp::endpoint remote() const; + /// Access requester remote address + boost::asio::ip::address remote_address() const; + /// Access requester remote host name + std::string remote_host_name() const; }; } // End namespace http #endif // HTTP_HTTP_HTTPREQUEST_H diff --git a/Online/HTTP/src/HttpRequest.cpp b/Online/HTTP/src/HttpRequest.cpp index 0b49a04ca..09772f12b 100644 --- a/Online/HTTP/src/HttpRequest.cpp +++ b/Online/HTTP/src/HttpRequest.cpp @@ -34,6 +34,8 @@ // C/C++ include files #include <cstring> +using namespace boost; + /// Access header by name const http::HttpHeader* http::HttpRequest::header(const std::string& name) const { for ( const auto& h : headers ) { @@ -42,3 +44,28 @@ const http::HttpHeader* http::HttpRequest::header(const std::string& name) const } return 0; } + +/// Access source host +asio::ip::tcp::endpoint http::HttpRequest::remote() const { + if ( this->socket ) { + system::error_code ec; + asio::ip::tcp::endpoint end = this->socket->remote_endpoint(ec); + if ( !ec ) { + return end; + } + throw std::runtime_error("HttpRequest::remote: Cannot determine remote " + "endpoint: "+ec.message()); + } + throw std::runtime_error("HttpRequest::remote: Invalid socket handle."); +} + +/// Access requester remote address +boost::asio::ip::address http::HttpRequest::remote_address() const { + return this->remote().address(); +} + +/// Access requester remote host name +std::string http::HttpRequest::remote_host_name() const { + return this->remote().address().to_string(); +} + diff --git a/Online/HTTP/src/HttpServer.cpp b/Online/HTTP/src/HttpServer.cpp index cb5bdd316..45c464f6e 100644 --- a/Online/HTTP/src/HttpServer.cpp +++ b/Online/HTTP/src/HttpServer.cpp @@ -50,9 +50,10 @@ HttpConnection::HttpConnection(asio::ip::tcp::socket s, { if ( buffer_size > 0 ) buffer.resize(buffer_size); else buffer.resize(16*1024); - request.state = HttpRequestParser::indeterminate; - request.start = system_clock::now(); - request.netio = std::chrono::time_point<system_clock>(system_clock::duration::zero()); + request.socket = &this->socket; + request.state = HttpRequestParser::indeterminate; + request.start = system_clock::now(); + request.netio = std::chrono::time_point<system_clock>(system_clock::duration::zero()); request.handling = std::chrono::time_point<system_clock>(system_clock::duration::zero()); } diff --git a/Online/Storage/src/server/fdb_db_server.cpp b/Online/Storage/src/server/fdb_db_server.cpp index 5f2b63cbb..3ec17e764 100644 --- a/Online/Storage/src/server/fdb_db_server.cpp +++ b/Online/Storage/src/server/fdb_db_server.cpp @@ -30,7 +30,8 @@ using namespace std; using namespace Online::storage; static const string PATTERN_NEXT = "/next?prefix="; - +static constexpr const double kByte = 1024e0; +static constexpr const double MByte = kByte*kByte; /// Standard destructor template <> http::basic_http_server<db>::handler_t::~handler_t() { @@ -58,14 +59,18 @@ http::basic_http_server<db>::handler_t::handle_get(const request_t& req, reply_t if ( !error_code_ok(ec, this->debug) ) { header_t h(http::constants::error_cause,"Failed to get "+req.uri+" ["+ec.message()+"]"); - ::lib_rtl_output(LIB_RTL_ERROR,"DELETE: %s %-20s = %s", + ::lib_rtl_output(LIB_RTL_ERROR,"GET: %s %-20s = %s", reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); rep.headers.emplace_back(move(h)); } else { + bool mb = length > 3*MByte; rep = reply_t::stock_reply(reply_t::temp_redirect); rep.headers.emplace_back(http::constants::location,access_name); + ::lib_rtl_output(LIB_RTL_INFO,"GET: '%s' %.1f %cB [%s]", + access_name.c_str(), double(length)/(mb ? MByte : kByte), mb ? 'M' : 'k', + req.remote_address().to_string().c_str()); } return write; } @@ -98,8 +103,12 @@ http::basic_http_server<db>::handler_t::handle_delete(const request_t& req, repl rep.headers.emplace_back(move(h)); } else { + bool mb = length > 3*MByte; rep = reply_t::stock_reply(reply_t::temp_redirect); rep.headers.emplace_back(http::constants::location,access_name); + ::lib_rtl_output(LIB_RTL_INFO,"DELETE:'%s' %.1f %cB [%s]", + access_name.c_str(), double(length)/(mb ? MByte : kByte), mb ? 'M' : 'k', + req.remote_address().to_string().c_str()); } return write; } @@ -148,12 +157,16 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t rep.headers.emplace_back(move(h)); } else { + bool mb = len > 3*MByte; rep = reply_t::stock_reply(reply_t::temp_redirect); rep.content.clear(); rep.headers.clear(); rep.headers.emplace_back(http::constants::location,access_name); rep.headers.emplace_back(http::constants::data_length,len); rep.headers.emplace_back(http::constants::date,date); + ::lib_rtl_output(LIB_RTL_INFO,"PUT: '%s' %.1f %cB [%s]", + req.uri.c_str(), double(len)/(mb ? MByte : kByte), mb ? 'M' : 'k', + req.remote_address().to_string().c_str()); } } return write; -- GitLab From 5d343b14206258fa86a3c6f5aec87253ca4c32fb Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Tue, 23 Mar 2021 18:58:38 +0100 Subject: [PATCH 08/26] Results from event building debugging --- Online/Dataflow/options/StorageWriter.opts | 7 +- Online/Dataflow/src/Storage/StorageWriter.cpp | 4 +- Online/Dataflow/src/framework/Receiver.cpp | 8 +- Online/EventBuilding/EventBuilding/BU.hpp | 1 + Online/EventBuilding/include/mbm_writer.hpp | 1 + .../EventBuilding/include/mbm_writer_impl.hpp | 20 +- Online/EventBuilding/options/EB_RU_algo.opts | 2 +- .../options/EB_Transport_properties.opts | 4 +- Online/EventBuilding/src/BU.cpp | 51 +-- Online/EventBuilding/src/pcie40_reader.cpp | 10 +- Online/FarmConfig/job/Controller.sh | 2 +- Online/FarmConfig/job/EBPass.sh | 35 +- Online/FarmConfig/options/EBMBM.opts | 2 +- .../GaudiOnline/components/EventProcessor.cpp | 11 +- Online/GaudiOnline/components/InputAlg.cpp | 28 +- Online/GaudiOnline/components/InputAlg.h | 7 + .../GaudiOnline/components/OnlineEventApp.cpp | 19 + Online/GaudiOnline/components/Passthrough.cpp | 12 +- Online/GaudiOnline/src/EventAccess.cpp | 25 +- Online/OnlineBase/src/MBM/mbmlib_client.cpp | 4 +- Online/OnlineBase/src/MBM/mbmlib_server.cpp | 3 +- Online/PCIE40Data/PCIE40Data/RawBank40.h | 175 ++++++---- Online/PCIE40Data/PCIE40Data/pcie40.h | 74 ++-- Online/PCIE40Data/PCIE40Data/pcie40_writer.h | 10 +- Online/PCIE40Data/src/pcie40decoder.cpp | 107 +++--- Online/Storage/Storage/fdb_dbase.h | 113 ++++-- Online/Storage/Storage/fdb_server.h | 12 +- Online/Storage/src/server/fdb_SQLite.cpp | 328 ++++++++---------- Online/Storage/src/server/fdb_SQLite.h | 135 ++----- Online/Storage/src/server/fdb_SQLite_bind.cpp | 56 +-- .../Storage/src/server/fdb_SQLite_direct.cpp | 147 -------- Online/Storage/src/server/fdb_db_server.cpp | 32 +- Online/Storage/src/server/fdb_dbase.cpp | 217 ++++++++++++ Online/Storage/src/server/sqlite_test.cpp | 91 ++--- Online/Tell1Data/Tell1Data/Tell1Bank.h | 178 +++++----- 35 files changed, 1055 insertions(+), 876 deletions(-) create mode 100644 Online/Storage/src/server/fdb_dbase.cpp diff --git a/Online/Dataflow/options/StorageWriter.opts b/Online/Dataflow/options/StorageWriter.opts index cd7003a10..1bb434963 100644 --- a/Online/Dataflow/options/StorageWriter.opts +++ b/Online/Dataflow/options/StorageWriter.opts @@ -27,15 +27,16 @@ MBM.Buffers = {"Events"}; Manager.Algorithms = {"Dataflow_StorageWriter/Writer"}; Writer.Server = "XXEB09.lbdaq.cern.ch:8000"; Writer.Mount = "/objects/data/testing"; -Writer.FileSizeMB = 800; Writer.FileSizeMB = 40; +Writer.FileSizeMB = 800; +Writer.FileSizeMB = 100; Writer.MinFileSizeMB = 10; Writer.WriteErrorRetry = 100000; Writer.WriteErrorSleep = 3000; Writer.FileNameFormat = 1; Writer.PollTimeout = 100000; // micro-seconds -Writer.NumBuffers = 6; -Writer.NumThreads = 4; +Writer.NumBuffers = 5; +Writer.NumThreads = 3; Writer.CancelTimeout = 100; // seconds Writer.ForceMDF = true; // diff --git a/Online/Dataflow/src/Storage/StorageWriter.cpp b/Online/Dataflow/src/Storage/StorageWriter.cpp index c8b61a392..e5851dd8d 100644 --- a/Online/Dataflow/src/Storage/StorageWriter.cpp +++ b/Online/Dataflow/src/Storage/StorageWriter.cpp @@ -419,9 +419,9 @@ string StorageWriter::makeFileName() const { ::gmtime_r(&now, &tm); ::strftime(txt1,sizeof(txt1),"%Y-%m-%d__%H-%M",&tm); ::strftime(txt2,sizeof(txt2),"%H-%M-%S",&tm); - ::snprintf(fname, sizeof(fname), "%s/%010d/%s/%010d_%s_%s_%d.raw", + ::snprintf(fname, sizeof(fname), "%s/%010d/%s/%010d_%s_%s_%06d_%d.raw", m_mount.c_str(), m_curr_run, txt1, m_curr_run, - RTL::nodeNameShort().c_str(), txt2, ++seq); + RTL::nodeNameShort().c_str(), txt2, ::lib_rtl_pid(), ++seq); break; } case 0: // Fall-through diff --git a/Online/Dataflow/src/framework/Receiver.cpp b/Online/Dataflow/src/framework/Receiver.cpp index 42d0d644f..aa4954290 100755 --- a/Online/Dataflow/src/framework/Receiver.cpp +++ b/Online/Dataflow/src/framework/Receiver.cpp @@ -59,10 +59,10 @@ int Receiver::initialize() { if ( DF_SUCCESS == sc ) { try { m_recvEvents = true; - declareMonitor("Events","IN", m_recvReq=0, "Total number of events received."); - declareMonitor("Events","OUT", m_evtDecl=0, "Total number of events declared to MBM."); - declareMonitor("ErrorsIn", m_recvError=0, "Total number of receive errors."); - declareMonitor("BytesIn", m_recvBytes=0, "Total number of bytes received from clients."); + declareMonitor("Events", "IN", m_recvReq=0, "Total number of events received."); + declareMonitor("Events", "OUT", m_evtDecl=0, "Total number of events declared to MBM."); + declareMonitor("ErrorsIn", m_recvError=0, "Total number of receive errors."); + declareMonitor("BytesIn", m_recvBytes=0, "Total number of bytes received from clients."); if ( 0 == context.mbm ) return error("Failed to access buffer manager service."); else if ( DF_SUCCESS != (sc=subscribeNetwork()) ) diff --git a/Online/EventBuilding/EventBuilding/BU.hpp b/Online/EventBuilding/EventBuilding/BU.hpp index 68a63dd46..7a95bd047 100644 --- a/Online/EventBuilding/EventBuilding/BU.hpp +++ b/Online/EventBuilding/EventBuilding/BU.hpp @@ -114,6 +114,7 @@ namespace EB { int _MEP_count = 0; int _discarted_MEP_count = 0; int _incomplete_MEP_count = 0; + int _corrupted_MEP_count = 0; std::vector<int> _incomplete_MEP_srcs; buffer_interface::Buffer_writer<MEP::MEP_header>* _recv_buff; diff --git a/Online/EventBuilding/include/mbm_writer.hpp b/Online/EventBuilding/include/mbm_writer.hpp index 9f67343b8..b1152826f 100644 --- a/Online/EventBuilding/include/mbm_writer.hpp +++ b/Online/EventBuilding/include/mbm_writer.hpp @@ -38,6 +38,7 @@ public: protected: Online::DataflowComponent::Context* _context; std::unique_ptr<MBM::Producer> _producer; + }; #include "mbm_writer_impl.hpp" diff --git a/Online/EventBuilding/include/mbm_writer_impl.hpp b/Online/EventBuilding/include/mbm_writer_impl.hpp index 2251637f0..732082a66 100644 --- a/Online/EventBuilding/include/mbm_writer_impl.hpp +++ b/Online/EventBuilding/include/mbm_writer_impl.hpp @@ -57,7 +57,7 @@ T* Mbm_writer<T>::try_write_next_element(size_t size) status = _producer->getSpaceTry(size); - if (status == MBM_NORMAL) { + if(status == MBM_NORMAL){ MBM::EventDesc& dsc = _producer->event(); ret_val = reinterpret_cast<T*>(dsc.data); dsc.len = size; @@ -68,15 +68,6 @@ T* Mbm_writer<T>::try_write_next_element(size_t size) dsc.mask[1] = ~0x0; dsc.mask[2] = ~0x0; dsc.mask[3] = ~0x0; - - // TODO move this to write_complete - status = _producer->declareEvent(); - if (status != MBM_NORMAL) { - // TODO add proper error handling, like cancelling pending requests - std::ostringstream err_mess; - err_mess << "Mbm_writer: write_complete declareEvent error " << status; - throw std::runtime_error(err_mess.str()); - } } else if (status == MBM_NO_ROOM) { ret_val = NULL; } else { @@ -91,6 +82,7 @@ T* Mbm_writer<T>::try_write_next_element(size_t size) template<class T> void Mbm_writer<T>::write_complete() { + // TODO status and ret_Val should consistent int status; if (!this->is_set()) { std::ostringstream err_mess; @@ -98,6 +90,14 @@ void Mbm_writer<T>::write_complete() throw std::logic_error(err_mess.str()); } + status = _producer->declareEvent(); + if (status != MBM_NORMAL) { + // TODO add proper error handling, like cancelling pending requests + std::ostringstream err_mess; + err_mess << "Mbm_writer: write_complete declareEvent error " << status; + throw std::runtime_error(err_mess.str()); + } + status = _producer->sendSpace(); if (status != MBM_NORMAL) { // TODO add proper error handling, like cancelli pending requests diff --git a/Online/EventBuilding/options/EB_RU_algo.opts b/Online/EventBuilding/options/EB_RU_algo.opts index ad70f5203..793a81403 100755 --- a/Online/EventBuilding/options/EB_RU_algo.opts +++ b/Online/EventBuilding/options/EB_RU_algo.opts @@ -1,6 +1,6 @@ RU.MDF_filename = "/home/fpisani/mc_data/biger_file.mdf"; RU.PCIe40_ids = {}; -RU.PCIe40_names = { 'tdtel022_0', 'tdtel022_1', 'tdtel031_0', 'tdtel031_1', 'tdtel032_0', 'tdtel033_0', 'tdtel033_1', 'tdtel042_0', 'tdtel042_1' }; +RU.PCIe40_names = { 'tdtel022_0', 'tdtel022_1', 'tdtel041_0', 'tdtel041_1', 'tdtel042_0', 'tdtel042_1', 'tdtel043_0', 'tdtel043_1', 'tdtel062_0', 'tdtel062_1', 'tdtel063_0', 'tdtel063_1' }; RU.PCIe40_timeout = 30; RU.buffer_sizes = {}; RU.buffer_type = { 3, 3, 3, 3, 3, 3 }; diff --git a/Online/EventBuilding/options/EB_Transport_properties.opts b/Online/EventBuilding/options/EB_Transport_properties.opts index f44bef70b..9ce1e0eb8 100755 --- a/Online/EventBuilding/options/EB_Transport_properties.opts +++ b/Online/EventBuilding/options/EB_Transport_properties.opts @@ -2,7 +2,7 @@ EB_transport.BU_ranks = {}; EB_transport.MPI_errors_return = FALSE; EB_transport.RU_ranks = {}; EB_transport.RUs_per_nic = {}; -EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.d25197.json"; +EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.J25197.json"; EB_transport.n_par_mess = 0; -EB_transport.n_sources_per_ru = { 1, 1, 3, 2, 1, 1 }; +EB_transport.n_sources_per_ru = { 1, 1, 3, 3, 2, 2 }; EB_transport.src_ids = {}; diff --git a/Online/EventBuilding/src/BU.cpp b/Online/EventBuilding/src/BU.cpp index 3b4ecc4bd..d16903321 100644 --- a/Online/EventBuilding/src/BU.cpp +++ b/Online/EventBuilding/src/BU.cpp @@ -36,6 +36,7 @@ EB::BU::BU(const std::string& nam, DataflowContext& ctxt) : Transport_unit(nam, declareMonitor("MEP_counter", _MEP_count, "Number of build MEPs in the RUN"); declareMonitor("discarted_MEP_counter", _discarted_MEP_count, "Number of discarted MEPs in the RUN"); declareMonitor("incomplete_MEP_counter", _incomplete_MEP_count, "Number of incomplete MEPs in the RUN"); + declareMonitor("corrupted_MEP_counter", _corrupted_MEP_count, "Number of corrupted MEPs in the RUN"); logger.set_name(name); } @@ -431,31 +432,34 @@ int EB::BU::run() // At the end of the run there is no MEP to check if (!_end_of_run) { - check_MEP(); - - if (logger.is_active(Online::PrintLevel::DEBUG)) { - logger.debug << "active MEP\n"; - if (logger.is_active(Online::PrintLevel::VERBOSE)) { - _curr_mep->print(logger.verbose, true); - logger.verbose << std::flush; - } else { - _curr_mep->print(logger.debug, false); + if(check_MEP()){ + if (logger.is_active(Online::PrintLevel::DEBUG)) { + logger.debug << "active MEP\n"; + if (logger.is_active(Online::PrintLevel::VERBOSE)) { + _curr_mep->print(logger.verbose, true); + logger.verbose << std::flush; + } else { + _curr_mep->print(logger.debug, false); + } + logger.debug << std::flush; } - logger.debug << std::flush; - } - if (_write_to_file[_my_idx]) { - // TODO add error checking - _file_writer.write(_curr_mep); - } + if (_write_to_file[_my_idx]) { + // TODO add error checking + _file_writer.write(_curr_mep); + } - _curr_mep = NULL; - if (_discarted) { - _discard_buff->write_complete(); - _discarted_MEP_count++; + _curr_mep = NULL; + if (_discarted) { + _discard_buff->write_complete(); + _discarted_MEP_count++; + } else { + _recv_buff->write_complete(); + _MEP_count++; + } } else { - _recv_buff->write_complete(); - _MEP_count++; + logger.error << " CORRUPTED MEP received" << std::flush; + _corrupted_MEP_count++; } } @@ -651,7 +655,7 @@ bool EB::BU::check_MEP() reinterpret_cast<MFP::MFP_header*>(mep_start + _data_offset_words[k] * MEP::MEP_WORD_SIZE); if (!header->is_valid()) { logger.error << "corrupted MEP: invalid header\n"; - _curr_mep->print(logger.error, true); + _curr_mep->print(logger.debug, true); logger.error << std::flush; return false; } @@ -660,7 +664,7 @@ bool EB::BU::check_MEP() curr_ev_id = header->ev_id; } else if (curr_ev_id != header->ev_id) { logger.error << "corrupted MEP: mixed ev ids got " << header->ev_id << " expected " << curr_ev_id << "\n"; - _curr_mep->print(logger.error, true); + _curr_mep->print(logger.debug, true); logger.error << std::flush; return false; } @@ -735,6 +739,7 @@ void EB::BU::reset_counters() logger.info << "resetting counters" << std::flush; _MEP_count = 0; _discarted_MEP_count = 0; + _corrupted_MEP_count = 0; _incomplete_MEP_count = 0; std::fill(_incomplete_MEP_srcs.begin(), _incomplete_MEP_srcs.end(), 0); } diff --git a/Online/EventBuilding/src/pcie40_reader.cpp b/Online/EventBuilding/src/pcie40_reader.cpp index 9794750a6..7edbfb863 100644 --- a/Online/EventBuilding/src/pcie40_reader.cpp +++ b/Online/EventBuilding/src/pcie40_reader.cpp @@ -452,6 +452,14 @@ int PCIe40_frag_reader::build_MFP() get_padding(_size_list.back(), 1 << _align); MFP::MFP_header* new_header = _internal_buffer_writer.try_write_next_element(MFP_size); if (new_header != NULL) { + if((_frag_list.size() != _type_list.size()) || (_frag_list.size() != _size_list.size())){ + bool flag = true; + while(flag){ + sleep(1); + } + } + + new_header->magic = MFP::MFP_magic; new_header->n_banks = _frag_list.size(); new_header->packet_size = MFP_size; @@ -520,4 +528,4 @@ void PCIe40_frag_reader::cancel() std::vector<std::tuple<void*, size_t>> PCIe40_frag_reader::get_full_buffer() { return _internal_buffer_reader.get_full_buffer(); -} \ No newline at end of file +} diff --git a/Online/FarmConfig/job/Controller.sh b/Online/FarmConfig/job/Controller.sh index da768bfa6..548fde6fe 100755 --- a/Online/FarmConfig/job/Controller.sh +++ b/Online/FarmConfig/job/Controller.sh @@ -37,7 +37,7 @@ if test -z "${DIM_DNS_NODE}"; then fi; # exec -a ${UTGID} `which gentest.exe` libSmiController.so smi_controller \ - -print=3 \ + -print=4 \ -logger=fifo \ -part=${PARTITION_NAME} \ -dns=${DIM_DNS_NODE} \ diff --git a/Online/FarmConfig/job/EBPass.sh b/Online/FarmConfig/job/EBPass.sh index a282a8614..c0b2fc734 100755 --- a/Online/FarmConfig/job/EBPass.sh +++ b/Online/FarmConfig/job/EBPass.sh @@ -9,39 +9,46 @@ # # ========================================================================= # -cat > /tmp/${UTGID}.py <<EOF +rm -rf /tmp/${UTGID}.py; +# +cat > /tmp/${UTGID}.py <</EOF import Configurables import GaudiOnline -import OnlineEnvBase +import OnlineEnvBase as OnlineEnv import os from Configurables import Online__FlowManager as FlowManager -application = GaudiOnline.Passthrough(outputLevel=OnlineEnvBase.OutputLevel, - partitionName=OnlineEnvBase.PartitionName, - partitionID=OnlineEnvBase.PartitionID, +application = GaudiOnline.Passthrough(outputLevel=OnlineEnv.OutputLevel, + partitionName=OnlineEnv.PartitionName, + partitionID=OnlineEnv.PartitionID, classType=GaudiOnline.Class1) items = os.environ['UTGID'].split('_') id = items[-1] application.setup_fifolog() -application.log(GaudiOnline.MSG_INFO,'+++ OnlineEnv: %s Level:%d',OnlineEnvBase.__file__,OnlineEnvBase.OutputLevel) +application.log(GaudiOnline.MSG_INFO,'+++ OnlineEnv: %s Level:%d',OnlineEnv.__file__,OnlineEnv.OutputLevel) application.setup_mbm_access('Events_'+id, True) writer = None writer = application.setup_mbm_output('EventOutput') writer.MBM_buffer = 'Output' writer.MBM_maxConsumerWait = 10 # -application.setup_hive(FlowManager("EventLoop"), 40) -application.setup_algorithms(writer, 0.05) +application.setup_hive(FlowManager("EventLoop"), 30) +application.setup_algorithms(writer=writer, acceptRate=0.01) application.setup_monitoring() +# +# application.monSvc.DimUpdateInterval = 1 application.config.numEventThreads = 1 -application.config.MBM_numConnections = 1 -application.config.MBM_numEventThreads = 1 +application.config.MBM_numConnections = 7 +application.config.MBM_numEventThreads = 8 application.config.MBM_requests = [ - 'EvType=3;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0', - 'EvType=2;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0', - 'EvType=1;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=USER;Frequency=PERC;Perc=100.0' + 'EvType=3;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0', + 'EvType=2;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0', + 'EvType=1;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0' ] -EOF +/EOF +# +# export PYTHONPATH=/group/online/dataflow/options/${PARTITION}/HLT:${PYTHONPATH}; +export RTL_SLEEP_ON_FATAL=1; exec -a ${UTGID} python `which gaudirun.py` /tmp/${UTGID}.py --application=OnlineEvents diff --git a/Online/FarmConfig/options/EBMBM.opts b/Online/FarmConfig/options/EBMBM.opts index bbc5f2dae..cd3a10d31 100644 --- a/Online/FarmConfig/options/EBMBM.opts +++ b/Online/FarmConfig/options/EBMBM.opts @@ -6,4 +6,4 @@ Manager.Setup = {"Dataflow_MBMServer/MEPManager"}; Manager.Services = {"Dataflow_UI/UI"}; MEPManager.PartitionBuffers = @OnlineEnv.PartitionBuffers; MEPManager.PartitionName = @OnlineEnv.PartitionName; -MEPManager.InitFlags = "-s=1000000 -e=20 -u=30 -b=12 -t=2 -f -i=Events_0 -c -s=1000000 -e=20 -u=30 -b=12 -f -i=Events_1 -c -s=500000 -e=30 -u=10 -b=12 -f -i=Output -c"; +MEPManager.InitFlags = "-s=1000000 -e=50 -u=20 -b=12 -t=1 -n=0 -f -i=Events_0 -c -s=1000000 -e=50 -u=20 -b=12 -t=1 -n=1 -f -i=Events_1 -c -s=500000 -e=50 -u=5 -b=12 -t=1 -f -i=Output -c"; diff --git a/Online/GaudiOnline/components/EventProcessor.cpp b/Online/GaudiOnline/components/EventProcessor.cpp index 0f539d242..0f8a0f90e 100644 --- a/Online/GaudiOnline/components/EventProcessor.cpp +++ b/Online/GaudiOnline/components/EventProcessor.cpp @@ -121,10 +121,15 @@ namespace Online { for(size_t i=0; i < event->num_bank_collections(); ++i) { const auto* bc = event->bank_collection(i); for(const auto* b=bc->begin(); b != bc->end(); b = bc->next(b)) { - result->adoptBank((LHCb::RawBank*)b, false); + if ( b->type() < b->LastType ) { + result->adoptBank((LHCb::RawBank*)b, false); + } + } + for(const auto* b=bc->special_begin(); b != bc->special_end(); b = bc->next(b)) { + if ( b->type() < b->LastType ) { + result->adoptBank((LHCb::RawBank*)b, false); + } } - for(const auto* b=bc->special_begin(); b != bc->special_end(); b = bc->next(b)) - result->adoptBank((LHCb::RawBank*)b, false); } return result; } diff --git a/Online/GaudiOnline/components/InputAlg.cpp b/Online/GaudiOnline/components/InputAlg.cpp index 04ca7fdeb..98a8a434f 100644 --- a/Online/GaudiOnline/components/InputAlg.cpp +++ b/Online/GaudiOnline/components/InputAlg.cpp @@ -56,6 +56,28 @@ StatusCode Online::InputAlg::stop() { return StatusCode::SUCCESS; } +/// Add error banks to the data service +Online::InputAlg::daq_error_t +Online::InputAlg::get_errors(const event_traits::record_t& record) const { + daq_error_t banks; + auto* event = record.tell40_event; + for(size_t i=0; i < event->num_bank_collections(); ++i) { + const auto* bc = event->bank_collection(i); + for(const auto* b=bc->begin(); b != bc->end(); b = bc->next(b)) { + if ( b->type() >= RawBank40::DaqErrorBase ) { + banks.emplace_back(b); + } + } + for(const auto* b=bc->special_begin(); b != bc->special_end(); b = bc->next(b)) { + if ( b->type() < b->LastType ) { + banks.emplace_back(b); + } + } + } + return banks; +} + + /// Execute single event StatusCode Online::InputAlg::process(EventContext const& /* ctxt */) const { EventAccess::event_t e(m_io->pop()); @@ -65,7 +87,11 @@ StatusCode Online::InputAlg::process(EventContext const& /* ctxt */) const { event = create_event<event_traits::tell1>(e.second->at(e.first)); } else if ( e.second->type() == event_traits::tell40::data_type ) { - event = create_event<event_traits::tell40>(e.second->at(e.first)); + const auto& record = e.second->at(e.first); + event = create_event<event_traits::tell40>(record); + if ( m_declareData.value() ) { + m_daqError.put(move(get_errors(record))); + } } if ( m_declareData.value() ) { m_rawEvent.put(move(event)); diff --git a/Online/GaudiOnline/components/InputAlg.h b/Online/GaudiOnline/components/InputAlg.h index a920a5638..b4a99e369 100644 --- a/Online/GaudiOnline/components/InputAlg.h +++ b/Online/GaudiOnline/components/InputAlg.h @@ -32,12 +32,16 @@ namespace Online { class InputAlg : public EventProcessor { protected: + typedef std::vector<const RawBank40*> daq_error_t; + Gaudi::Property<bool> m_declareData{this,"DeclareData", true, "Declare or drop data"}; Gaudi::Property<bool> m_enableHalt{this, "EnableHalt", true, "Enable halt the event loop"}; Gaudi::Property<std::string> m_ioService{this, "IOServiceName","Online::IOService/IOService","Name of IO service"}; DataObjectWriteHandle<LHCb::RawEvent> m_rawEvent{this, "RawLocation", LHCb::RawEventLocation::Default}; DataObjectWriteHandle<evt_desc_t> m_rawGuard{this, "RawGuard", LHCb::RawEventLocation::Default+"Guard"}; + DataObjectWriteHandle<daq_error_t> m_daqError{this, "DAQErrors", LHCb::RawEventLocation::Default+"DAQErrors"}; + /// Reference to the I/O service SmartIF<IOService> m_io; @@ -46,6 +50,9 @@ namespace Online { /// Allow to escape halt mutable int m_halt; + /// Add error banks to the data service + daq_error_t get_errors(const event_traits::record_t& record) const; + public: using EventProcessor::EventProcessor; /// Initialize the algorithm diff --git a/Online/GaudiOnline/components/OnlineEventApp.cpp b/Online/GaudiOnline/components/OnlineEventApp.cpp index 67f495c91..16cb0324e 100644 --- a/Online/GaudiOnline/components/OnlineEventApp.cpp +++ b/Online/GaudiOnline/components/OnlineEventApp.cpp @@ -259,6 +259,25 @@ int OnlineEventApp::startApplication() { for(size_t i=0; i<m_config->mbm.num_event_threads; ++i) m_eventAccess.emplace_back(make_unique<thread>(&OnlineEventApp::event_access,ref(*this),i)); +#if 0 + static bool first = true; + if ( first ) { + static sa_sigaction new_act; + static sa_sigaction old_act; + first = false; + new_act.sa_handler = nullptr; + new_act.sa_sigaction = segv_handler; + new_act.sa_flags = SA_SIGINFO; + new_act.sa_mask = 0; + new_act.sa_restorer = nullptr; + sigemptyset(&new_act.sa_mask); + sigaddset(&new_act.sa_mask,SIGSEGV); + + if ( 0 != ::sigaction(SIGSEGV, &new_act, &old_act) ) { + m_logger->error("+++ Failed to install SEGV handler."); + } + } +#endif return status; } return m_logger->error("+++ Inconsistent thread state! [FSM failure]"); diff --git a/Online/GaudiOnline/components/Passthrough.cpp b/Online/GaudiOnline/components/Passthrough.cpp index 32387b86e..5393fb9a2 100644 --- a/Online/GaudiOnline/components/Passthrough.cpp +++ b/Online/GaudiOnline/components/Passthrough.cpp @@ -145,11 +145,13 @@ StatusCode Passthrough::execute(EventContext const& ctxt) const { } /// If mothing is specified everything will be downscaled - if ( m_downScale.empty() && m_rate < 1.0 ) { - double frac = double(::rand()) / double(RAND_MAX); - if ( m_rate < frac ) { - execState(ctxt).setFilterPassed(false); - return _halt(delay); + if ( m_downScale.empty() && m_triggerTypesToPass.empty() && m_calibTypesToPass.empty() ) { + if ( m_rate < 1.0 ) { + double frac = double(::rand()) / double(RAND_MAX); + if ( m_rate < frac ) { + execState(ctxt).setFilterPassed(false); + return _halt(delay); + } } execState(ctxt).setFilterPassed(true); return _halt(delay); diff --git a/Online/GaudiOnline/src/EventAccess.cpp b/Online/GaudiOnline/src/EventAccess.cpp index 4d095c1b5..6b7c2b547 100644 --- a/Online/GaudiOnline/src/EventAccess.cpp +++ b/Online/GaudiOnline/src/EventAccess.cpp @@ -247,25 +247,35 @@ namespace { /// The event is a PCIE40 MEP structure with multiple events, which must be decoded std::unique_ptr<pcie40::event_collection_t> EventAccess::convertPCIE40MEP(datapointer_t start, size_t len) const { - datapointer_t end = start + len; std::unique_ptr<pcie40::event_collection_t> events; + datapointer_t end = start + len; + pcie40::decoder_t decoder; + if ( end > start ) { - pcie40::decoder_t decoder; static constexpr size_t max_bank = 20000; static constexpr size_t max_source = 2000; static constexpr size_t max_packing = 35000; const auto* pmep_hdr = reinterpret_cast<const pcie40::mep_header_t*>(start); if ( 0 == pmep_hdr->size ) { m_logger->error("MEP with invalid size encountered: %d", pmep_hdr->size); + decoder.initialize(events,0); return events; } if ( 0 == pmep_hdr->num_source || pmep_hdr->num_source > max_source ) { m_logger->error("MEP with invalid number of sources encountered: %d", pmep_hdr->num_source); + decoder.initialize(events,0); return events; } - uint32_t packing = pmep_hdr->multi_fragment(0)->header.packing; + uint32_t packing = pmep_hdr->multi_fragment(0)->header.packing; if ( 0 == packing || packing > max_packing ) { - m_logger->error("MEP with invalid packing factor encountered: %d", packing); + decoder.initialize(events,0); + try { + throw std::runtime_error("MEP with invalid packing factor encountered"); + } + catch(...) { + m_logger->error("MEP with invalid packing factor encountered: %d", packing); + return events; + } return events; } /// Header checks done. Now decode the buffer @@ -283,6 +293,12 @@ EventAccess::convertPCIE40MEP(datapointer_t start, size_t len) const { } ++num_bank; } + for( const pcie40::bank_t* b=coll->special_begin(); b != coll->special_end(); b=coll->next(b) ) { + if ( b->magic() != pcie40::bank_t::MagicPattern ) { + _error("Bad magic pattern in raw bank!\n"); + } + ++num_bank; + } } if ( num_bank == 0 ) { m_logger->error("Event with ZERO banks encountered"); @@ -294,6 +310,7 @@ EventAccess::convertPCIE40MEP(datapointer_t start, size_t len) const { return events; } m_logger->error("convertMultiMDF: Encountered empty MDF burst!"); + decoder.initialize(events,0); return events; } diff --git a/Online/OnlineBase/src/MBM/mbmlib_client.cpp b/Online/OnlineBase/src/MBM/mbmlib_client.cpp index 095294c99..6abede2e1 100755 --- a/Online/OnlineBase/src/MBM/mbmlib_client.cpp +++ b/Online/OnlineBase/src/MBM/mbmlib_client.cpp @@ -146,9 +146,9 @@ int _mbm_shutdown (void* /* param */) { } ::lib_rtl_sleep(200); for(int i=0; i<cnt; ++i) { // Now close communication channels and - bm = ids[i]; // unmape the global sections. + bm = ids[i]; // unmape the global sections if owned. bm->communication.close_server(); - if ( bm->buff_add ) { + if ( bm->buff_add && bm->own_buffer ) { ::lib_rtl_unmap_section(bm->buff_add); } bm->buff_add = 0; diff --git a/Online/OnlineBase/src/MBM/mbmlib_server.cpp b/Online/OnlineBase/src/MBM/mbmlib_server.cpp index e86943c30..0c3d206e7 100755 --- a/Online/OnlineBase/src/MBM/mbmlib_server.cpp +++ b/Online/OnlineBase/src/MBM/mbmlib_server.cpp @@ -1247,7 +1247,8 @@ int mbmsrv_get_event(ServerBMID bm, MSG& msg) { ServerBMID_t::LOCK lock(bm->lockid); USER* u = CHECKED_CONSUMER(msg.user); if ( u->state == S_wevent ) { - ::lib_rtl_output(LIB_RTL_ERROR,"++bm_server++ Too many calls to mbm_get_event\n"); + ::lib_rtl_output(LIB_RTL_ERROR,"++bm_server++ Too many calls to " + "mbm_get_event '%s'\n", u->name); return MBM_NO_REPLY; } _mbmsrv_rel_event(bm, u); diff --git a/Online/PCIE40Data/PCIE40Data/RawBank40.h b/Online/PCIE40Data/PCIE40Data/RawBank40.h index 78d20aaf3..d896a1073 100644 --- a/Online/PCIE40Data/PCIE40Data/RawBank40.h +++ b/Online/PCIE40Data/PCIE40Data/RawBank40.h @@ -68,84 +68,105 @@ namespace Online { /// Dummy Constructor RawBank40() = default; - enum BankType { - Other=1, - L0Calo=0, // 0 - L0DU, // 1 - PrsE, // 2 - EcalE, // 3 - HcalE, // 4 - PrsTrig, // 5 - EcalTrig, // 6 - HcalTrig, // 7 - Velo, // 8 - Rich, // 9 - TT, // 10 - IT, // 11 - OT, // 12 - Muon, // 13 - L0PU, // 14 - DAQ, // 15 - ODIN, // 16 - HltDecReports, // 17 - VeloFull, // 18 - TTFull, // 19 - ITFull, // 20 - EcalPacked, // 21 - HcalPacked, // 22 - PrsPacked, // 23 - L0Muon, // 24 - ITError, // 25 - TTError, // 26 - ITPedestal, // 27 - TTPedestal, // 28 - VeloError, // 29 - VeloPedestal, // 30 - VeloProcFull, // 31 - OTRaw, // 32 - OTError, // 33 - EcalPackedError, // 34 - HcalPackedError, // 35 - PrsPackedError, // 36 - L0CaloFull, // 37 - L0CaloError, // 38 - L0MuonCtrlAll, // 39 - L0MuonProcCand, // 40 - L0MuonProcData, // 41 - L0MuonRaw, // 42 - L0MuonError, // 43 - GaudiSerialize, // 44 - GaudiHeader, // 45 - TTProcFull, // 46 - ITProcFull, // 47 - TAEHeader, // 48 - MuonFull, // 49 - MuonError, // 50 - TestDet, // 51 - L0DUError, // 52 - HltRoutingBits, // 53 - HltSelReports, // 54 - HltVertexReports,// 55 - HltLumiSummary, // 56 - L0PUFull, // 57 - L0PUError, // 58 - DstBank, // 59 - DstData, // 60 - DstAddress, // 61 - FileID, // 62 - VP, // 63 - FTCluster, // 64 - VL, // 65 - UT, // 66 - UTFull, // 67 - UTError, // 68 - UTPedestal, // 69 - HC, // 70 - HltTrackReports, // 71 - HCError, // 72 - VPRetinaCluster, // 73 + + /// Define bank types for RawBank + enum BankType { + L0Calo = 0, // 0 + L0DU, // 1 + PrsE, // 2 + EcalE, // 3 + HcalE, // 4 + PrsTrig, // 5 + EcalTrig, // 6 + HcalTrig, // 7 + Velo, // 8 + Rich, // 9 + TT, // 10 + IT, // 11 + OT, // 12 + Muon, // 13 + L0PU, // 14 + DAQ, // 15 + ODIN, // 16 + HltDecReports, // 17 + VeloFull, // 18 + TTFull, // 19 + ITFull, // 20 + EcalPacked, // 21 + HcalPacked, // 22 + PrsPacked, // 23 + L0Muon, // 24 + ITError, // 25 + TTError, // 26 + ITPedestal, // 27 + TTPedestal, // 28 + VeloError, // 29 + VeloPedestal, // 30 + VeloProcFull, // 31 + OTRaw, // 32 + OTError, // 33 + EcalPackedError, // 34 + HcalPackedError, // 35 + PrsPackedError, // 36 + L0CaloFull, // 37 + L0CaloError, // 38 + L0MuonCtrlAll, // 39 + L0MuonProcCand, // 40 + L0MuonProcData, // 41 + L0MuonRaw, // 42 + L0MuonError, // 43 + GaudiSerialize, // 44 + GaudiHeader, // 45 + TTProcFull, // 46 + ITProcFull, // 47 + TAEHeader, // 48 + MuonFull, // 49 + MuonError, // 50 + TestDet, // 51 + L0DUError, // 52 + HltRoutingBits, // 53 + HltSelReports, // 54 + HltVertexReports, // 55 + HltLumiSummary, // 56 + L0PUFull, // 57 + L0PUError, // 58 + DstBank, // 59 + DstData, // 60 + DstAddress, // 61 + FileID, // 62 + VP, // 63 + FTCluster, // 64 + VL, // 65 + UT, // 66 + UTFull, // 67 + UTError, // 68 + UTPedestal, // 69 + HC, // 70 + HltTrackReports, // 71 + HCError, // 72 + VPRetinaCluster, // 73 + FTGeneric, // 74 + FTCalibration, // 75 + FTNZS, // 76 + Calo, // 77 + CaloError, // 78 + MuonSpecial, // 79 + RichCommissioning, // 80 + RichError, // 81 + FTSpecial, // 82 + // Add new types here. Don't forget to update also RawBank.cpp - LastType // LOOP Marker; add new bank types ONLY before! + LastType, // LOOP Marker; add new bank types ONLY before! + Other = 1, + DaqErrorBase = 192, // Lowest DaqError type available in Run 3 (EDMS2100937) + // Banks above are reserved for DAQ, add only generic DaqError types below. + DaqErrorFragmentThrottled = 250, + DaqErrorBXIDCorrupted = 251, + DaqErrorBXIDJump = 252, + DaqErrorFragmentMissing = 253, + DaqErrorFragmentTruncated = 254, + DaqErrorInvalid = 255 + // 255 is the highest type allowed by the Run 3 raw-data format (8-bit unsigned) }; /// Magic pattern for Raw bank headers diff --git a/Online/PCIE40Data/PCIE40Data/pcie40.h b/Online/PCIE40Data/PCIE40Data/pcie40.h index 0eaaa1dab..46089556d 100644 --- a/Online/PCIE40Data/PCIE40Data/pcie40.h +++ b/Online/PCIE40Data/PCIE40Data/pcie40.h @@ -290,8 +290,9 @@ namespace Online { event_t& operator=(const event_t& copy) = delete; public: - static std::pair<bool,size_t> collection_offset(unsigned char type); - std::pair<size_t, const bank_t*> get(unsigned char type) const; + static size_t collection_offset_source_id(unsigned short src_id); + static std::pair<bool,size_t> collection_offset(unsigned short src_id, unsigned char type); + std::pair<size_t, const bank_t*> get(unsigned short src_id, unsigned char type) const; size_t num_bank_collections() const; const bank_collection_t* bank_collection(size_t i) const; size_t total_length() const; @@ -347,6 +348,8 @@ namespace Online { namespace params { typedef unsigned short detsource_t; + static constexpr size_t INVALID_BANK_TYPE = ~0x0UL; + static constexpr detsource_t sourceid_ODIN = 0; static constexpr detsource_t sourceid_VP = 2; static constexpr detsource_t sourceid_UT = 5; @@ -389,14 +392,14 @@ namespace Online { maxTell40Other }; static constexpr size_t collectionSizeODIN = sizeof(bank_collection_t); - static constexpr size_t collectionSizeVP = sizeof(bank_collection_t)+(maxTell40VP-1)*sizeof(bank_t); - static constexpr size_t collectionSizeUT = sizeof(bank_collection_t)+(maxTell40UT-1)*sizeof(bank_t); - static constexpr size_t collectionSizeFT = sizeof(bank_collection_t)+(maxTell40FT-1)*sizeof(bank_t); - static constexpr size_t collectionSizeEcal = sizeof(bank_collection_t)+(maxTell40Ecal-1)*sizeof(bank_t); - static constexpr size_t collectionSizeHcal = sizeof(bank_collection_t)+(maxTell40Hcal-1)*sizeof(bank_t); - static constexpr size_t collectionSizeMuon = sizeof(bank_collection_t)+(maxTell40Muon-1)*sizeof(bank_t); - static constexpr size_t collectionSizeRich = sizeof(bank_collection_t)+(maxTell40Rich-1)*sizeof(bank_t); - static constexpr size_t collectionSizeOther = sizeof(bank_collection_t)+(maxTell40Other-1)*sizeof(bank_t); + static constexpr size_t collectionSizeVP = sizeof(bank_collection_t)+(maxTell40VP*sizeof(bank_t)); + static constexpr size_t collectionSizeUT = sizeof(bank_collection_t)+(maxTell40UT*sizeof(bank_t)); + static constexpr size_t collectionSizeFT = sizeof(bank_collection_t)+(maxTell40FT*sizeof(bank_t)); + static constexpr size_t collectionSizeEcal = sizeof(bank_collection_t)+(maxTell40Ecal*sizeof(bank_t)); + static constexpr size_t collectionSizeHcal = sizeof(bank_collection_t)+(maxTell40Hcal*sizeof(bank_t)); + static constexpr size_t collectionSizeMuon = sizeof(bank_collection_t)+(maxTell40Muon*sizeof(bank_t)); + static constexpr size_t collectionSizeRich = sizeof(bank_collection_t)+(maxTell40Rich*sizeof(bank_t)); + static constexpr size_t collectionSizeOther = sizeof(bank_collection_t)+(maxTell40Other*sizeof(bank_t)); static constexpr size_t collectionOffset = event_t::data_offset; static constexpr size_t collectionOffsetODIN = collectionOffset; @@ -555,37 +558,56 @@ namespace Online { return p; } - inline std::pair<bool,size_t> event_t::collection_offset(unsigned char type) { + inline size_t event_t::collection_offset_source_id(unsigned short src_id) { + switch(src_id) { + case params::sourceid_top5_ODIN: return params::collectionOffsetODIN; + case params::sourceid_top5_VP: return params::collectionOffsetVP; + case params::sourceid_top5_UT: return params::collectionOffsetUT; + case params::sourceid_top5_FT: return params::collectionOffsetFT; + case params::sourceid_top5_Ecal: return params::collectionOffsetEcal; + case params::sourceid_top5_Hcal: return params::collectionOffsetHcal; + case params::sourceid_top5_Muon: return params::collectionOffsetMuon; + case params::sourceid_top5_Rich: return params::collectionOffsetRich; + default: return params::collectionOffsetOther; + } + } + + inline std::pair<bool,size_t> event_t::collection_offset(unsigned short src_id, unsigned char type) { + size_t offset = event_t::collection_offset_source_id(src_id¶ms::sourceid_TOP5); switch ( type ) { case bank_t::ODIN: - return std::make_pair(true,params::collectionOffsetODIN); case bank_t::VP: - return std::make_pair(true,params::collectionOffsetVP); case bank_t::UT: - return std::make_pair(true,params::collectionOffsetUT); - case bank_t::UTFull: - case bank_t::UTError: - case bank_t::UTPedestal: - return std::make_pair(false,params::collectionOffsetUT); case bank_t::FTCluster: - return std::make_pair(true,params::collectionOffsetFT); case bank_t::Rich: - return std::make_pair(true,params::collectionOffsetRich); case bank_t::EcalPacked: - return std::make_pair(true,params::collectionOffsetEcal); case bank_t::HcalPacked: - return std::make_pair(true,params::collectionOffsetHcal); case bank_t::Muon: - return std::make_pair(true,params::collectionOffsetMuon); + return std::make_pair(true,offset); + + case bank_t::UTFull: + case bank_t::UTError: + case bank_t::UTPedestal: + return std::make_pair(false,params::collectionOffsetUT); + case bank_t::Other: + case bank_t::TestDet: return std::make_pair(true,params::collectionOffsetOther); + case bank_t::DaqErrorBase: + case bank_t::DaqErrorFragmentThrottled: + case bank_t::DaqErrorBXIDCorrupted: + case bank_t::DaqErrorBXIDJump: + case bank_t::DaqErrorFragmentMissing: + case bank_t::DaqErrorFragmentTruncated: + case bank_t::DaqErrorInvalid: + return std::make_pair(false,params::collectionOffsetOther); default: - throw std::runtime_error("Unknown bank type!"); + return std::make_pair(false,params::INVALID_BANK_TYPE); } } - inline std::pair<size_t, const bank_t*> event_t::get(unsigned char type) const { - auto ret = event_t::collection_offset(type); + inline std::pair<size_t, const bank_t*> event_t::get(unsigned short src_id, unsigned char type) const { + auto ret = event_t::collection_offset(src_id, type); const auto* c = add_ptr<bank_collection_t>(banks,ret.second); if ( ret.first ) { return std::make_pair(c->length,c->banks); diff --git a/Online/PCIE40Data/PCIE40Data/pcie40_writer.h b/Online/PCIE40Data/PCIE40Data/pcie40_writer.h index 155dd8a30..1769f8144 100644 --- a/Online/PCIE40Data/PCIE40Data/pcie40_writer.h +++ b/Online/PCIE40Data/PCIE40Data/pcie40_writer.h @@ -67,7 +67,7 @@ namespace Online { event_writer_t& operator=(const event_writer_t& copy) = delete; public: - std::pair<size_t, bank_t*> get(unsigned char type); + std::pair<size_t, bank_t*> get(unsigned short src_id, unsigned char type); bank_collection_t* bank_collection(size_t which); }; @@ -121,10 +121,12 @@ namespace Online { return add_ptr<bank_t>(banks, sizeof(bank_t) * which); } - inline std::pair<size_t, bank_t*> event_writer_t::get(unsigned char type) { - auto ret = event_t::collection_offset(type); + inline std::pair<size_t, bank_t*> event_writer_t::get(unsigned short src_id, unsigned char type) { + auto ret = event_t::collection_offset(src_id, type); auto* c = add_ptr<bank_collection_t>(banks, ret.second); - if ( ret.first ) { + if ( ret.second == params::INVALID_BANK_TYPE ) + return std::make_pair(0, nullptr); + else if ( ret.first ) { return std::make_pair(c->length, c->banks); } return std::make_pair(c->specials, add_ptr<bank_t>(c, (c->capacity - c->specials)*sizeof(bank_t))); diff --git a/Online/PCIE40Data/src/pcie40decoder.cpp b/Online/PCIE40Data/src/pcie40decoder.cpp index 5fdf66175..3afadbde0 100644 --- a/Online/PCIE40Data/src/pcie40decoder.cpp +++ b/Online/PCIE40Data/src/pcie40decoder.cpp @@ -21,19 +21,28 @@ #include <cstdarg> #include <vector> #include <map> +#include <unistd.h> using namespace Online; using namespace Online::pcie40; #define __UNLIKELY( x ) __builtin_expect( ( x ), 0 ) + namespace { - void _error(const char* fmt,...) { + + void _error(const char* ,...) {} + + void _error2(const char* fmt,...) { + char text[1024]; va_list args; va_start(args,fmt); - ::vprintf(fmt,args); + ::vsnprintf(text, sizeof(text),fmt,args); va_end(args); + text[sizeof(text)-1] = 0; + ::write(fileno(stdout), text, strlen(text)); } - inline bank_t& allocate_regular_bank(event_t* e, size_t offset) { + + inline bank_t* allocate_regular_bank(event_t* e, size_t offset) { bank_collection_writer_t* c = add_ptr<bank_collection_writer_t>(e,offset); ++c->length; #ifdef PCIE40_ENABLE_CHECKS @@ -47,13 +56,13 @@ namespace { if ( __UNLIKELY(b.magic() != b.MagicPattern) ) { _error("Bad magic pattern in bank!\n"); } - return b; + return &b; #else - return c->banks[c->length-1]; + return &c->banks[c->length-1]; #endif } - inline bank_t& allocate_special_bank(event_t* e, size_t offset) { + inline bank_t* allocate_special_bank(event_t* e, size_t offset) { bank_collection_writer_t* c = add_ptr<bank_collection_writer_t>(e,offset); ++c->specials; #ifdef PCIE40_ENABLE_CHECKS @@ -63,63 +72,79 @@ namespace { if ( c->length + c->specials > c->capacity ) { _error("Bank allocation error: Too many banks!\n"); } - bank_t& b = c->banks[c->length - c->specials]; + bank_t& b = c->banks[c->capacity - c->specials]; if ( b.magic() != b.MagicPattern ) { _error("Bad magic pattern in bank!\n"); } - return b; + return &b; #else - return c->banks[c->capacity - c->specials]; + return &c->banks[c->capacity - c->specials]; #endif } - inline bank_t& allocate_bank(event_t* e, unsigned char type) { - auto ret = event_t::collection_offset(type); - if ( __UNLIKELY(!ret.first) ) - allocate_special_bank(e,ret.second); + inline bank_t* allocate_bank(event_t* e, unsigned short src_id, unsigned char type) { + auto ret = event_t::collection_offset(src_id, type); + if ( __UNLIKELY(ret.second == params::INVALID_BANK_TYPE) ) + return nullptr; + else if ( __UNLIKELY(!ret.first) ) + return allocate_special_bank(e,ret.second); return allocate_regular_bank(e,ret.second); } size_t link_fragment(event_collection_writer_t* ec, const multi_fragment_t* mfp, bool debug) { size_t num_bank = 0; - unsigned short src_id = mfp->header.source_id; + unsigned short src_id = mfp->header.source_id; const size_t align = mfp->header.alignment; const uint8_t version = mfp->header.version; const uint8_t *typs = mfp->types(); const uint16_t *lens = mfp->sizes(); const frontend_data_t *curr = mfp->data(); - for (size_t j = 0, cnt = 0, n=mfp->packingFactor(); cnt<n; ++cnt, ++typs, ++lens) { - auto typx = *typs; + bool run = true; + if ( (const char*)typs > ((const char*)mfp)+6*sizeof(int) ) { + _error2("Inconsistent TYPES pointer\n"); + while(run) ::sleep(1); + } + for (size_t j=0, cnt=0, n=mfp->packingFactor(); cnt<n; ++cnt, ++typs, ++lens) { + auto typx = *typs; + auto length = *lens; event_t* e = ec->at(j); - bank_t& b = allocate_bank(e, typx); -#ifdef PCIE40_ENABLE_CHECKS - if ( __UNLIKELY(e->magic != MAGIC_PATTERN) ) { - _error("Data corruption in event!\n"); - } - if ( __UNLIKELY(b.magic() != b.MagicPattern) ) { - _error("Data corruption in raw bank!\n"); + bank_t* b = allocate_bank(e, src_id, typx); + + if ( nullptr == b ) { + _error2("ERROR +++ Ignore bank: Unknown bank type:%3d Source ID:%3d Size:%4d vsn:%d j:%ld cnt:%ld n:%ld\n", + int(typx), int(src_id), int(length), int(version), j, cnt, n); + while(run) ::sleep(1); } + else { +#ifdef PCIE40_ENABLE_CHECKS + if ( __UNLIKELY(e->magic != MAGIC_PATTERN) ) { + _error("Data corruption in event!\n"); + } + if ( __UNLIKELY(b->magic() != b->MagicPattern) ) { + _error("Data corruption in raw bank!\n"); + } #endif - const void* data = curr->data(); - if ( debug ) { - ::printf("+++ Reading bank[%2ld] at %p len:%4d %p typ:%3d src:%3d vsn:%d\n", - cnt, (void*)data, int(*lens), (void*)((char*)data + *lens), - typx, src_id, version); - } - //b.setMagic(); - b.setSize(*lens); - b.setSourceID(src_id); - b.setData(data); - b.setType((RawBank40::BankType)typx); - b.setVersion(version); + const void* data = curr->data(); + if ( debug ) { + ::printf("+++ Reading bank[%2ld] at %p len:%4d %p typ:%3d src:%3d vsn:%d\n", + cnt, (void*)data, int(length), (void*)((char*)data + length), + typx, src_id, version); + } + //b->setMagic(); + b->setSize(length); + b->setSourceID(src_id); + b->setData(data); + b->setType((RawBank40::BankType)typx); + b->setVersion(version); #if 0 - int* p1 = (int*)b.data(); - int* p2 = (int*)curr->data(); - for(size_t i=0; i<b.size()/sizeof(int); ++i) - p1[i] = p2[i]; + int* p1 = (int*)b->data(); + int* p2 = (int*)curr->data(); + for(size_t i=0; i<b->size()/sizeof(int); ++i) + p1[i] = p2[i]; #endif - ++num_bank; - curr = curr->next(*lens, align); + ++num_bank; + } + curr = curr->next(length, align); ++j; } return num_bank; diff --git a/Online/Storage/Storage/fdb_dbase.h b/Online/Storage/Storage/fdb_dbase.h index aa8050fd3..f4c2415d7 100644 --- a/Online/Storage/Storage/fdb_dbase.h +++ b/Online/Storage/Storage/fdb_dbase.h @@ -47,6 +47,73 @@ namespace Online { static std::unique_ptr<fdb_dbase_t> create(Args... args); public: + + /// Data file properties + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.02.2021 + */ + struct file_t { + std::string name, state, size, date, host; + }; + + /// Interface class to database handler + /** + * To implement a new database technology in principle only + * this class needs to be re-implemented e.g. using MYSQL. + * + * \author M.Frank + * \version 1.0 + * \date 02.02.2021 + */ + class handler_t { + public: + typedef fdb_dbase_t::file_t file_t; + static constexpr int STATE_OPEN = 0; + static constexpr int STATE_WRITTEN = 1; + static constexpr int STATE_READ = 2; + + bool _inited = false; + int _debug { 0 }; + + /// Check the existence of a given object in the database + virtual boost::system::error_code + query_file(const std::string& object_name, file_t& file, int state) = 0; + + public: + + /// Initializing constructor + handler_t(int dbg); + + /// Default destructor + virtual ~handler_t(); + + /// Check the existence of a given object in the database + virtual boost::system::error_code query(std::string& object, + std::string& host, + std::string& date, + std::size_t& length); + + /// Check the existence of a given object in the database + virtual boost::system::error_code next( std::string& object, + std::string& host, + std::string& date, + std::size_t& length); + + /// Add a new object to the database + virtual boost::system::error_code add( const std::string& object, + const std::string& date, + std::size_t length, + const std::string& host) = 0; + + /// Remove an object from the database + virtual boost::system::error_code del( const std::string& object) = 0; + + /// lock database record + virtual boost::system::error_code set( const std::string& object, int value) = 0; + }; /// File database lock /** * \author M.Frank @@ -56,6 +123,7 @@ namespace Online { class lock_t final { /// Reference to locked object fdb_dbase_t* fdb; + public: /// Standard constructor lock_t(fdb_dbase_t* db) : fdb(db) { fdb->lock(); } @@ -63,59 +131,60 @@ namespace Online { ~lock_t() { fdb->unlock(); } }; - std::mutex _lock; - std::string _server; - std::string _location_prefix; - - std::string network_file(const std::string& obj) const { - return "http://" + _server + (obj[0] == '/' ? obj : '/'+obj); - } + std::mutex _lock; + std::string _server; + std::string _location_prefix; + std::unique_ptr<handler_t> _engine; + int _debug {0}; public: /// Standard constructor - fdb_dbase_t(const std::string& server) - : _server(server), _location_prefix("/objects") {} + fdb_dbase_t(const std::string& server); /// Standard destructor - virtual ~fdb_dbase_t() { _lock.unlock(); } + virtual ~fdb_dbase_t(); /// Take global lock of the database obect virtual void lock() { _lock.lock(); } /// Release global lock of the database obect virtual void unlock() { _lock.unlock(); } - /// Check the existence of a given object in the database - virtual boost::system::error_code query_next( const std::string& object_name, - std::string& access_name, - std::string& date, - std::size_t& length) = 0; + /// Access the node specification of the next free node + virtual std::string get_free_host_uri(const std::string& ) const; + + /// Transform the object name to the object key for the database lookup + virtual std::string object_key(const std::string& obj) const; + + /// Transform the object key and the host specs to the network name of the file + virtual std::string network_file(const std::string& host, const std::string& obj) const; + /// Check the existence of a given object in the database virtual boost::system::error_code query_object( const std::string& object_name, std::string& access_name, std::string& date, - std::size_t& length) = 0; + std::size_t& length); + /// Get the next object with the name matching the prefix virtual boost::system::error_code delete_next( const std::string& prefix, std::string& access_name, std::string& date, - std::size_t& length) = 0; + std::size_t& length); + /// Check the existence of a given object in the database virtual boost::system::error_code delete_object(const std::string& object_name, std::string& access_name, std::string& date, - std::size_t& length) = 0; - /// Check the existence of a given object in the database - virtual boost::system::error_code lock_object( const std::string& object_name) = 0; + std::size_t& length); /// Add a new object to the database virtual boost::system::error_code add_object( const std::string& object_name, const std::string& date, std::size_t length, - std::string& access_name) = 0; + std::string& access_name); /// Update the object state of an object in the database virtual boost::system::error_code update_object_state(const std::string& object_name, - const std::string& state) = 0; + const std::string& state); }; } // End namespace storage } // End namespace Online diff --git a/Online/Storage/Storage/fdb_server.h b/Online/Storage/Storage/fdb_server.h index 9ac1c18ff..4df72257b 100644 --- a/Online/Storage/Storage/fdb_server.h +++ b/Online/Storage/Storage/fdb_server.h @@ -132,12 +132,12 @@ namespace Online { file_dir.c_str(), ::strerror(errno)); ::exit(EINVAL); } - if ( !file_dir.empty() ) { - ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ File repository: %s", file_dir.c_str()); - } - if ( !server.empty() ) { - ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ Server URI: %s", server.c_str()); - } + std::stringstream str; + if ( !file_dir.empty() ) str << " File repository: " << file_dir; + if ( !server.empty() ) str << " Server URI: " << server; + if ( !local.empty() ) str << " Local URI: " << local; + ::lib_rtl_output(LIB_RTL_ALWAYS, "+++%s", str.str().c_str()); + uri_t uri(local); ptr = std::make_unique<http::basic_http_server<T> >(uri.host, uri.port); ptr->implementation->buffer_size = buffer_size; diff --git a/Online/Storage/src/server/fdb_SQLite.cpp b/Online/Storage/src/server/fdb_SQLite.cpp index 661ac40e7..3cfee40e0 100644 --- a/Online/Storage/src/server/fdb_SQLite.cpp +++ b/Online/Storage/src/server/fdb_SQLite.cpp @@ -22,220 +22,170 @@ using namespace boost; using namespace Online::storage; -namespace { - template<typename T> inline int _prt(const T* o) { - return o->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG; - } -} -system::error_code -fdb_SQLite::handler_t::initialize(const std::string& db_name) { - std::string err; - const char* nam = db_name.c_str(); - this->database = sqlite::database::open(db_name, err); - if ( this->database.db ) { - const char* sql = "CREATE TABLE IF NOT EXISTS Files (" - "Name TEXT PRIMARY KEY, " - "State INT NOT NULL, " - "Size INT NOT NULL, " - "Date TEXT NOT NULL, " - "Access TEXT NOT NULL)"; - if ( this->database.execute(err, sql) != sqlite::traits::OK ) { - ::lib_rtl_output(LIB_RTL_ERROR,"%s: Create table: SQL error: %s", nam, err.c_str()); - return system::error_code(system::errc::permission_denied, system::system_category()); - } - ::lib_rtl_output(LIB_RTL_INFO,"FDB Database server: Database %s", nam); - ::lib_rtl_output(LIB_RTL_INFO,"FDB Database server: Table FILES created successfully"); - return system::error_code(system::errc::success, system::system_category()); - } - ::lib_rtl_output(LIB_RTL_INFO,"%s: FAILED to access db: [%s]", nam, err.c_str()); - return system::error_code(system::errc::permission_denied, system::system_category()); +/// Initializing constructor +SQLite_handler_t::SQLite_handler_t(const std::string& p, int dbg) + : fdb_dbase_t::handler_t(dbg), path(p) +{ + this->init(path); } -/// Check the existence of a given object in the database -system::error_code -fdb_SQLite::handler_t::query(std::string& object, - std::string& access, - std::string& date, - std::size_t& length) { - File file; - system::error_code ec = this->query_file(object, file, handler_t::STATE_WRITTEN); - if ( ec == system::errc::success ) { - object = file.name; - access = file.access; - length = ::atol(file.size.c_str()); - date = file.date; - ::lib_rtl_output(_prt(this), "Query '%s'", object.c_str()); - return ec; +/// Default destructor +SQLite_handler_t::~SQLite_handler_t() { + this->fini(); + if ( this->database.handle() ) { + this->database.commit(); + this->database.close(); } - ::lib_rtl_output(LIB_RTL_ALWAYS,"Query: file '%s' NOT FOUND!", object.c_str()); - return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); } -/// Check the existence of a given object in the database -boost::system::error_code -fdb_SQLite::handler_t::next( std::string& object, - std::string& access, - std::string& date, - std::size_t& length) { - File file; - system::error_code ec = this->query_file(object, file, handler_t::STATE_WRITTEN); - if ( ec == system::errc::success ) { - object = file.name; - access = file.access; - length = ::atol(file.size.c_str()); - date = file.date; - ec = this->lock(file.name, handler_t::STATE_READ); - if ( ec == system::errc::success ) { - ::lib_rtl_output(_prt(this), "Next '%s'", object.c_str()); +/// Initialize object +system::error_code +SQLite_handler_t::init(const std::string& db_name) { + if ( !this->_inited ) { + std::string err; + const char* nam = db_name.c_str(); + this->database = sqlite::database::open(db_name, err); + if ( this->database.db ) { + const char* sql = "CREATE TABLE IF NOT EXISTS Files (" + "Name TEXT PRIMARY KEY, " + "State INT NOT NULL, " + "Size INT NOT NULL, " + "Date TEXT NOT NULL, " + "Host TEXT NOT NULL)"; + if ( this->database.execute(err, sql) != sqlite::traits::OK ) { + ::lib_rtl_output(LIB_RTL_ERROR,"%s: Create table: SQL error: %s", nam, err.c_str()); + return system::error_code(system::errc::permission_denied, system::system_category()); + } + ::lib_rtl_output(LIB_RTL_INFO,"FDB Database server: Database %s", nam); + ::lib_rtl_output(LIB_RTL_INFO,"FDB Database server: Table FILES created successfully"); + /// Create prepared statement to insert records + this->insert_record.prepare(database, + "INSERT INTO Files (Name, State, Size, Date, Host) " + "VALUES (?, ?, ?, ?, ?)"); + this->delete_record.prepare(database, + "DELETE FROM Files WHERE Name=?1 "); + this->query_record.prepare (database, + "SELECT Name, State, Size, Date, Host FROM Files " + "WHERE Name LIKE ?1 AND State = ?2 "); + this->lock_record.prepare (database, + "UPDATE Files SET State=?1 WHERE Name=?2 "); + this->_inited = true; return system::error_code(system::errc::success, system::system_category()); } - return system::error_code(system::errc::protocol_error, system::system_category()); + ::lib_rtl_output(LIB_RTL_INFO,"%s: FAILED to access db: [%s]", nam, err.c_str()); + return system::error_code(system::errc::permission_denied, system::system_category()); } - ::lib_rtl_output(LIB_RTL_ALWAYS,"Query: file '%s' NOT FOUND!", object.c_str()); - return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); + return system::error_code(system::errc::success, system::system_category()); } -/// Standard destructor -fdb_SQLite::~fdb_SQLite() { - engine.reset(); -} - -/// Check the existence of a given object in the database -system::error_code fdb_SQLite::query_object(const std::string& object, - std::string& access, - std::string& date, - std::size_t& length) -{ - std::string obj = object; - auto ec = this->engine->query(obj, access, date, length); - access = this->network_file(access); - if ( ec == system::errc::success ) { - ::lib_rtl_output(_prt(this), "Lookup '%s'", object.c_str()); - return ec; +/// Finalize object +void SQLite_handler_t::fini() { + if ( this->_inited ) { + this->insert_record.finalize(); + this->delete_record.finalize(); + this->query_record.finalize(); + this->lock_record.finalize(); + this->database.close(); + this->_inited = false; } - ::lib_rtl_output(LIB_RTL_ALWAYS,"FAILED to lookup object: '%s' [%s]", - object.c_str(), ec.message().c_str()); - return ec; } - + /// Check the existence of a given object in the database -system::error_code fdb_SQLite::query_next( const std::string& object, - std::string& access, - std::string& date, - std::size_t& length) -{ - std::string obj = object; - auto ec = this->engine->query(obj, access, date, length); - access = this->network_file(access); - if ( ec == system::errc::success ) { - ::lib_rtl_output(_prt(this), "Lookup '%s'", object.c_str()); - return ec; +system::error_code +SQLite_handler_t::query_file(const std::string& object_name, file_t& file, int state) { + this->query_record.reset(); + this->query_record.bind(1, object_name); + this->query_record.bind(2, state); + int ret = this->query_record.execute(); + while ( ret == SQLITE_ROW ) { + file.name = query_record.get<std::string>(0); + file.state = query_record.get<std::string>(1); + file.size = query_record.get<std::string>(2); + file.date = query_record.get<std::string>(3); + file.host = query_record.get<std::string>(4); + return system::error_code(system::errc::success, system::system_category()); } - ::lib_rtl_output(LIB_RTL_ALWAYS,"FAILED to lookup object: '%s' [%s]", - object.c_str(), ec.message().c_str()); - return ec; + return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); } -/// Check the existence of the next object of a sequence -system::error_code fdb_SQLite::delete_next( const std::string& object, - std::string& access, - std::string& date, - std::size_t& length) -{ - std::string obj = object; - auto ec = this->engine->next(obj, access, date, length); - if ( ec == system::errc::success ) { - ec = this->engine->del(_location_prefix+access); - access = this->network_file(access); - if ( ec == system::errc::success ) { - ::lib_rtl_output(_prt(this), "Lookup '%s'", object.c_str()); - return ec; - } +/// Add a new object to the database +system::error_code +SQLite_handler_t::add (const std::string& object_name, + const std::string& date, + std::size_t length, + const std::string& host) { + this->insert_record.reset(); + this->insert_record.bind(1, object_name); + this->insert_record.bind(2, STATE_OPEN); + this->insert_record.bind(3, length); + this->insert_record.bind(4, date); + this->insert_record.bind(5, host); + this->database.begin(); + int ret = this->insert_record.execute(); + this->insert_record.reset(); + this->database.commit(); + if ( ret == SQLITE_DONE ) { + return system::error_code(system::errc::success, system::system_category()); } - ::lib_rtl_output(LIB_RTL_ERROR,"FAILED to lookup sequence object: '%s' [%s]", - object.c_str(), ec.message().c_str()); - return ec; -} - -/// Check the existence of a given object in the database -system::error_code fdb_SQLite::delete_object(const std::string& object, - std::string& access, - std::string& date, - std::size_t& length) -{ - std::string obj = object; - auto ec = this->engine->query(obj, access, date, length); - if ( ec == system::errc::success ) { - ec = this->engine->del(object); - access = this->network_file(access); - if ( ec == system::errc::success ) { - ::lib_rtl_output(_prt(this), "Remove '%s'", object.c_str()); - return ec; - } + else if ( ret == SQLITE_CONSTRAINT_PRIMARYKEY ) { + return system::error_code(system::errc::file_exists, system::system_category()); + } + else if ( ret == SQLITE_CONSTRAINT_FOREIGNKEY ) { + return system::error_code(system::errc::file_exists, system::system_category()); } - ::lib_rtl_output(_prt(this), "FAILED to remove: '%s' [%s]", - object.c_str(), ec.message().c_str()); - return ec; + else if ( ret != SQLITE_OK ) { + return system::error_code(system::errc::file_exists, system::system_category()); + } + return system::error_code(system::errc::success, system::system_category()); } -/// Check the existence of a given object in the database -system::error_code fdb_SQLite::lock_object(const std::string& object) -{ - size_t length = 0; - std::string access, date, obj = object; - auto ec = this->engine->query(obj, access, date, length); - if ( ec == system::errc::success ) { - return this->engine->lock(object, handler_t::STATE_READ); - } +/// Remove an object from the database +system::error_code +SQLite_handler_t::del (const std::string& object_name) { + this->delete_record.reset(); + this->delete_record.bind(1, object_name); + this->database.begin(); + int ret = this->delete_record.execute(); + this->delete_record.reset(); + this->database.commit(); + if ( ret == SQLITE_OK ) + return system::error_code(system::errc::success, system::system_category()); + else if ( ret == SQLITE_DONE ) + return system::error_code(system::errc::success, system::system_category()); return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); } -/// Add a new object to the database +/// lock database record system::error_code -fdb_SQLite::add_object(const std::string& object, - const std::string& date, - std::size_t length, - std::string& access) -{ - std::string acc = object.substr(_location_prefix.length(), std::string::npos); - auto ec = this->engine->add(object, date, length, acc); - if ( ec == system::errc::success ) { - ::lib_rtl_output(_prt(this), "Add '%s' %s %s", object.c_str(), - this->debug > 1 ? "access:" : "", - this->debug > 1 ? acc.c_str() : ""); - access = this->network_file(acc); - return ec; - } - ::lib_rtl_output(LIB_RTL_ERROR,"FAILED to add: '%s' access: %s [%s]", - object.c_str(), access.c_str(), ec.message().c_str()); - return ec; +SQLite_handler_t::set (const std::string& object_name, int value) { + this->lock_record.reset(); + this->lock_record.bind(1, value); + this->lock_record.bind(2, object_name); + this->database.begin(); + int ret = this->lock_record.execute(); + this->lock_record.reset(); + this->database.commit(); + if ( ret == SQLITE_DONE ) + return system::error_code(system::errc::success, system::system_category()); + else if ( ret == SQLITE_OK ) + return system::error_code(system::errc::success, system::system_category()); + return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); } -/// Update the object state of an object in the database -boost::system::error_code -fdb_SQLite::update_object_state(const std::string& object, const std::string& state) { - handler_t::File file; - system::error_code ec; - std::string obj = object; - if ( state == "WRITTEN" ) { - ec = this->engine->query_file(obj, file, handler_t::STATE_OPEN); - if ( ec == system::errc::success ) { - ec = this->engine->lock(object, handler_t::STATE_WRITTEN); - } - } - else if ( state == "READ" ) { - ec = this->engine->query_file(obj, file, handler_t::STATE_WRITTEN); - if ( ec == system::errc::success ) { - ec = this->engine->lock(object, handler_t::STATE_READ); - } - } - if ( ec == system::errc::success ) { - ::lib_rtl_output(this->debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, - "Update '%s' NEW state: %s", object.c_str(), state.c_str()); - return ec; - } - ::lib_rtl_output(LIB_RTL_ERROR,"FAILED update '%s' to state: %s [%s]", - object.c_str(), state.c_str(), ec.message().c_str()); - return ec; +#include <Storage/fdb_server.h> +class Online::storage::db::traits { +public: + typedef fdb_dbase_t dbase_t; +}; + +extern "C" int fdb_sqlite_server(int argc, char** argv) { + Online::storage::SrvRun<Online::storage::db> s; + s.create(argc, argv); + auto* sql = new Online::storage::fdb_dbase_t(s.server); + sql->_debug = s.application_debug; + sql->_engine.reset(new SQLite_handler_t(s.dbase, s.application_debug)); + s.ptr->implementation->dbase.reset(sql); + s.start(); + return 0; } diff --git a/Online/Storage/src/server/fdb_SQLite.h b/Online/Storage/src/server/fdb_SQLite.h index 3ddb34347..d20063d52 100644 --- a/Online/Storage/src/server/fdb_SQLite.h +++ b/Online/Storage/src/server/fdb_SQLite.h @@ -27,126 +27,53 @@ namespace Online { /// Namespace for the storage interface namespace storage { - /// Simple implementation of a storage database based on a file system directory. + /// Interface class to database handler /** + * To implement a new database technology in principle only + * this class needs to be re-implemented e.g. using MYSQL. * * \author M.Frank * \version 1.0 * \date 02.02.2021 */ - class fdb_SQLite : public fdb_dbase_t { + class SQLite_handler_t : public fdb_dbase_t::handler_t { public: + std::string path; + sqlite::database database; + sqlite::statement insert_record; + sqlite::statement query_record; + sqlite::statement delete_record; + sqlite::statement lock_record; - /// Interface class to database handler - /** - * - * \author M.Frank - * \version 1.0 - * \date 02.02.2021 - */ - class handler_t { - public: - static constexpr int STATE_OPEN = 0; - static constexpr int STATE_WRITTEN = 1; - static constexpr int STATE_READ = 1; - - sqlite::database database; - int debug { 0 }; - struct File { - std::string name, state, size, date, access; - }; - /// Check the existence of a given object in the database - virtual boost::system::error_code - query_file(const std::string& object_name, File& file, int state) = 0; + /// Check the existence of a given object in the database + virtual boost::system::error_code + query_file(const std::string& object_name, file_t& file, int state) override; - public: - handler_t(int dbg) - : debug(dbg) - { - } - virtual ~handler_t() { - if ( database.handle() ) { - database.commit(); - database.close(); - } - } - - boost::system::error_code initialize(const std::string& db_name); - - /// Check the existence of a given object in the database - virtual boost::system::error_code query(std::string& object_name, - std::string& access_name, - std::string& date, - std::size_t& length); - /// Check the existence of a given object in the database - virtual boost::system::error_code next( std::string& object_name, - std::string& access_name, - std::string& date, - std::size_t& length); - /// Add a new object to the database - virtual boost::system::error_code add( const std::string& object_name, - const std::string& date, - std::size_t length, - const std::string& access_name) = 0; - /// Remove an object from the database - virtual boost::system::error_code del( const std::string& object_name) = 0; - /// lock database record - virtual boost::system::error_code lock( const std::string& object_name, int value) = 0; - }; - - std::unique_ptr<handler_t> engine; - int debug {0}; - public: - /// Standard constructor - using fdb_dbase_t::fdb_dbase_t; - /// Standard destructor - virtual ~fdb_SQLite(); + /// Initializing constructor + SQLite_handler_t(const std::string& p, int dbg); - /// Check the existence of a given object in the database - virtual boost::system::error_code - query_next( const std::string& object_name, - std::string& access_name, - std::string& date, - std::size_t& length) override; + /// Default destructor + virtual ~SQLite_handler_t(); - /// Check the existence of a given object in the database - virtual boost::system::error_code - query_object( const std::string& object_name, - std::string& access_name, - std::string& date, - std::size_t& length) override; + /// Initialize object + boost::system::error_code init(const std::string& db_name); - /// Check the existence of the next object of a sequence - virtual boost::system::error_code - delete_next( const std::string& object_name, - std::string& access_name, - std::string& date, - std::size_t& length) override; + /// Finalize object + void fini(); - /// Check the existence of a given object in the database - virtual boost::system::error_code - delete_object(const std::string& object_name, - std::string& access_name, - std::string& date, - std::size_t& length) override; - - /// Check the existence of a given object in the database - virtual boost::system::error_code - lock_object( const std::string& object_name) override; - /// Add a new object to the database - virtual boost::system::error_code - add_object( const std::string& object_name, - const std::string& date, - std::size_t length, - std::string& access_name) override; - - /// Update the object state of an object in the database - virtual boost::system::error_code - update_object_state(const std::string& object_name, - const std::string& state) override; + boost::system::error_code add( const std::string& object, + const std::string& date, + std::size_t length, + const std::string& host) override; + + /// Remove an object from the database + boost::system::error_code del( const std::string& object) override; + + /// lock database record + boost::system::error_code set( const std::string& object, int value) override; }; } // End namespace storage } // End namespace Online diff --git a/Online/Storage/src/server/fdb_SQLite_bind.cpp b/Online/Storage/src/server/fdb_SQLite_bind.cpp index bea063ef6..0c136cbaa 100644 --- a/Online/Storage/src/server/fdb_SQLite_bind.cpp +++ b/Online/Storage/src/server/fdb_SQLite_bind.cpp @@ -12,6 +12,7 @@ // // Author : Markus Frank //========================================================================== +#if 0 // Framework include files #include "fdb_SQLite.h" @@ -24,12 +25,6 @@ namespace { class handler_t : public Online::storage::fdb_SQLite::handler_t { public: - std::string path; - sqlite::statement insert_record; - sqlite::statement query_record; - sqlite::statement delete_record; - sqlite::statement lock_record; - bool inited = false; /// Standard constructor handler_t(const std::string& p, int dbg) @@ -43,38 +38,6 @@ namespace { this->fini(); } - /// Initialize object - void init(const std::string& db_name) { - if ( !inited ) { - if ( this->initialize(db_name) == system::errc::success ) { - /// Create prepared statement to insert records - this->insert_record.prepare(database, - "INSERT INTO Files (Name, State, Size, Date, Access) " - "VALUES (?, ?, ?, ?, ?)"); - this->delete_record.prepare(database, - "DELETE FROM Files WHERE Name=?1 "); - this->query_record.prepare (database, - "SELECT Name, State, Size, Date, Access FROM Files " - "WHERE Name LIKE ?1 AND State = ?2 "); - this->lock_record.prepare (database, - "UPDATE Files SET State=?1 WHERE Name=?2 "); - } - inited = true; - } - } - - /// Finalize object - void fini() { - if ( inited ) { - this->insert_record.finalize(); - this->delete_record.finalize(); - this->query_record.finalize(); - this->lock_record.finalize(); - this->database.close(); - inited = false; - } - } - /// Check the existence of a given object in the database virtual system::error_code query_file(const std::string& object_name, File& file, int state) override { this->query_record.reset(); @@ -82,11 +45,11 @@ namespace { this->query_record.bind(2, state); int ret = this->query_record.execute(); while ( ret == SQLITE_ROW ) { - file.name = query_record.get<std::string>(0); - file.state = query_record.get<std::string>(1); - file.size = query_record.get<std::string>(2); - file.date = query_record.get<std::string>(3); - file.access = query_record.get<std::string>(4); + file.name = query_record.get<std::string>(0); + file.state = query_record.get<std::string>(1); + file.size = query_record.get<std::string>(2); + file.date = query_record.get<std::string>(3); + file.host = query_record.get<std::string>(4); return system::error_code(system::errc::success, system::system_category()); } return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); @@ -96,13 +59,13 @@ namespace { virtual system::error_code add (const std::string& object_name, const std::string& date, std::size_t length, - const std::string& access_name) override { + const std::string& host) override { this->insert_record.reset(); this->insert_record.bind(1, object_name); this->insert_record.bind(2, STATE_OPEN); this->insert_record.bind(3, length); this->insert_record.bind(4, date); - this->insert_record.bind(5, access_name); + this->insert_record.bind(5, host); this->database.begin(); int ret = this->insert_record.execute(); this->insert_record.reset(); @@ -138,7 +101,7 @@ namespace { } /// lock database record - virtual system::error_code lock (const std::string& object_name, int value) override { + virtual system::error_code set (const std::string& object_name, int value) override { this->lock_record.reset(); this->lock_record.bind(1, value); this->lock_record.bind(2, object_name); @@ -171,3 +134,4 @@ extern "C" int fdb_sqlite_bind_server(int argc, char** argv) { s.start(); return 0; } +#endif diff --git a/Online/Storage/src/server/fdb_SQLite_direct.cpp b/Online/Storage/src/server/fdb_SQLite_direct.cpp index a1ef87ac4..e69de29bb 100644 --- a/Online/Storage/src/server/fdb_SQLite_direct.cpp +++ b/Online/Storage/src/server/fdb_SQLite_direct.cpp @@ -1,147 +0,0 @@ -//========================================================================== -// LHCb Online software suite -//-------------------------------------------------------------------------- -// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) -// All rights reserved. -// -// For the licensing terms see OnlineSys/LICENSE. -// -//-------------------------------------------------------------------------- -// -// Package : RPC -// -// Author : Markus Frank -//========================================================================== -// -// gdb --args gentest.exe libStorageClient.so fdb_sqlite_direct_server -local_port=8000 -fssrv_sser=pluscc08 -fssrv_port=8001 -threads=1 -database=/home/frankm/storage_files.dbase -log=/proc/self/fd/2 -// -// Framework include files -#include "fdb_SQLite.h" -#include <RTL/rtl.h> - -// C/C++ files - -using namespace boost; - -namespace { - - constexpr static int RELEASE_BUFF_SIZE = 10000000; - class handler_t : public Online::storage::fdb_SQLite::handler_t { - public: - struct Reader { - std::vector<File> files; - static int callback(void* ptr, int /* argc */, char** argv, char** /* name */) { - Reader* rdr = (Reader*)ptr; - File file {argv[0], argv[1], argv[2], argv[3], argv[4]}; - rdr->files.emplace_back(file); - return 0; - } - }; - - handler_t(const std::string& db_name, int dbg) - : Online::storage::fdb_SQLite::handler_t(dbg) - { - std::string error; - this->initialize(db_name); - this->database.execute_sql(error,"PRAGMA cache_size=10000"); - this->database.execute_sql(error,"PRAGMA journal_mode=OFF"); - } - - /// Check the existence of a given object in the database - virtual system::error_code - query_file(const std::string& object_name, File& file, int state) override { - Reader reader; - std::string error; - int ret = this->database.execute_sql(error, - &reader, - Reader::callback, - "SELECT Name, State, Size, Date, Access FROM Files " - "WHERE Name LIKE '%s%%' AND State=%d LIMIT 1", - object_name.c_str(), state); - ::sqlite3_release_memory(RELEASE_BUFF_SIZE); - if ( ret != sqlite::traits::OK ) { - ::lib_rtl_output(LIB_RTL_ALWAYS,"%s: Query: file %s SQL Error [%s]", - this->database.name(), object_name.c_str(), error.c_str()); - return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); - } - else if ( reader.files.empty() ) { - ::lib_rtl_output(LIB_RTL_ALWAYS,"%s: Query: file %s NOT FOUND!", - this->database.name(), object_name.c_str()); - return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); - } - file = std::move(reader.files[0]); - return system::error_code(system::errc::success, system::system_category()); - } - - /// Add a new object to the database - virtual system::error_code add (const std::string& object_name, - const std::string& date, - std::size_t length, - const std::string& access_name) override { - std::string error; - int ret = this->database.execute_sql(error, - "INSERT INTO Files (Name, State, Size, Date, Access) " - "VALUES ('%s', %d, %ld, '%s', '%s')", - object_name.c_str(), STATE_OPEN, length, - date.c_str(), access_name.c_str()); - ::sqlite3_release_memory(RELEASE_BUFF_SIZE); - if ( ret == SQLITE_CONSTRAINT_PRIMARYKEY ) { - return system::error_code(system::errc::file_exists, system::system_category()); - } - else if ( ret == SQLITE_CONSTRAINT_FOREIGNKEY ) { - return system::error_code(system::errc::file_exists, system::system_category()); - } - else if ( ret != SQLITE_OK ) { - return system::error_code(system::errc::file_exists, system::system_category()); - } - return system::error_code(system::errc::success, system::system_category()); - } - - /// Remove an object from the database - virtual system::error_code del (const std::string& object_name) override { - std::string error; - int ret = this->database.execute_sql(error, - "DELETE FROM Files WHERE Name='%s'", - object_name.c_str()); - int num = this->database.changes(); - num = 1;// This does not work: - ::sqlite3_release_memory(RELEASE_BUFF_SIZE); - if ( ret == SQLITE_OK && num == 1 ) - return system::error_code(system::errc::success, system::system_category()); - return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); - } - - /// lock database record - virtual system::error_code lock (const std::string& object_name, int value) override { - std::string error; - int ret = this->database.execute_sql(error, - "UPDATE Files SET State=%d" - " WHERE Name='%s' AND State=%d", - value, object_name.c_str(), value); - int num = database.changes(); - num = 1;// This does not work: - ::sqlite3_release_memory(RELEASE_BUFF_SIZE); - if ( ret == SQLITE_OK && num == 1 ) - return system::error_code(system::errc::success, system::system_category()); - return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); - } - }; -} - -#include <Storage/fdb_server.h> -class Online::storage::db::traits { -public: - typedef fdb_dbase_t dbase_t; -}; - -extern "C" int fdb_sqlite_direct_server(int argc, char** argv) { - Online::storage::SrvRun<Online::storage::db> s; - s.create(argc, argv); - auto* sql = new Online::storage::fdb_SQLite(s.server); - sql->engine.reset(new handler_t(s.dbase, s.application_debug)); - sql->debug = s.application_debug; - s.ptr->implementation->dbase.reset(sql); - s.start(); - return 0; -} - diff --git a/Online/Storage/src/server/fdb_db_server.cpp b/Online/Storage/src/server/fdb_db_server.cpp index 3ec17e764..4e6dc0a29 100644 --- a/Online/Storage/src/server/fdb_db_server.cpp +++ b/Online/Storage/src/server/fdb_db_server.cpp @@ -29,10 +29,20 @@ public: using namespace std; using namespace Online::storage; -static const string PATTERN_NEXT = "/next?prefix="; static constexpr const double kByte = 1024e0; static constexpr const double MByte = kByte*kByte; +static inline string object_name(const string& obj, string opt="") { + static const string PATTERN_NEXT = "/next?prefix="; + bool get_next = (0 == ::strncasecmp(obj.c_str(), + PATTERN_NEXT.c_str(), + PATTERN_NEXT.length())); + if ( get_next ) { + return obj.substr(PATTERN_NEXT.length())+opt; + } + return obj; +} + /// Standard destructor template <> http::basic_http_server<db>::handler_t::~handler_t() { } @@ -41,14 +51,11 @@ template <> http::basic_http_server<db>::handler_t::~handler_t() { template <> http::HttpRequestHandler::continue_action http::basic_http_server<db>::handler_t::handle_get(const request_t& req, reply_t& rep) { size_t length = 0; - string date, access_name; - bool get_next = (0 == ::strncasecmp(req.uri.c_str(),PATTERN_NEXT.c_str(),PATTERN_NEXT.length())); + string date, access_name, obj = object_name(req.uri); + boost::system::error_code ec; { dbase_t::lock_t lock(dbase.get()); - if ( get_next ) - ec = dbase->query_next(req.uri.substr(PATTERN_NEXT.length()), access_name, date, length); - else - ec = dbase->query_object(req.uri, access_name, date, length); + ec = dbase->query_object(req.uri, access_name, date, length); } if ( ec == boost::system::errc::no_such_file_or_directory ) rep = reply_t::stock_reply(reply_t::not_found); @@ -79,14 +86,11 @@ http::basic_http_server<db>::handler_t::handle_get(const request_t& req, reply_t template <> http::HttpRequestHandler::continue_action http::basic_http_server<db>::handler_t::handle_delete(const request_t& req, reply_t& rep) { size_t length = 0; - string date, access_name; - bool get_next = (0 == ::strncasecmp(req.uri.c_str(),PATTERN_NEXT.c_str(),PATTERN_NEXT.length())); + string date, access_name, obj = object_name(req.uri,"%"); + boost::system::error_code ec; { dbase_t::lock_t lock(dbase.get()); - if ( get_next ) - ec = dbase->delete_next(req.uri.substr(PATTERN_NEXT.length())+'%', access_name, date, length); - else - ec = dbase->delete_object(req.uri, access_name, date, length); + ec = dbase->delete_object(obj, access_name, date, length); } if ( ec == boost::system::errc::no_such_file_or_directory ) rep = reply_t::stock_reply(reply_t::not_found); @@ -107,7 +111,7 @@ http::basic_http_server<db>::handler_t::handle_delete(const request_t& req, repl rep = reply_t::stock_reply(reply_t::temp_redirect); rep.headers.emplace_back(http::constants::location,access_name); ::lib_rtl_output(LIB_RTL_INFO,"DELETE:'%s' %.1f %cB [%s]", - access_name.c_str(), double(length)/(mb ? MByte : kByte), mb ? 'M' : 'k', + obj.c_str(), double(length)/(mb ? MByte : kByte), mb ? 'M' : 'k', req.remote_address().to_string().c_str()); } return write; diff --git a/Online/Storage/src/server/fdb_dbase.cpp b/Online/Storage/src/server/fdb_dbase.cpp new file mode 100644 index 000000000..83e6ad294 --- /dev/null +++ b/Online/Storage/src/server/fdb_dbase.cpp @@ -0,0 +1,217 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Package : RPC +// +// Author : Markus Frank +//========================================================================== + +// Framework include files +#include <Storage/fdb_dbase.h> +#include <RTL/rtl.h> + +// C/C++ files + +using namespace boost; +using namespace Online::storage; + +namespace { + template<typename T> inline int _prt(const T* o) { + return o->_debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG; + } +} + +/// Initializing constructor +fdb_dbase_t::handler_t::handler_t(int dbg) : _debug(dbg) { +} + +/// Default destructor +fdb_dbase_t::handler_t::~handler_t() { +} + +/// Check the existence of a given object in the database +system::error_code +fdb_dbase_t::handler_t::query(std::string& object, + std::string& host, + std::string& date, + std::size_t& length) { + file_t file; + system::error_code ec = this->query_file(object, file, handler_t::STATE_WRITTEN); + if ( ec == system::errc::success ) { + object = file.name; + host = file.host; + length = ::atol(file.size.c_str()); + date = file.date; + ::lib_rtl_output(_prt(this), "Query '%s'", object.c_str()); + return ec; + } + ::lib_rtl_output(LIB_RTL_ALWAYS,"Query: file '%s' NOT FOUND!", object.c_str()); + return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); +} + +/// Check the existence of a given object in the database +boost::system::error_code +fdb_dbase_t::handler_t::next( std::string& object, + std::string& host, + std::string& date, + std::size_t& length) { + file_t file; + system::error_code ec = this->query_file(object, file, handler_t::STATE_WRITTEN); + if ( ec == system::errc::success ) { + object = file.name; + host = file.host; + length = ::atol(file.size.c_str()); + date = file.date; + ec = this->set(object, handler_t::STATE_READ); + if ( ec == system::errc::success ) { + ::lib_rtl_output(_prt(this), "Next '%s'", object.c_str()); + return system::error_code(system::errc::success, system::system_category()); + } + return system::error_code(system::errc::protocol_error, system::system_category()); + } + ::lib_rtl_output(LIB_RTL_ALWAYS,"Query: file '%s' NOT FOUND!", object.c_str()); + return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); +} + +/// Standard constructor +fdb_dbase_t::fdb_dbase_t(const std::string& server) + : _server(server), _location_prefix("/objects") { +} + +/// Standard destructor +fdb_dbase_t::~fdb_dbase_t() { + _engine.reset(); + _lock.unlock(); +} + +/// Access the node specification of the next free node +std::string fdb_dbase_t::get_free_host_uri(const std::string& ) const { + return "http://" + _server; +} + +/// Transform the object name to the object key for the database lookup +std::string fdb_dbase_t::object_key(const std::string& obj) const { + return obj.substr(_location_prefix.length(), std::string::npos); +} + +/// Transform the object key and the host specs to the network name of the file +std::string fdb_dbase_t::network_file(const std::string& host, const std::string& obj) const { + return host + (obj[0] == '/' ? obj : '/'+obj); +} + +/// Check the existence of a given object in the database +system::error_code fdb_dbase_t::query_object(const std::string& object, + std::string& access, + std::string& date, + std::size_t& length) +{ + std::string host, obj = this->object_key(object); + auto ec = this->_engine->query(obj, host, date, length); + if ( ec == system::errc::success ) { + access = this->network_file(host, obj); + ::lib_rtl_output(_prt(this), "Lookup '%s'", obj.c_str()); + return ec; + } + ::lib_rtl_output(LIB_RTL_ALWAYS,"FAILED to lookup object: '%s' [%s]", + object.c_str(), ec.message().c_str()); + return ec; +} + +/// Check the existence of the next object of a sequence +system::error_code fdb_dbase_t::delete_next( const std::string& object, + std::string& access, + std::string& date, + std::size_t& length) +{ + std::string host, obj = this->object_key(object); + auto ec = this->_engine->next(obj, host, date, length); + if ( ec == system::errc::success ) { + ec = this->_engine->del(obj); + access = this->network_file(host, obj); + if ( ec == system::errc::success ) { + ::lib_rtl_output(_prt(this), "Lookup '%s'", obj.c_str()); + return ec; + } + } + ::lib_rtl_output(LIB_RTL_ERROR,"FAILED to lookup sequence object: '%s' [%s]", + object.c_str(), ec.message().c_str()); + return ec; +} + +/// Check the existence of a given object in the database +system::error_code fdb_dbase_t::delete_object(const std::string& object, + std::string& access, + std::string& date, + std::size_t& length) +{ + std::string host, obj = this->object_key(object); + auto ec = this->_engine->query(obj, host, date, length); + if ( ec == system::errc::success ) { + ec = this->_engine->del(obj); + access = this->network_file(host, obj); + if ( ec == system::errc::success ) { + ::lib_rtl_output(_prt(this), "Remove '%s'", obj.c_str()); + return ec; + } + } + ::lib_rtl_output(_prt(this), "FAILED to remove: '%s' [%s]", + object.c_str(), ec.message().c_str()); + return ec; +} + +/// Add a new object to the database +system::error_code +fdb_dbase_t::add_object(const std::string& object, + const std::string& date, + std::size_t length, + std::string& access) +{ + std::string obj = this->object_key(object); + std::string host = this->get_free_host_uri(object); + auto ec = this->_engine->add(obj, date, length, host); + if ( ec == system::errc::success ) { + access = this->network_file(host, obj); + ::lib_rtl_output(_prt(this), "Add '%s' %s %s", object.c_str(), + this->_debug > 1 ? "access:" : "", + this->_debug > 1 ? access.c_str() : ""); + return ec; + } + ::lib_rtl_output(LIB_RTL_ERROR,"FAILED to add: '%s' access: %s [%s]", + object.c_str(), access.c_str(), ec.message().c_str()); + return ec; +} + +/// Update the object state of an object in the database +boost::system::error_code +fdb_dbase_t::update_object_state(const std::string& object, const std::string& state) { + file_t file; + system::error_code ec; + std::string obj = this->object_key(object); + if ( state == "WRITTEN" ) { + ec = this->_engine->query_file(obj, file, handler_t::STATE_OPEN); + if ( ec == system::errc::success ) { + ec = this->_engine->set(obj, handler_t::STATE_WRITTEN); + } + } + else if ( state == "READ" ) { + ec = this->_engine->query_file(obj, file, handler_t::STATE_WRITTEN); + if ( ec == system::errc::success ) { + ec = this->_engine->set(obj, handler_t::STATE_READ); + } + } + if ( ec == system::errc::success ) { + ::lib_rtl_output(this->_debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, + "Update '%s' NEW state: %s", object.c_str(), state.c_str()); + return ec; + } + ::lib_rtl_output(LIB_RTL_ERROR,"FAILED update '%s' to state: %s [%s]", + object.c_str(), state.c_str(), ec.message().c_str()); + return ec; +} diff --git a/Online/Storage/src/server/sqlite_test.cpp b/Online/Storage/src/server/sqlite_test.cpp index 3ebc8f0d4..af0577abe 100644 --- a/Online/Storage/src/server/sqlite_test.cpp +++ b/Online/Storage/src/server/sqlite_test.cpp @@ -27,6 +27,7 @@ gentest.exe libStorageServer.so fdb_cli_dumpdb -database=/home/frankm/storage_fi #include <CPP/sqlite.h> #include <HTTP/HttpHeader.h> #include <RTL/Logger.h> +#include "fdb_SQLite.h" // C/C++ include files #include <fstream> @@ -35,7 +36,11 @@ gentest.exe libStorageServer.so fdb_cli_dumpdb -database=/home/frankm/storage_fi using namespace std; using namespace sqlite; +// +// +//========================================================================== namespace { + struct Setup { string name, database, stream, log, entry; size_t count = 0; @@ -71,7 +76,9 @@ namespace { } }; } - +// +// +//========================================================================== extern "C" int sqlite_create_database(int argc, char** argv) { Setup setup("sqlite_create_database", argc, argv); auto db = setup.db(); @@ -81,7 +88,7 @@ extern "C" int sqlite_create_database(int argc, char** argv) { "State INT NOT NULL, " "Size INT NOT NULL, " "Date TEXT NOT NULL, " - "Access TEXT NOT NULL)"; + "Host TEXT NOT NULL)"; if ( db.execute(error, sql) != sqlite::traits::OK ) { ::lib_rtl_output(LIB_RTL_ERROR,"%s: failed to execute %s : %s", db.name(), sql, error.c_str()); @@ -90,30 +97,31 @@ extern "C" int sqlite_create_database(int argc, char** argv) { db.close(); return 0; } - +// +// +//========================================================================== extern "C" int sqlite_populate_database(int argc, char** argv) { Setup setup("sqlite_populate_database", argc, argv); auto db = setup.db(); try { sqlite::statement insert = statement::create(db, - "INSERT INTO Files ('Name', 'State', 'Size', 'Date', 'Access') " + "INSERT INTO Files ('Name', 'State', 'Size', 'Date', 'Host') " "VALUES ( ? , ? , ? , ? , ? )"); for(size_t i=0; i<setup.count; ++i) { std::string date = http::HttpHeader::now(); + std::string host = "http://127.0.0.1:8100"; size_t length = 4*1024; - char obj[1024], access[1024]; + char obj[1024]; ::snprintf(obj, sizeof(obj), "/objects/%08ld/%s/%010ld_%08ld.raw", setup.run, setup.stream.c_str(), setup.run, i); - ::snprintf(access, sizeof(access), "/files/daqarea/lhcb/%08ld/%s/%010ld_%08ld.raw", - setup.run, setup.stream.c_str(), setup.run, i); insert.reset(); insert.bind(1, obj); insert.bind(2, 0); insert.bind(3, length); insert.bind(4, date); - insert.bind(5, access); + insert.bind(5, host); int ret = insert.execute(); if ( ret != sqlite::traits::OK ) { ::lib_rtl_output(LIB_RTL_ERROR,"%s: failed INSERT: %s",db.name(),insert.errmsg()); @@ -130,24 +138,25 @@ extern "C" int sqlite_populate_database(int argc, char** argv) { } return 0; } - +// +// +//========================================================================== extern "C" int sqlite_populate_database_direct(int argc, char** argv) { Setup setup("sqlite_populate_database", argc, argv); auto db = setup.db(); try { for(size_t i=0; i<setup.count; ++i) { std::string error, date = http::HttpHeader::now(); + std::string host = "http://127.0.0.1:8100"; size_t length = 4*1024; - char obj[1024], access[1024]; + char obj[1024]; ::snprintf(obj, sizeof(obj), "/objects/%08ld/%s/%010ld_%08ld.raw", setup.run, setup.stream.c_str(), setup.run, i); - ::snprintf(access, sizeof(access), "/files/daqarea/lhcb/%08ld/%s/%010ld_%08ld.raw", - setup.run, setup.stream.c_str(), setup.run, i); int ret = db.execute_sql(error, - "INSERT INTO Files ('Name', 'State', 'Size', 'Date', 'Access') " + "INSERT INTO Files ('Name', 'State', 'Size', 'Date', 'Host') " "VALUES ( '%s' , %d , %ld , '%s' , '%s' )", - obj, 0, length, date.c_str(), access); + obj, 0, length, date.c_str(), host.c_str()); if ( ret != sqlite::traits::OK ) { ::lib_rtl_output(LIB_RTL_ERROR,"%s: failed INSERT: %s",db.name(),db.errmsg()); return EINVAL; @@ -161,16 +170,16 @@ extern "C" int sqlite_populate_database_direct(int argc, char** argv) { } return 0; } - +// +// +//========================================================================== extern "C" int sqlite_read_database_direct(int argc, char** argv) { - struct File { - std::string name, state, size, date, access; - }; + typedef Online::storage::fdb_dbase_t::file_t file_t; struct Reader { - std::vector<File> files; + std::vector<file_t> files; static int callback(void* ptr, int /* argc */, char **argv, char **/*name*/) { Reader* rdr = (Reader*)ptr; - File file {argv[0], argv[1], argv[2], argv[3], argv[4]}; + file_t file {argv[0], argv[1], argv[2], argv[3], argv[4]}; rdr->files.emplace_back(file); return 0; } @@ -181,7 +190,7 @@ extern "C" int sqlite_read_database_direct(int argc, char** argv) { int ret = db.execute_sql(error, &reader, Reader::callback, - "SELECT Name, State, Size, Date, Access" + "SELECT Name, State, Size, Date, Host" " FROM Files" " WHERE Size>%d",0); if ( ret != sqlite::traits::OK ) { @@ -189,8 +198,8 @@ extern "C" int sqlite_read_database_direct(int argc, char** argv) { try { std::stringstream s; for(size_t i=0; i<reader.files.size(); ++i) { - const File& f = reader.files[i]; - s << f.name << " '" << f.state << "' '" << f.size << "' '" << f.date << "' " << f.access; + const file_t& f = reader.files[i]; + s << f.name << " '" << f.state << "' '" << f.size << "' '" << f.date << "' " << f.host; ::lib_rtl_output(LIB_RTL_INFO, "TABLE: %s", s.str().c_str()); s.str(""); } @@ -202,16 +211,16 @@ extern "C" int sqlite_read_database_direct(int argc, char** argv) { } return 0; } - +// +// +//========================================================================== extern "C" int sqlite_update_database_direct(int argc, char** argv) { - struct File { - std::string name, state, size, date, access; - }; + typedef Online::storage::fdb_dbase_t::file_t file_t; struct Reader { - std::vector<File> files; + std::vector<file_t> files; static int callback(void* ptr, int /* argc */, char** argv, char** /* name */) { Reader* rdr = (Reader*)ptr; - File file {argv[0], argv[1], argv[2], argv[3], argv[4]}; + file_t file {argv[0], argv[1], argv[2], argv[3], argv[4]}; rdr->files.emplace_back(file); return 0; } @@ -224,14 +233,14 @@ extern "C" int sqlite_update_database_direct(int argc, char** argv) { int ret = db.execute_sql(error, &reader, Reader::callback, - "SELECT Name, State, Size, Date, Access" + "SELECT Name, State, Size, Date, Host" " FROM Files" " WHERE Size>%d",0); if ( ret != sqlite::traits::OK ) { ::lib_rtl_output(LIB_RTL_INFO, "SQL Failure: %s", error.c_str()); } for(size_t i=0; i<reader.files.size(); ++i) { - const File& f = reader.files[i]; + const file_t& f = reader.files[i]; if ( f.state == "0" ) { ret = db.execute_sql(error, "UPDATE Files" @@ -262,11 +271,11 @@ extern "C" int sqlite_update_database_direct(int argc, char** argv) { // //========================================================================== extern "C" int fdb_cli_dumpdb(int argc, char** argv) { - struct File { std::string name, state, size, date, access; }; + typedef Online::storage::fdb_dbase_t::file_t file_t; struct Reader { static int callback(void* ptr, int /* argc */, char **argv, char **/*name*/) { - std::vector<File>* f = (std::vector<File>*)ptr; - File file {argv[0], argv[1], argv[2], argv[3], argv[4]}; + std::vector<file_t>* f = (std::vector<file_t>*)ptr; + file_t file {argv[0], argv[1], argv[2], argv[3], argv[4]}; f->emplace_back(file); return 0; } @@ -278,7 +287,7 @@ extern "C" int fdb_cli_dumpdb(int argc, char** argv) { " -help Print this help. \n"); }); string error; - std::vector<File> files; + std::vector<file_t> files; sqlite::database database = setup.db(); if ( database.handle() == nullptr ) { ::lib_rtl_output(LIB_RTL_ERROR,"%s: Can't open database [%s]", @@ -286,16 +295,16 @@ extern "C" int fdb_cli_dumpdb(int argc, char** argv) { return EINVAL; } int ret = database.execute_sql(error, &files, Reader::callback, - "SELECT Name, State, Size, Date, Access" + "SELECT Name, State, Size, Date, Host" " FROM Files" " WHERE Size>0"); if ( ret == sqlite::traits::OK ) { for(size_t i=0; i<files.size(); ++i) { - const File& f = files[i]; - ::lib_rtl_output(LIB_RTL_INFO, "%4ld File: %s %9s bytes %s", - i, f.name.c_str(), f.size.c_str(), f.date.c_str()); - ::lib_rtl_output(LIB_RTL_INFO, " Key: %s State:%s", - f.access.c_str(), f.state.c_str()); + const file_t& f = files[i]; + ::lib_rtl_output(LIB_RTL_INFO, "%4ld File: %s Host: %s %s", + i, f.name.c_str(), f.host.c_str(), f.date.c_str()); + ::lib_rtl_output(LIB_RTL_INFO, " State:%s %9s bytes", + f.state.c_str(), f.size.c_str()); } } database.close(); diff --git a/Online/Tell1Data/Tell1Data/Tell1Bank.h b/Online/Tell1Data/Tell1Data/Tell1Bank.h index 0c7d302b1..4f4e263f5 100755 --- a/Online/Tell1Data/Tell1Data/Tell1Bank.h +++ b/Online/Tell1Data/Tell1Data/Tell1Bank.h @@ -57,88 +57,104 @@ namespace Online { public: - enum BankType { - L0Calo=0, // 0 - L0DU, // 1 - PrsE, // 2 - EcalE, // 3 - HcalE, // 4 - PrsTrig, // 5 - EcalTrig, // 6 - HcalTrig, // 7 - Velo, // 8 - Rich, // 9 - TT, // 10 - IT, // 11 - OT, // 12 - Muon, // 13 - L0PU, // 14 - DAQ, // 15 - ODIN, // 16 - HltDecReports, // 17 - VeloFull, // 18 - TTFull, // 19 - ITFull, // 20 - EcalPacked, // 21 - HcalPacked, // 22 - PrsPacked, // 23 - L0Muon, // 24 - ITError, // 25 - TTError, // 26 - ITPedestal, // 27 - TTPedestal, // 28 - VeloError, // 29 - VeloPedestal, // 30 - VeloProcFull, // 31 - OTRaw, // 32 - OTError, // 33 - EcalPackedError, // 34 - HcalPackedError, // 35 - PrsPackedError, // 36 - L0CaloFull, // 37 - L0CaloError, // 38 - L0MuonCtrlAll, // 39 - L0MuonProcCand, // 40 - L0MuonProcData, // 41 - L0MuonRaw, // 42 - L0MuonError, // 43 - GaudiSerialize, // 44 - GaudiHeader, // 45 - TTProcFull, // 46 - ITProcFull, // 47 - TAEHeader, // 48 - MuonFull, // 49 - MuonError, // 50 - TestDet, // 51 - L0DUError, // 52 - HltRoutingBits, // 53 - HltSelReports, // 54 - HltVertexReports,// 55 - HltLumiSummary, // 56 - L0PUFull, // 57 - L0PUError, // 58 - DstBank, // 59 - DstData, // 60 - DstAddress, // 61 - FileID, // 62 - VP, // 63 - FTCluster, // 64 - VL, // 65 - UT, // 66 - UTFull, // 67 - UTError, // 68 - UTPedestal, // 69 - HC, // 70 - HltTrackReports, // 71 - HCError, // 72 - VPRetinaCluster, // 73 - FTGeneric, // 74 - FTCalibration, // 75 - FTNZS, // 76 - // Add new types here. Don't forget to update also RawBank.cpp - LastType, // LOOP Marker; add new bank types ONLY before! + /// Define bank types for RawBank + enum BankType { + L0Calo = 0, // 0 + L0DU, // 1 + PrsE, // 2 + EcalE, // 3 + HcalE, // 4 + PrsTrig, // 5 + EcalTrig, // 6 + HcalTrig, // 7 + Velo, // 8 + Rich, // 9 + TT, // 10 + IT, // 11 + OT, // 12 + Muon, // 13 + L0PU, // 14 + DAQ, // 15 + ODIN, // 16 + HltDecReports, // 17 + VeloFull, // 18 + TTFull, // 19 + ITFull, // 20 + EcalPacked, // 21 + HcalPacked, // 22 + PrsPacked, // 23 + L0Muon, // 24 + ITError, // 25 + TTError, // 26 + ITPedestal, // 27 + TTPedestal, // 28 + VeloError, // 29 + VeloPedestal, // 30 + VeloProcFull, // 31 + OTRaw, // 32 + OTError, // 33 + EcalPackedError, // 34 + HcalPackedError, // 35 + PrsPackedError, // 36 + L0CaloFull, // 37 + L0CaloError, // 38 + L0MuonCtrlAll, // 39 + L0MuonProcCand, // 40 + L0MuonProcData, // 41 + L0MuonRaw, // 42 + L0MuonError, // 43 + GaudiSerialize, // 44 + GaudiHeader, // 45 + TTProcFull, // 46 + ITProcFull, // 47 + TAEHeader, // 48 + MuonFull, // 49 + MuonError, // 50 + TestDet, // 51 + L0DUError, // 52 + HltRoutingBits, // 53 + HltSelReports, // 54 + HltVertexReports, // 55 + HltLumiSummary, // 56 + L0PUFull, // 57 + L0PUError, // 58 + DstBank, // 59 + DstData, // 60 + DstAddress, // 61 + FileID, // 62 + VP, // 63 + FTCluster, // 64 + VL, // 65 + UT, // 66 + UTFull, // 67 + UTError, // 68 + UTPedestal, // 69 + HC, // 70 + HltTrackReports, // 71 + HCError, // 72 + VPRetinaCluster, // 73 + FTGeneric, // 74 + FTCalibration, // 75 + FTNZS, // 76 + Calo, // 77 + CaloError, // 78 + MuonSpecial, // 79 + RichCommissioning, // 80 + RichError, // 81 + FTSpecial, // 82 - DaqErrorBase = 192 // Lowest DaqError type available in Run 3 (EDMS2100937) + // Add new types here. Don't forget to update also RawBank.cpp + LastType, // LOOP Marker; add new bank types ONLY before! + + DaqErrorBase = 192, // Lowest DaqError type available in Run 3 (EDMS2100937) + // Banks above are reserved for DAQ, add only generic DaqError types below. + DaqErrorFragmentThrottled = 250, + DaqErrorBXIDCorrupted = 251, + DaqErrorBXIDJump = 252, + DaqErrorFragmentMissing = 253, + DaqErrorFragmentTruncated = 254, + DaqErrorInvalid = 255 + // 255 is the highest type allowed by the Run 3 raw-data format (8-bit unsigned) }; /// Magic pattern for Raw bank headers -- GitLab From 8f842d68731d8d0651146b8de7b7eee0376c4858 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Sun, 28 Mar 2021 21:17:38 +0200 Subject: [PATCH 09/26] Data flow updates --- .../Dataflow/src/framework/DataflowTask.cpp | 2 +- .../options/EB_Transport_properties.opts | 2 +- Online/EventBuilding/src/BU.cpp | 2 +- Online/EventBuilding/src/RU.cpp | 2 +- Online/EventData/CMakeLists.txt | 23 ++ .../EventData}/EventAccess.h | 20 +- .../EventData}/EventOutput.h | 8 +- .../EventData}/EventTraits.h | 22 +- .../EventData}/FileEventAccess.h | 8 +- .../EventData}/FileEventOutput.h | 8 +- .../EventData}/MBMEventAccess.h | 8 +- .../EventData}/MBMEventOutput.h | 8 +- .../EventData}/Macros.h | 6 +- .../EventData}/NetEventAccess.h | 12 +- .../src/EventAccess.cpp | 24 +- .../src/EventOutput.cpp | 2 +- .../src/FileEventAccess.cpp | 2 +- .../src/FileEventOutput.cpp | 2 +- .../src/MBMEventAccess.cpp | 2 +- .../src/MBMEventOutput.cpp | 2 +- .../src/NetEventAccess.cpp | 2 +- Online/FarmConfig/job/EBPass.sh | 9 +- Online/GaudiOnline/CMakeLists.txt | 9 +- .../GaudiOnline/GaudiOnline/Configuration.h | 8 +- Online/GaudiOnline/GaudiOnline/ITaskFSM.h | 1 + .../GaudiOnline/OnlineApplication.h | 1 - .../GaudiOnline/QueuedFlowManager.h | 9 +- Online/GaudiOnline/components/Components.cpp | 6 +- .../GaudiOnline/components/EventProcessor.cpp | 389 ++++++++++-------- .../GaudiOnline/components/EventProcessor.h | 48 ++- Online/GaudiOnline/components/ExpandTAE.cpp | 2 +- Online/GaudiOnline/components/ForceTAE.cpp | 2 +- Online/GaudiOnline/components/IOService.h | 2 +- Online/GaudiOnline/components/InputAlg.cpp | 50 +-- Online/GaudiOnline/components/InputAlg.h | 21 +- .../GaudiOnline/components/OnlineEventApp.cpp | 6 +- .../GaudiOnline/components/OnlineEventApp.h | 7 +- Online/GaudiOnline/components/OutputAlg.cpp | 63 ++- Online/GaudiOnline/components/Passthrough.cpp | 140 +++---- .../python/GaudiOnline/OnlineApplication.py | 2 + .../python/GaudiOnline/Passthrough.py | 3 +- .../python/GaudiOnline/ReadDatafile.py | 3 +- .../GaudiOnline/configureEventConsumer.py | 2 +- Online/GaudiOnline/src/Configuration.cpp | 4 +- Online/GaudiOnline/src/OnlineApplication.cpp | 4 +- Online/GaudiOnline/src/QueuedFlowManager.cpp | 2 + .../components/RawEventTestDump.cpp | 24 +- Online/OnlineBase/OnlineBase/RTL/Logger.h | 11 + Online/OnlineBase/src/RTL/Logger.cpp | 25 +- Online/PCIE40Data/PCIE40Data/pcie40.h | 12 +- Online/PCIE40Data/PCIE40Data/pcie40decoder.h | 15 +- Online/PCIE40Data/src/pcie40decoder.cpp | 279 ++++++++----- Online/SmiController/src/SmiController.cpp | 6 +- 53 files changed, 786 insertions(+), 546 deletions(-) create mode 100755 Online/EventData/CMakeLists.txt rename Online/{GaudiOnline/GaudiOnline => EventData/EventData}/EventAccess.h (96%) rename Online/{GaudiOnline/GaudiOnline => EventData/EventData}/EventOutput.h (98%) rename Online/{GaudiOnline/GaudiOnline => EventData/EventData}/EventTraits.h (88%) rename Online/{GaudiOnline/GaudiOnline => EventData/EventData}/FileEventAccess.h (96%) rename Online/{GaudiOnline/GaudiOnline => EventData/EventData}/FileEventOutput.h (91%) rename Online/{GaudiOnline/GaudiOnline => EventData/EventData}/MBMEventAccess.h (95%) rename Online/{GaudiOnline/GaudiOnline => EventData/EventData}/MBMEventOutput.h (93%) rename Online/{GaudiOnline/GaudiOnline => EventData/EventData}/Macros.h (90%) rename Online/{GaudiOnline/GaudiOnline => EventData/EventData}/NetEventAccess.h (94%) rename Online/{GaudiOnline => EventData}/src/EventAccess.cpp (96%) rename Online/{GaudiOnline => EventData}/src/EventOutput.cpp (99%) rename Online/{GaudiOnline => EventData}/src/FileEventAccess.cpp (99%) rename Online/{GaudiOnline => EventData}/src/FileEventOutput.cpp (99%) rename Online/{GaudiOnline => EventData}/src/MBMEventAccess.cpp (99%) rename Online/{GaudiOnline => EventData}/src/MBMEventOutput.cpp (99%) rename Online/{GaudiOnline => EventData}/src/NetEventAccess.cpp (99%) diff --git a/Online/Dataflow/src/framework/DataflowTask.cpp b/Online/Dataflow/src/framework/DataflowTask.cpp index c8133babf..24de39174 100755 --- a/Online/Dataflow/src/framework/DataflowTask.cpp +++ b/Online/Dataflow/src/framework/DataflowTask.cpp @@ -320,7 +320,7 @@ int DataflowTask::unload() { app_terminate(); declareState(ST_OFFLINE); ::lib_rtl_sleep(100); - ::exit(0); + ::_exit(0); return DF_SUCCESS; } diff --git a/Online/EventBuilding/options/EB_Transport_properties.opts b/Online/EventBuilding/options/EB_Transport_properties.opts index 9ce1e0eb8..d365afbf6 100755 --- a/Online/EventBuilding/options/EB_Transport_properties.opts +++ b/Online/EventBuilding/options/EB_Transport_properties.opts @@ -2,7 +2,7 @@ EB_transport.BU_ranks = {}; EB_transport.MPI_errors_return = FALSE; EB_transport.RU_ranks = {}; EB_transport.RUs_per_nic = {}; -EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.J25197.json"; +EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.FV2976.json"; EB_transport.n_par_mess = 0; EB_transport.n_sources_per_ru = { 1, 1, 3, 3, 2, 2 }; EB_transport.src_ids = {}; diff --git a/Online/EventBuilding/src/BU.cpp b/Online/EventBuilding/src/BU.cpp index 53ac993d9..b13c621a2 100644 --- a/Online/EventBuilding/src/BU.cpp +++ b/Online/EventBuilding/src/BU.cpp @@ -585,7 +585,7 @@ int EB::BU::receive_sizes() if (_full_size_words == _data_offset_words[0]) { _full_size_words = 0; std::fill(_data_offset_words.begin(), _data_offset_words.end(), 0); - logger.info << "end of run" << std::flush; + logger.warning << "end of run" << std::flush; } else { // If only a subset of the MFPs are 0-sized the MEP is incomplete the src ids are reported and the BU goes into // ERROR diff --git a/Online/EventBuilding/src/RU.cpp b/Online/EventBuilding/src/RU.cpp index c17071f2d..e436c23f2 100644 --- a/Online/EventBuilding/src/RU.cpp +++ b/Online/EventBuilding/src/RU.cpp @@ -620,7 +620,7 @@ int EB::RU::load_mpfs() // the end of run will trigger a round of zero-sized MFPs _end_of_run = true; _send_empty = true; - logger.info << "end of run" << std::flush; + logger.warning << "end of run" << std::flush; break; } else { _end_of_run = false; diff --git a/Online/EventData/CMakeLists.txt b/Online/EventData/CMakeLists.txt new file mode 100755 index 000000000..7d3629849 --- /dev/null +++ b/Online/EventData/CMakeLists.txt @@ -0,0 +1,23 @@ +############################################################################### +# (c) Copyright 2000-2019 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +# Package: EventData +############################################################################### +gaudi_subdir(EventData v0r1) +# +gaudi_depends_on_subdirs(Online/Tell1Data + Online/PCIE40Data + Online/OnlineBase) +# +gaudi_add_library(EventData src/*.cpp + PUBLIC_HEADERS EventData + INCLUDE_DIRS OnlineBase Tell1Data PCIE40Data + LINK_LIBRARIES OnlineBase Tell1Data PCIE40Data) +target_compile_options(EventData PRIVATE -Wno-address-of-packed-member) diff --git a/Online/GaudiOnline/GaudiOnline/EventAccess.h b/Online/EventData/EventData/EventAccess.h similarity index 96% rename from Online/GaudiOnline/GaudiOnline/EventAccess.h rename to Online/EventData/EventData/EventAccess.h index 999d8d278..278d9df23 100644 --- a/Online/GaudiOnline/GaudiOnline/EventAccess.h +++ b/Online/EventData/EventData/EventAccess.h @@ -12,13 +12,12 @@ // // Author : Markus Frank //========================================================================== -#ifndef ONLINE_GAUDIONLINE_EVENTACCESS_H -#define ONLINE_GAUDIONLINE_EVENTACCESS_H +#ifndef ONLINE_EVENTDATA_EVENTACCESS_H +#define ONLINE_EVENTDATA_EVENTACCESS_H /// Framework include files -#include <GaudiOnline/Macros.h> -#include <GaudiOnline/EventTraits.h> - +#include <EventData/Macros.h> +#include <EventData/EventTraits.h> #include <CPP/Interactor.h> #include <RTL/Logger.h> #include "RTL/Pack.h" @@ -37,6 +36,7 @@ /// Online namespace declaration namespace Online { + /// Base class to access online event data /** @class EventAccess * @@ -129,6 +129,13 @@ namespace Online { /// Container to bank referentes to MDF headers containing event blocks burst_collection_t m_bursts; + /// Definition of the monitoring data block + /** + * + * \author M.Frank + * \version 1.0 + * \date 25/04/2019 + */ struct monitor_t { /// Monitoring item: Count of bursts which were received from the input source std::size_t burstsIn = 0; @@ -166,6 +173,7 @@ namespace Online { /// Default constructor EventAccess(std::unique_ptr<RTL::Logger>&& logger); + /// Standard destructor virtual ~EventAccess(); @@ -225,4 +233,4 @@ namespace Online { }; } // End namespace Online #include "RTL/Unpack.h" -#endif // ONLINE_GAUDIONLINE_EVENTACCESS_H +#endif // ONLINE_EVENTDATA_EVENTACCESS_H diff --git a/Online/GaudiOnline/GaudiOnline/EventOutput.h b/Online/EventData/EventData/EventOutput.h similarity index 98% rename from Online/GaudiOnline/GaudiOnline/EventOutput.h rename to Online/EventData/EventData/EventOutput.h index ba9712480..83bdd95f5 100644 --- a/Online/GaudiOnline/GaudiOnline/EventOutput.h +++ b/Online/EventData/EventData/EventOutput.h @@ -12,11 +12,11 @@ // // Author : Markus Frank //========================================================================== -#ifndef ONLINE_GAUDIONLINE_EVENTOUTPUT_H -#define ONLINE_GAUDIONLINE_EVENTOUTPUT_H +#ifndef ONLINE_EVENTDATA_EVENTOUTPUT_H +#define ONLINE_EVENTDATA_EVENTOUTPUT_H /// Framework include files -#include <GaudiOnline/Macros.h> +#include <EventData/Macros.h> #include <Tell1Data/Tell1Bank.h> #include <CPP/Interactor.h> #include <RTL/Logger.h> @@ -223,4 +223,4 @@ namespace Online { } } // End namespace Online -#endif // ONLINE_GAUDIONLINE_EVENTOUTPUT_H +#endif // ONLINE_EVENTDATA_EVENTOUTPUT_H diff --git a/Online/GaudiOnline/GaudiOnline/EventTraits.h b/Online/EventData/EventData/EventTraits.h similarity index 88% rename from Online/GaudiOnline/GaudiOnline/EventTraits.h rename to Online/EventData/EventData/EventTraits.h index 86f5fe93d..f8f89db03 100644 --- a/Online/GaudiOnline/GaudiOnline/EventTraits.h +++ b/Online/EventData/EventData/EventTraits.h @@ -12,8 +12,8 @@ // // Author : Markus Frank //========================================================================== -#ifndef ONLINE_GAUDIONLINE_EVENTTRAITS_H -#define ONLINE_GAUDIONLINE_EVENTTRAITS_H +#ifndef ONLINE_EVENTDATA_EVENTTRAITS_H +#define ONLINE_EVENTDATA_EVENTTRAITS_H /// Framework include files #include <PCIE40Data/pcie40.h> @@ -66,6 +66,16 @@ namespace Online { record_t(const pcie40::event_t* p) { pointer = p; } record_t(const record_t& p) { pointer = p.data; } operator bool () const { return data != 0; } + bool is_tell1() const { + return data != nullptr && + tell1_event->size0() > 0 && + tell1_event->size0() == tell1_event->size1() && + tell1_event->size0() == tell1_event->size2(); + } + bool is_tell40() const { + return data != nullptr && + tell40_event->pattern() == pcie40::MAGIC_PATTERN; + } }; /// Event container definition for bursts: Contains only the MDFHeader banks typedef std::vector<record_t> event_collection_t; @@ -83,6 +93,9 @@ namespace Online { static event_type& get_event(record_t r) { return *(r.tell1_event); } + static const odin_type* get_odin(const bank_type* b) { + return b ? b->begin<odin_type>() : nullptr; + } static size_t num_collections(const event_type&) { return 1; } @@ -117,6 +130,9 @@ namespace Online { static event_type& get_event(record_t r) { return *(r.tell40_event); } + static const odin_type* get_odin(const bank_type* b) { + return b ? b->begin<odin_type>() : nullptr; + } static size_t num_collections(const event_type& e) { return e.num_bank_collections(); } @@ -147,4 +163,4 @@ namespace Online { } } // End namespace Online -#endif // ONLINE_GAUDIONLINE_EVENTTRAITS_H +#endif // ONLINE_EVENTDATA_EVENTTRAITS_H diff --git a/Online/GaudiOnline/GaudiOnline/FileEventAccess.h b/Online/EventData/EventData/FileEventAccess.h similarity index 96% rename from Online/GaudiOnline/GaudiOnline/FileEventAccess.h rename to Online/EventData/EventData/FileEventAccess.h index 0f68b5cdb..cb68e925b 100644 --- a/Online/GaudiOnline/GaudiOnline/FileEventAccess.h +++ b/Online/EventData/EventData/FileEventAccess.h @@ -12,11 +12,11 @@ // // Author : Markus Frank //========================================================================== -#ifndef ONLINE_GAUDIONLINE_FILEEVENTACCESS_H -#define ONLINE_GAUDIONLINE_FILEEVENTACCESS_H +#ifndef ONLINE_EVENTDATA_FILEEVENTACCESS_H +#define ONLINE_EVENTDATA_FILEEVENTACCESS_H /// Framework include files -#include "GaudiOnline/EventAccess.h" +#include "EventData/EventAccess.h" #include "Tell1Data/DirectoryScan.h" #include "MBM/Requirement.h" @@ -119,4 +119,4 @@ namespace Online { virtual void close() override; }; } // End namespace Online -#endif // ONLINE_GAUDIONLINE_FILEEVENTACCESS_H +#endif // ONLINE_EVENTDATA_FILEEVENTACCESS_H diff --git a/Online/GaudiOnline/GaudiOnline/FileEventOutput.h b/Online/EventData/EventData/FileEventOutput.h similarity index 91% rename from Online/GaudiOnline/GaudiOnline/FileEventOutput.h rename to Online/EventData/EventData/FileEventOutput.h index ec0dd6065..65e3937e9 100644 --- a/Online/GaudiOnline/GaudiOnline/FileEventOutput.h +++ b/Online/EventData/EventData/FileEventOutput.h @@ -12,11 +12,11 @@ // // Author : Markus Frank //========================================================================== -#ifndef ONLINE_GAUDIONLINE_FILEEVENTOUTPUT_H -#define ONLINE_GAUDIONLINE_FILEEVENTOUTPUT_H +#ifndef ONLINE_EVENTDATA_FILEEVENTOUTPUT_H +#define ONLINE_EVENTDATA_FILEEVENTOUTPUT_H /// Framework include files -#include <GaudiOnline/EventOutput.h> +#include <EventData/EventOutput.h> /// C/C++ include files #include <list> @@ -61,4 +61,4 @@ namespace Online { virtual int cancel() override; }; } // End namespace Online -#endif // ONLINE_GAUDIONLINE_FILEEVENTOUTPUT_H +#endif // ONLINE_EVENTDATA_FILEEVENTOUTPUT_H diff --git a/Online/GaudiOnline/GaudiOnline/MBMEventAccess.h b/Online/EventData/EventData/MBMEventAccess.h similarity index 95% rename from Online/GaudiOnline/GaudiOnline/MBMEventAccess.h rename to Online/EventData/EventData/MBMEventAccess.h index 696de69dd..847cc399f 100644 --- a/Online/GaudiOnline/GaudiOnline/MBMEventAccess.h +++ b/Online/EventData/EventData/MBMEventAccess.h @@ -12,11 +12,11 @@ // // Author : Markus Frank //========================================================================== -#ifndef ONLINE_GAUDIONLINE_MBMEVENTACCESS_H -#define ONLINE_GAUDIONLINE_MBMEVENTACCESS_H +#ifndef ONLINE_EVENTDATA_MBMEVENTACCESS_H +#define ONLINE_EVENTDATA_MBMEVENTACCESS_H /// Framework include files -#include "GaudiOnline/EventAccess.h" +#include "EventData/EventAccess.h" #include "MBM/Requirement.h" #include "MBM/bmdef.h" @@ -113,4 +113,4 @@ namespace Online { virtual void close() override; }; } // End namespace Online -#endif // ONLINE_GAUDIONLINE_MBMEVENTACCESS_H +#endif // ONLINE_EVENTDATA_MBMEVENTACCESS_H diff --git a/Online/GaudiOnline/GaudiOnline/MBMEventOutput.h b/Online/EventData/EventData/MBMEventOutput.h similarity index 93% rename from Online/GaudiOnline/GaudiOnline/MBMEventOutput.h rename to Online/EventData/EventData/MBMEventOutput.h index 99e9b3793..7a60e2978 100644 --- a/Online/GaudiOnline/GaudiOnline/MBMEventOutput.h +++ b/Online/EventData/EventData/MBMEventOutput.h @@ -12,11 +12,11 @@ // // Author : Markus Frank //========================================================================== -#ifndef ONLINE_GAUDIONLINE_MBMEVENTOUTPUT_H -#define ONLINE_GAUDIONLINE_MBMEVENTOUTPUT_H +#ifndef ONLINE_EVENTDATA_MBMEVENTOUTPUT_H +#define ONLINE_EVENTDATA_MBMEVENTOUTPUT_H /// Framework include files -#include <GaudiOnline/EventOutput.h> +#include <EventData/EventOutput.h> #include <MBM/BufferInfo.h> #include <MBM/bmdef.h> @@ -83,4 +83,4 @@ namespace Online { virtual int cancel() override; }; } // End namespace Online -#endif // ONLINE_GAUDIONLINE_MBMEVENTOUTPUT_H +#endif // ONLINE_EVENTDATA_MBMEVENTOUTPUT_H diff --git a/Online/GaudiOnline/GaudiOnline/Macros.h b/Online/EventData/EventData/Macros.h similarity index 90% rename from Online/GaudiOnline/GaudiOnline/Macros.h rename to Online/EventData/EventData/Macros.h index 60977c670..388d900a9 100644 --- a/Online/GaudiOnline/GaudiOnline/Macros.h +++ b/Online/EventData/EventData/Macros.h @@ -12,8 +12,8 @@ // // Author : Markus Frank //========================================================================== -#ifndef ONLINE_GAUDIONLINE_MACROS_H -#define ONLINE_GAUDIONLINE_MACROS_H +#ifndef ONLINE_EVENTDATA_MACROS_H +#define ONLINE_EVENTDATA_MACROS_H /// Framework include files @@ -38,4 +38,4 @@ namespace Online { x& operator=(const x& copy) = delete; \ x& operator=(x&& copy) = delete -#endif // ONLINE_GAUDIONLINE_MACROS_H +#endif // ONLINE_EVENTDATA_MACROS_H diff --git a/Online/GaudiOnline/GaudiOnline/NetEventAccess.h b/Online/EventData/EventData/NetEventAccess.h similarity index 94% rename from Online/GaudiOnline/GaudiOnline/NetEventAccess.h rename to Online/EventData/EventData/NetEventAccess.h index 2132458ad..a0e246393 100644 --- a/Online/GaudiOnline/GaudiOnline/NetEventAccess.h +++ b/Online/EventData/EventData/NetEventAccess.h @@ -12,14 +12,14 @@ // // Author : Markus Frank //========================================================================== -#ifndef ONLINE_GAUDIONLINE_NETEVENTACCESS_H -#define ONLINE_GAUDIONLINE_NETEVENTACCESS_H +#ifndef ONLINE_EVENTDATA_NETEVENTACCESS_H +#define ONLINE_EVENTDATA_NETEVENTACCESS_H /// Framework include files #define TRANSFER_NS BoostAsio -#include "NET/Transfer.h" -#include "MBM/Requirement.h" -#include "GaudiOnline/EventAccess.h" +#include <NET/Transfer.h> +#include <MBM/Requirement.h> +#include <EventData/EventAccess.h> /// Online namespace declaration namespace Online { @@ -113,4 +113,4 @@ namespace Online { virtual void close() override; }; } // End namespace Online -#endif // ONLINE_GAUDIONLINE_NETEVENTACCESS_H +#endif // ONLINE_EVENTDATA_NETEVENTACCESS_H diff --git a/Online/GaudiOnline/src/EventAccess.cpp b/Online/EventData/src/EventAccess.cpp similarity index 96% rename from Online/GaudiOnline/src/EventAccess.cpp rename to Online/EventData/src/EventAccess.cpp index 6b7c2b547..c1c1eea2e 100644 --- a/Online/GaudiOnline/src/EventAccess.cpp +++ b/Online/EventData/src/EventAccess.cpp @@ -15,7 +15,7 @@ //========================================================================== /// Framework include files -#include <GaudiOnline/EventAccess.h> +#include <EventData/EventAccess.h> #include <Tell1Data/EventHeader.h> #include <Tell1Data/Tell1Decoder.h> #include <PCIE40Data/pcie40decoder.h> @@ -235,21 +235,23 @@ EventAccess::convertPCIE40MEP(datapointer_t start, size_t /* len */) const { namespace { using namespace pcie40; - - void _error(const char* fmt,...) { - va_list args; - va_start(args,fmt); - ::vprintf(fmt,args); - va_end(args); - } } /// The event is a PCIE40 MEP structure with multiple events, which must be decoded std::unique_ptr<pcie40::event_collection_t> EventAccess::convertPCIE40MEP(datapointer_t start, size_t len) const { + struct _printer { + RTL::Logger& logger; + _printer(RTL::Logger& e) : logger(e) {} + /// Printout operator + void operator()(int lvl, const char* source, const char* msg) { + logger.printmsg(lvl, source, msg); + } + } printer(*m_logger); + pcie40::decoder_t::logger_t logger = printer; + pcie40::decoder_t decoder(logger); std::unique_ptr<pcie40::event_collection_t> events; datapointer_t end = start + len; - pcie40::decoder_t decoder; if ( end > start ) { static constexpr size_t max_bank = 20000; @@ -289,13 +291,13 @@ EventAccess::convertPCIE40MEP(datapointer_t start, size_t len) const { const pcie40::bank_collection_t* coll=ev->bank_collection(j); for( const pcie40::bank_t* b=coll->begin(); b != coll->end(); b=coll->next(b) ) { if ( b->magic() != pcie40::bank_t::MagicPattern ) { - _error("Bad magic pattern in raw bank!\n"); + m_logger->error("Bad magic pattern in raw bank!\n"); } ++num_bank; } for( const pcie40::bank_t* b=coll->special_begin(); b != coll->special_end(); b=coll->next(b) ) { if ( b->magic() != pcie40::bank_t::MagicPattern ) { - _error("Bad magic pattern in raw bank!\n"); + m_logger->error("Bad magic pattern in raw bank!\n"); } ++num_bank; } diff --git a/Online/GaudiOnline/src/EventOutput.cpp b/Online/EventData/src/EventOutput.cpp similarity index 99% rename from Online/GaudiOnline/src/EventOutput.cpp rename to Online/EventData/src/EventOutput.cpp index 686be7b52..62da98a3e 100644 --- a/Online/GaudiOnline/src/EventOutput.cpp +++ b/Online/EventData/src/EventOutput.cpp @@ -15,7 +15,7 @@ //========================================================================== /// Framework include files -#include "GaudiOnline/EventOutput.h" +#include "EventData/EventOutput.h" #include "Tell1Data/EventHeader.h" #include "Tell1Data/Tell1Decoder.h" #include "RTL/rtl.h" diff --git a/Online/GaudiOnline/src/FileEventAccess.cpp b/Online/EventData/src/FileEventAccess.cpp similarity index 99% rename from Online/GaudiOnline/src/FileEventAccess.cpp rename to Online/EventData/src/FileEventAccess.cpp index dde72835e..dcb79f247 100644 --- a/Online/GaudiOnline/src/FileEventAccess.cpp +++ b/Online/EventData/src/FileEventAccess.cpp @@ -21,7 +21,7 @@ //========================================================================== /// Framework include files -#include <GaudiOnline/FileEventAccess.h> +#include <EventData/FileEventAccess.h> #include <MBM/bmdef.h> /// C/C++ include files diff --git a/Online/GaudiOnline/src/FileEventOutput.cpp b/Online/EventData/src/FileEventOutput.cpp similarity index 99% rename from Online/GaudiOnline/src/FileEventOutput.cpp rename to Online/EventData/src/FileEventOutput.cpp index 0b84c2dfb..4c9ea338c 100644 --- a/Online/GaudiOnline/src/FileEventOutput.cpp +++ b/Online/EventData/src/FileEventOutput.cpp @@ -15,7 +15,7 @@ //========================================================================== /// Framework include files -#include <GaudiOnline/FileEventOutput.h> +#include <EventData/FileEventOutput.h> #include <Tell1Data/RawFile.h> #include "RTL/strdef.h" diff --git a/Online/GaudiOnline/src/MBMEventAccess.cpp b/Online/EventData/src/MBMEventAccess.cpp similarity index 99% rename from Online/GaudiOnline/src/MBMEventAccess.cpp rename to Online/EventData/src/MBMEventAccess.cpp index 9a1eb323d..3687d6a8b 100644 --- a/Online/GaudiOnline/src/MBMEventAccess.cpp +++ b/Online/EventData/src/MBMEventAccess.cpp @@ -14,7 +14,7 @@ //========================================================================== /// Framework include files -#include "GaudiOnline/MBMEventAccess.h" +#include "EventData/MBMEventAccess.h" /// C/C++ include files #include <iomanip> diff --git a/Online/GaudiOnline/src/MBMEventOutput.cpp b/Online/EventData/src/MBMEventOutput.cpp similarity index 99% rename from Online/GaudiOnline/src/MBMEventOutput.cpp rename to Online/EventData/src/MBMEventOutput.cpp index e50dd2575..4dfeb85e9 100644 --- a/Online/GaudiOnline/src/MBMEventOutput.cpp +++ b/Online/EventData/src/MBMEventOutput.cpp @@ -15,7 +15,7 @@ //========================================================================== /// Framework include files -#include <GaudiOnline/MBMEventOutput.h> +#include <EventData/MBMEventOutput.h> #include <Tell1Data/EventHeader.h> #include <Tell1Data/Tell1Decoder.h> diff --git a/Online/GaudiOnline/src/NetEventAccess.cpp b/Online/EventData/src/NetEventAccess.cpp similarity index 99% rename from Online/GaudiOnline/src/NetEventAccess.cpp rename to Online/EventData/src/NetEventAccess.cpp index ee8af60c4..7e2488a9f 100644 --- a/Online/GaudiOnline/src/NetEventAccess.cpp +++ b/Online/EventData/src/NetEventAccess.cpp @@ -21,7 +21,7 @@ //========================================================================== /// Framework include files -#include "GaudiOnline/NetEventAccess.h" +#include "EventData/NetEventAccess.h" #include "CPP/TimeSensor.h" #include "CPP/Event.h" #include "MBM/bmdef.h" diff --git a/Online/FarmConfig/job/EBPass.sh b/Online/FarmConfig/job/EBPass.sh index c0b2fc734..f220525a2 100755 --- a/Online/FarmConfig/job/EBPass.sh +++ b/Online/FarmConfig/job/EBPass.sh @@ -36,11 +36,16 @@ application.setup_hive(FlowManager("EventLoop"), 30) application.setup_algorithms(writer=writer, acceptRate=0.01) application.setup_monitoring() # +writer.UseRawData = False +application.input.DeclareData = True +application.input.DeclareEvent = False +application.input.DeclareErrors = False +# # application.monSvc.DimUpdateInterval = 1 application.config.numEventThreads = 1 application.config.MBM_numConnections = 7 -application.config.MBM_numEventThreads = 8 +application.config.MBM_numEventThreads = 6 application.config.MBM_requests = [ 'EvType=3;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0', 'EvType=2;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0', @@ -50,5 +55,5 @@ application.config.MBM_requests = [ # # export PYTHONPATH=/group/online/dataflow/options/${PARTITION}/HLT:${PYTHONPATH}; -export RTL_SLEEP_ON_FATAL=1; +###export RTL_SLEEP_ON_FATAL=1; exec -a ${UTGID} python `which gaudirun.py` /tmp/${UTGID}.py --application=OnlineEvents diff --git a/Online/GaudiOnline/CMakeLists.txt b/Online/GaudiOnline/CMakeLists.txt index 661a910a6..3df57fd2e 100755 --- a/Online/GaudiOnline/CMakeLists.txt +++ b/Online/GaudiOnline/CMakeLists.txt @@ -20,6 +20,7 @@ gaudi_depends_on_subdirs(GaudiKernel Online/Parsers Online/Tell1Data Online/PCIE40Data + Online/EventData Online/OnlineBase) # add_definitions(-DDD4HEP_USE_TINYXML) @@ -36,14 +37,14 @@ include_directories(SYSTEM ${Boost_INCLUDE_DIRS} ${ROOT_INCLUDE_DIRS} ${TBB_INCL # gaudi_add_library(GaudiOnline src/*.cpp PUBLIC_HEADERS GaudiOnline - INCLUDE_DIRS dim OnlineBase Tell1Data PCIE40Data - LINK_LIBRARIES dim dimlogLib Boost OnlineBase Tell1Data PCIE40Data GaudiKernel) + INCLUDE_DIRS dim OnlineBase Tell1Data PCIE40Data EventData + LINK_LIBRARIES dim dimlogLib Boost OnlineBase Tell1Data PCIE40Data EventData GaudiKernel) target_compile_options(GaudiOnline PRIVATE -Wno-address-of-packed-member) target_compile_definitions(GaudiOnline PRIVATE -DMessageSvc=OnlineMessageSvc) # gaudi_add_module(GaudiOnlineComp components/*.cpp src/*.cpp - INCLUDE_DIRS OnlineBase ROOT ${NUMA_INCLUDE_DIR} Tell1Data PCIE40Data GaudiKernel cppgsl - LINK_LIBRARIES Tell1Data PCIE40Data GaudiKernel GaudiAlgLib DAQEventLib dim dimlogLib OnlineBase GaudiOnline RPC Parsers ${NUMA_LIBRARIES} -lrt) + INCLUDE_DIRS OnlineBase ROOT ${NUMA_INCLUDE_DIR} Tell1Data PCIE40Data EventData GaudiKernel cppgsl + LINK_LIBRARIES Tell1Data PCIE40Data EventData GaudiKernel GaudiAlgLib DAQEventLib dim dimlogLib OnlineBase GaudiOnline RPC Parsers ${NUMA_LIBRARIES} -lrt) target_compile_options(GaudiOnlineComp PRIVATE -Wno-address-of-packed-member) # target_compile_definitions(GaudiOnlineComp PRIVATE -DMessageSvc=OnlineMessageSvc) diff --git a/Online/GaudiOnline/GaudiOnline/Configuration.h b/Online/GaudiOnline/GaudiOnline/Configuration.h index c9f3888dc..063376d23 100644 --- a/Online/GaudiOnline/GaudiOnline/Configuration.h +++ b/Online/GaudiOnline/GaudiOnline/Configuration.h @@ -16,10 +16,10 @@ #define GAUDIONLINE_CONFIGURATION_H /// Framework include files -#include "GaudiOnline/MBMEventAccess.h" -#include "GaudiOnline/NetEventAccess.h" -#include "GaudiOnline/FileEventAccess.h" -#include "GaudiKernel/Service.h" +#include <EventData/MBMEventAccess.h> +#include <EventData/NetEventAccess.h> +#include <EventData/FileEventAccess.h> +#include <GaudiKernel/Service.h> /// C/C++ include files diff --git a/Online/GaudiOnline/GaudiOnline/ITaskFSM.h b/Online/GaudiOnline/GaudiOnline/ITaskFSM.h index 86b308e4e..4f18b2826 100755 --- a/Online/GaudiOnline/GaudiOnline/ITaskFSM.h +++ b/Online/GaudiOnline/GaudiOnline/ITaskFSM.h @@ -15,6 +15,7 @@ #ifndef ONLINE_GAUDIONLINE_ITASKFSM_H #define ONLINE_GAUDIONLINE_ITASKFSM_H +/// C/C++ include files #include <string> // Forward declarations diff --git a/Online/GaudiOnline/GaudiOnline/OnlineApplication.h b/Online/GaudiOnline/GaudiOnline/OnlineApplication.h index 325f5ce0e..e0fcc1093 100644 --- a/Online/GaudiOnline/GaudiOnline/OnlineApplication.h +++ b/Online/GaudiOnline/GaudiOnline/OnlineApplication.h @@ -17,7 +17,6 @@ /// Framework include files #include <GaudiOnline/ITaskFSM.h> -#include <GaudiOnline/Macros.h> #include "Configuration.h" #include <Gaudi/Application.h> diff --git a/Online/GaudiOnline/GaudiOnline/QueuedFlowManager.h b/Online/GaudiOnline/GaudiOnline/QueuedFlowManager.h index b2c2cb7cc..a828ec835 100644 --- a/Online/GaudiOnline/GaudiOnline/QueuedFlowManager.h +++ b/Online/GaudiOnline/GaudiOnline/QueuedFlowManager.h @@ -16,11 +16,10 @@ #define ONLINE_GAUDIONLINE_QUEUEDFLOWMANAGER_H /// Framework include files -#include "GaudiOnline/Macros.h" -#include "GaudiKernel/SmartIF.h" -#include "GaudiKernel/EventContext.h" -#include "GaudiKernel/IHiveWhiteBoard.h" -#include "GaudiKernel/ThreadLocalContext.h" +#include <GaudiKernel/SmartIF.h> +#include <GaudiKernel/EventContext.h> +#include <GaudiKernel/IHiveWhiteBoard.h> +#include <GaudiKernel/ThreadLocalContext.h> #include <Gaudi/Interfaces/IQueueingEventProcessor.h> #include <mutex> diff --git a/Online/GaudiOnline/components/Components.cpp b/Online/GaudiOnline/components/Components.cpp index 76c7a8f48..469f395be 100644 --- a/Online/GaudiOnline/components/Components.cpp +++ b/Online/GaudiOnline/components/Components.cpp @@ -17,12 +17,12 @@ using namespace Online; #include <Gaudi/PluginService.h> -#include <GaudiOnline/MBMEventAccess.h> +#include <EventData/MBMEventAccess.h> DECLARE_FACTORY(MBMEventAccess, Gaudi::PluginService::Factory<EventAccess*(std::unique_ptr<RTL::Logger>&&)>) -#include <GaudiOnline/NetEventAccess.h> +#include <EventData/NetEventAccess.h> DECLARE_FACTORY(NetEventAccess, Gaudi::PluginService::Factory<EventAccess*(std::unique_ptr<RTL::Logger>&&)>) -#include <GaudiOnline/FileEventAccess.h> +#include <EventData/FileEventAccess.h> DECLARE_FACTORY(FileEventAccess, Gaudi::PluginService::Factory<EventAccess*(std::unique_ptr<RTL::Logger>&&)>) diff --git a/Online/GaudiOnline/components/EventProcessor.cpp b/Online/GaudiOnline/components/EventProcessor.cpp index 0f8a0f90e..352b642be 100644 --- a/Online/GaudiOnline/components/EventProcessor.cpp +++ b/Online/GaudiOnline/components/EventProcessor.cpp @@ -15,15 +15,17 @@ /// Framework include files #include "EventProcessor.h" -#include <GaudiOnline/EventTraits.h> -#include <GaudiOnline/EventOutput.h> -#include <GaudiKernel/MsgStream.h> +#include <EventData/EventTraits.h> +#include <EventData/EventOutput.h> #include <Tell1Data/EventHeader.h> #include <Tell1Data/Tell1Decoder.h> #include <Tell1Data/RunInfo.h> #include <PCIE40Data/pcie40decoder.h> +#include <PCIE40Data/RawBank40.h> #include <PCIE40Data/sodin.h> +#include <GaudiKernel/MsgStream.h> + // C/C++ include files #include <exception> #include <stdexcept> @@ -31,31 +33,6 @@ using namespace std; using namespace Online; -namespace { - struct BankHeader { - /// Magic word (by definition 0xCBCB) - unsigned short magic; - /// Bank length in bytes (must be >= 0) - unsigned short length; - /// Bank type (must be >= 0) - unsigned char type; - /// Version identifier (must be >= 0) - unsigned char version; - /// Source ID (valid source IDs are > 0; invalid ones 0) - short sourceID; - BankHeader() = delete; - BankHeader(BankHeader&&) = delete; - BankHeader(const BankHeader&) = delete; - template<typename T> BankHeader(unsigned short m, const T* c) - : magic(m), type(c->type()), version(c->version()) - { - length = c->size() + sizeof(BankHeader); - } - BankHeader& operator=(BankHeader&&) = delete; - BankHeader& operator=(const BankHeader&) = delete; - }; -} - /// Initialize the algorithm StatusCode EventProcessor::initialize() { m_logger = make_unique<RTL::Logger>(RTL::Logger::getGlobalDevice(),name(),msgLevel()); @@ -87,153 +64,131 @@ StatusCode EventProcessor::execute(EventContext const& ctxt) const { return StatusCode::FAILURE; } +bool EventProcessor::type_found(int required_type, const std::vector<int>& good_types) const { + if ( good_types.empty() ) + return true; + if ( std::find( good_types.begin(), good_types.end(), required_type) == good_types.end() ) + return false; + return true; +} + + namespace Online { - /// Create raw event object from banks template <typename traits> - std::unique_ptr<LHCb::RawEvent> EventProcessor::create_event(const event_traits::record_t&) const { - return std::unique_ptr<LHCb::RawEvent>(); + const typename traits::odin_type* + EventProcessor::get_odin(event_traits::record_t event) const { + const auto* bank = get_odin_bank<traits>(event); + return bank ? traits::get_odin(bank) : nullptr; } + + template <typename traits> + const typename traits::bank_type* + EventProcessor::get_odin_bank(event_traits::record_t record) const { + auto& e = traits::get_event(record); + for(size_t i=0; i<traits::num_collections(e); ++i) { + typename traits::bank_iteration_t iter(e, i); + for(const auto* bnk = iter.begin(); bnk != iter.end(); bnk = iter.next(bnk) ) { + if ( bnk->type() == traits::bank_type::ODIN ) + return bnk; + } + } + return nullptr; + } + /// Create raw event object from banks template <> - std::unique_ptr<LHCb::RawEvent> EventProcessor::create_event<event_traits::tell1>(const event_traits::record_t& record) const { - auto event = make_unique<LHCb::RawEvent>(); - auto* header = record.tell1_event; - auto* start = record.data + header->sizeOf(header->headerVersion()); - auto* end = record.data + header->size(); + EventProcessor::evt_data_t + EventProcessor::create_event<event_traits::tell1>(event_traits::record_t record) const { + const event_traits::tell1::event_type& event = *record.tell1_event; + auto* start = record.data + event.sizeOf(event.headerVersion()); + auto* end = record.data + event.size(); size_t num_banks = 0; + evt_data_t evt_data; + + evt_data.reserve(512); while ( start < end ) { - LHCb::RawBank* b = (LHCb::RawBank*)start; - int len = b->totalSize(); - event->adoptBank(b, false); - start += len; + Tell1Bank* b = (Tell1Bank*)start; + evt_data.emplace_back(b, b->data()); + start += b->totalSize(); if ( ++num_banks < 2000 ) continue; m_logger->error("Exceeded maximum number of banks [Banks were dropped]"); break; } - return event; + return evt_data; } template <> - std::unique_ptr<LHCb::RawEvent> EventProcessor::create_event<event_traits::tell40>(const event_traits::record_t& record) const { - auto result = make_unique<LHCb::RawEvent>(); - auto* event = record.tell40_event; - for(size_t i=0; i < event->num_bank_collections(); ++i) { - const auto* bc = event->bank_collection(i); - for(const auto* b=bc->begin(); b != bc->end(); b = bc->next(b)) { - if ( b->type() < b->LastType ) { - result->adoptBank((LHCb::RawBank*)b, false); - } - } - for(const auto* b=bc->special_begin(); b != bc->special_end(); b = bc->next(b)) { - if ( b->type() < b->LastType ) { - result->adoptBank((LHCb::RawBank*)b, false); - } - } + EventProcessor::evt_data_t + EventProcessor::create_event<event_traits::tell40>(event_traits::record_t record) const { + const event_traits::tell40::event_type& event = *record.tell40_event; + evt_data_t evt_data; + evt_data.reserve(512); + for(size_t i=0; i < event.num_bank_collections(); ++i) { + const auto* bc = event.bank_collection(i); + for(const auto* b=bc->begin(); b != bc->end(); b = bc->next(b)) + evt_data.emplace_back((BankHeader*)b, b->data()); + for(const auto* b=bc->special_begin(); b != bc->special_end(); b = bc->next(b)) + evt_data.emplace_back((BankHeader*)b, b->data()); } - return result; + return evt_data; } template <typename traits> void EventProcessor::put_event(string& loc, const evt_desc_t& event, size_t eid) const { auto evt = create_event<traits>(event.second->at(eid)); + auto obj = make_unique<AnyDataWrapper<evt_data_t> >(move(evt)); loc += (eid<event.first ? "prev" : "next"); - if ( !eventSvc()->registerObject(loc, evt.release()).isSuccess() ) { - m_logger->except("EventProcessor","+++ Failed to register TAE leaf %s",loc.c_str()); + if ( eventSvc()->registerObject(loc, obj.get()).isSuccess() ) { + obj.release(); + return; } + m_logger->except("EventProcessor","+++ Failed to register TAE leaf %s",loc.c_str()); } - template <typename traits> - void EventProcessor::expand_tae(const evt_desc_t& event, bool force, double probability, size_t forced_half_window) const { - using namespace event_traits; - size_t eid = event.first; - auto record = event.second->at(eid); - auto* odin = get_odin_bank<traits>(record); - if ( odin ) { - const auto* info = (typename traits::odin_type*)odin->data(); - bool central = info->tae_central(); - size_t half_window = info->tae_window(); - if ( force ) { - double frac = double(::rand()) / double(RAND_MAX); - if ( frac <= probability ) { - central = true; - half_window = forced_half_window; - } - } - if ( central ) { - if ( eid > half_window && event.second->length-eid > half_window ) { - string prev, next; - traits::set_tae(record, half_window); - for ( size_t i=1; i<=half_window; ++i ) { - put_event<traits>(prev, event, eid-i); - put_event<traits>(next, event, eid+i); - } - } - } - } - } - - template <typename traits> - const typename traits::bank_type* - EventProcessor::get_odin_bank(const event_traits::record_t& record) const - { - auto& e = traits::get_event(record); - for(size_t i=0; i<traits::num_collections(e); ++i) { - typename traits::bank_iteration_t iter(e, i); - for(const auto* bnk = iter.begin(); bnk != iter.end(); bnk = iter.next(bnk) ) { - if ( bnk->type() == traits::bank_type::ODIN ) return bnk; - } + namespace { + template <typename bank_type> size_t + output_bank(EventOutput::transaction_t* tr, bank_type* b, const void* data) { + Tell1Bank* bnk = tr->get<Tell1Bank>(); + ::memcpy((void*)bnk, b, bank_type::hdrSize()); + ::memcpy(bnk->data(), data, b->totalSize()-bank_type::hdrSize()); + bnk->setMagic(); + tr->advance(b->totalSize()); + return b->totalSize(); } - return nullptr; } - - template <typename traits> - void EventProcessor::force_tae(const event_traits::record_t& record, - double probability, - size_t half_window) const - { - double frac = double(::rand()) / double(RAND_MAX); - if ( probability >= frac ) { - traits::set_tae(record, half_window); - } - } - + template <typename traits> - void EventProcessor::output_mdf(const LHCb::RawEvent& evt, - const std::vector<int>& routing_mask, - EventOutput& output) const + size_t EventProcessor::output_mdf(const evt_data_t& evt, + const mask_t& routing_mask, + EventOutput& output) const { typedef typename traits::bank_type bank_type; - const bank_type* daq_bank = nullptr; - const bank_type* hlt_bank = nullptr; - const size_t hdr_length = EventHeader::sizeOf(3); - size_t data_length = hdr_length; + const EventHeader* evthdr = nullptr; + const uint32_t* hlt_mask = nullptr; + const size_t hdr_len = EventHeader::sizeOf(3); + size_t data_len = hdr_len; - for(int i=0; i < bank_type::LastType; ++i) { - auto banks = evt.banks(LHCb::RawBank::BankType(i)); - for(const auto* bnk : banks ) { - const bank_type* b = (const bank_type*)bnk; - if ( i == bank_type::DAQ && b->version() == DAQ_STATUS_BANK ) { - daq_bank = reinterpret_cast<const bank_type*>(b); - continue; - } - else if ( i == bank_type::HltRoutingBits && !banks.empty() ) { - hlt_bank = reinterpret_cast<const bank_type*>(*(banks.begin())); - } - data_length += b->totalSize(); + for( const auto& dsc : evt ) { + const auto* bank = dsc.first; + if ( bank->type() == bank_type::DAQ && bank->version() == DAQ_STATUS_BANK ) { + evthdr = reinterpret_cast<const EventHeader*>(dsc.second); + continue; } + else if ( bank->type() == bank_type::HltRoutingBits ) { + hlt_mask = reinterpret_cast<const uint32_t*>(dsc.second); + } + data_len += bank->totalSize(); } unsigned int m[] = { ~0x0U, ~0x0U, ~0x0U, ~0x0U }; - if ( hlt_bank ) { - const unsigned int* hlt_mask = (const unsigned int*)hlt_bank->data(); + if ( hlt_mask ) { m[0] = hlt_mask[0]; m[1] = hlt_mask[1]; m[2] = hlt_mask[2]; } - else if ( daq_bank ) { - const EventHeader* hdr = (const EventHeader*)daq_bank->data(); - auto daq_mask = hdr->subHeader().H1->triggerMask(); + else if ( evthdr ) { + auto daq_mask = evthdr->subHeader().H1->triggerMask(); m[0] = daq_mask[0]; m[1] = daq_mask[1]; m[2] = daq_mask[2]; @@ -260,7 +215,7 @@ namespace Online { else if ( routing_mask.size() > 0 ) { m_logger->except("Unknown trigger masdk configuration. Cannot process event"); } - auto* tr = output.getTransaction(data_length); + auto* tr = output.getTransaction(data_len); if ( !tr ) { m_logger->except("Event output failed [Cannot reserve output space]"); } @@ -269,14 +224,13 @@ namespace Online { ev_hdr->setChecksum(0); ev_hdr->setCompression(0); ev_hdr->setHeaderVersion(3); - ev_hdr->setSize(data_length-hdr_length); + ev_hdr->setSize(data_len-hdr_len); ev_hdr->setDataType(EventHeader::BODY_TYPE_BANKS); ev_hdr->setSubheaderLength(sizeof(EventHeader::Header1)); auto* h = ev_hdr->subHeader().H1; h->setTriggerMask(m); - if ( daq_bank ) { - const EventHeader* hdr = (const EventHeader*)daq_bank->data(); - const auto* dh = hdr->subHeader().H1; + if ( evthdr ) { + const auto* dh = evthdr->subHeader().H1; h->setRunNumber(dh->runNumber()); h->setOrbitNumber(dh->orbitNumber()); h->setBunchID(dh->bunchID()); @@ -286,56 +240,157 @@ namespace Online { h->setOrbitNumber(0); h->setBunchID(0); } - tr->advance(hdr_length); - for(int i=0; i<bank_type::LastType; ++i) { - auto banks = evt.banks(LHCb::RawBank::BankType(i)); - int cnt = 0; - for(const auto* bnk : banks ) { - const bank_type* b = (const bank_type*)bnk; - if ( (void*)b != (void*)daq_bank ) { - BankHeader hdr(Tell1Bank::MagicPattern, b); - size_t hdr_len = sizeof(BankHeader); - if ( b->magic() != bank_type::MagicPattern ) { - m_logger->info("Bad bank[%d]: %d %ld %ld transaction length:%ld", - ++cnt,int(b->type()), hdr_len, b->totalSize(), tr->length); - } - tr->add_data(EventOutput::datapointer_t(&hdr), hdr_len); - tr->add_data(EventOutput::datapointer_t(b->data()), b->totalSize()-hdr_len); + tr->advance(hdr_len); + for(const auto& dsc : evt) { + const auto* bank = dsc.first; + if ( (void*)dsc.second != (void*)evthdr ) { + if ( bank->magic() != bank_type::MagicPattern ) { + m_logger->info("Bad bank: %d %ld transaction length:%ld", + int(bank->type()), bank->totalSize(), tr->length); } + output_bank(tr, bank, dsc.second); } } if ( !output.end(tr) ) { m_logger->except("Failed to commit event output!"); } + return data_len; + } + + /// Output PCIE40 or TELL1 event in MDF format + template <typename traits> size_t EventProcessor::output_mdf + (const typename traits::event_type& evt, const mask_t& routing_mask, EventOutput& output) const + { + event_traits::record_t record(&evt); + evt_data_t banks(create_event<traits>(record)); + return output_mdf<event_traits::tell1>(banks, routing_mask, output); + } + + /// Output PCIE40 event in MDF format + size_t EventProcessor::output_mdf(EventAccess::record_t record, + const mask_t& routing_mask, + EventOutput& output) const + { + if ( record.is_tell40() ) + return output_mdf<event_traits::tell40>(*record.tell40_event, routing_mask, output); + else if ( record.is_tell1() ) + return output_mdf<event_traits::tell1>(*record.tell1_event, routing_mask, output); + return 0; + } + + /// Output PCIE40 event in MDF format + void EventProcessor::output_pcie40(const EventAccess::guard_t* guard, EventOutput& output) const { + if ( guard && !guard->empty() ) { + EventAccess::record_t rec = guard->at(0); + const pcie40::event_collection_t* events = + pcie40::add_ptr<pcie40::event_collection_t>(rec.data, + - sizeof(pcie40::event_collection_t::capacity) + - sizeof(pcie40::event_collection_t::length) + - sizeof(pcie40::event_collection_t::mep) + - sizeof(pcie40::event_collection_t::magic)); + if ( nullptr != events->mep ) { + auto* tr = output.getTransaction(events->mep->size); + if ( !tr ) { + m_logger->except("+++ Event output failed [Cannot reserve output space]"); + } + tr->add_data(EventAccess::datapointer_t(events->mep), events->mep->size); + if ( !output.end(tr) ) { + m_logger->except("+++ Failed to commit event output!"); + } + } + } + } + + /// Add error banks to the data service + EventProcessor::evt_data_t EventProcessor::get_errors(event_traits::record_t record) const { + evt_data_t banks; + if ( record.is_tell40() ) { + auto& evt = event_traits::tell40::get_event(record); + for(size_t i=0; i < evt.num_bank_collections(); ++i) { + const auto* bc = evt.bank_collection(i); + for(const auto* b=bc->begin(); b != bc->end(); b = bc->next(b)) { + if ( b->type() >= RawBank40::DaqErrorBase ) + banks.emplace_back((event_traits::tell1::bank_type*)b, b->data()); + } + for(const auto* b=bc->special_begin(); b != bc->special_end(); b = bc->next(b)) { + if ( b->type() >= RawBank40::DaqErrorBase ) + banks.emplace_back((event_traits::tell1::bank_type*)b, b->data()); + } + } + } + return banks; + } + + template <typename traits> + void EventProcessor::expand_tae(const evt_desc_t& event, bool force, double probability, size_t forced_half_window) const { + using namespace event_traits; + size_t eid = event.first; + auto record = event.second->at(eid); + const auto* info = get_odin<traits>(record); + if ( info ) { + bool central = info->tae_central(); + size_t half_window = info->tae_window(); + if ( force ) { + double frac = double(::rand()) / double(RAND_MAX); + if ( frac <= probability ) { + central = true; + half_window = forced_half_window; + } + } + if ( central ) { + if ( eid > half_window && event.second->length-eid > half_window ) { + string prev, next; + traits::set_tae(record, half_window); + for ( size_t i=1; i<=half_window; ++i ) { + put_event<traits>(prev, event, eid-i); + put_event<traits>(next, event, eid+i); + } + } + } + } } + template <typename traits> - void EventProcessor::output_tae(const evt_desc_t& event, const std::vector<int>& routing_mask, EventOutput& output) const { + void EventProcessor::force_tae(event_traits::record_t record, + double probability, + size_t half_window) const + { + double frac = double(::rand()) / double(RAND_MAX); + if ( probability >= frac ) { + traits::set_tae(record, half_window); + } + } + + template <typename traits> + void EventProcessor::output_tae(const evt_desc_t& event, mask_t& routing_mask, EventOutput& output) const { if ( event.first || routing_mask.empty() || output.isCancelled() ) {} } template <typename traits> - void EventProcessor::output_mep(const evt_desc_t& event, const std::vector<int>& routing_mask, EventOutput& output) const { + void EventProcessor::output_mep(const evt_desc_t& event, mask_t& routing_mask, EventOutput& output) const { if ( event.first || routing_mask.empty() || output.isCancelled() ) {} } - + #define INSTANTIATE(x) \ + template const typename x::odin_type* \ + EventProcessor::get_odin< x >(event_traits::record_t r) const; \ template const typename x::bank_type* \ - EventProcessor::get_odin_bank< x >(const event_traits::record_t& r) const; \ + EventProcessor::get_odin_bank< x >(event_traits::record_t r) const; \ template void \ - EventProcessor::force_tae< x >(const event_traits::record_t& event, double probability, size_t half_window) const; \ + EventProcessor::force_tae< x >(event_traits::record_t event, double probability, size_t half_window) const; \ template void \ EventProcessor::put_event< x >(string& loc, const evt_desc_t& event, size_t eid) const; \ template void \ EventProcessor::expand_tae< x >(const evt_desc_t& event, bool force, double probability, size_t forced_half_window) const; \ + template size_t \ + EventProcessor::output_mdf< x >(const evt_data_t& event, const mask_t& routing_mask, EventOutput& output) const; \ + template size_t \ + EventProcessor::output_mdf< x >(const x::event_type& evt, const mask_t& routing_mask, EventOutput& output) const; \ template void \ - EventProcessor::output_mdf< x >(const LHCb::RawEvent& event, \ - const std::vector<int>& routing_mask, \ - EventOutput& output) const; \ - template void \ - EventProcessor::output_tae< x >(const evt_desc_t& event, const std::vector<int>& routing_mask, EventOutput& output) const; \ + EventProcessor::output_tae< x >(const evt_desc_t& event, const mask_t& routing_mask, EventOutput& output) const; \ template void \ - EventProcessor::output_mep< x >(const evt_desc_t& event, const std::vector<int>& routing_mask, EventOutput& output) const + EventProcessor::output_mep< x >(const evt_desc_t& event, const mask_t& routing_mask, EventOutput& output) const INSTANTIATE(event_traits::tell1); diff --git a/Online/GaudiOnline/components/EventProcessor.h b/Online/GaudiOnline/components/EventProcessor.h index fdb822c67..209b7a947 100644 --- a/Online/GaudiOnline/components/EventProcessor.h +++ b/Online/GaudiOnline/components/EventProcessor.h @@ -16,10 +16,9 @@ #define GAUDIONLINE_EVENTPROCESSOR_H /// Framework include files -#include <GaudiOnline/EventAccess.h> +#include <EventData/EventAccess.h> #include <GaudiKernel/Algorithm.h> #include <GaudiKernel/DataObjectHandle.h> -#include <Event/RawEvent.h> #include <RTL/Logger.h> /// Online namespace declaration @@ -27,7 +26,8 @@ namespace Online { /// Forward declarations class EventOutput; - + typedef Tell1Bank BankHeader; + /// Online algorithm to feed the TES with TAE events from the basic raw event. /** * @@ -36,35 +36,57 @@ namespace Online { * @date 25/04/2019 */ class EventProcessor : public Gaudi::Algorithm { - protected: + public: + typedef std::vector<std::pair<const BankHeader*, const void*> > evt_data_t; typedef std::pair<size_t, EventAccess::shared_guard_t> evt_desc_t; - + typedef const std::vector<int> mask_t; + + protected: /// Reference to output logger std::unique_ptr<RTL::Logger> m_logger; + bool type_found(int required, const std::vector<int>& good_types) const; + template <typename traits> - const typename traits::bank_type* get_odin_bank(const event_traits::record_t& record) const; + const typename traits::odin_type* get_odin(event_traits::record_t record) const; template <typename traits> - void force_tae(const event_traits::record_t& event, double probability, size_t half_window) const; + const typename traits::bank_type* get_odin_bank(event_traits::record_t event) const; template <typename traits> - void put_event(std::string& loc, const evt_desc_t& event, size_t eid) const; + void put_event(std::string& loc, const evt_desc_t& event, size_t eid) const; template <typename traits> - std::unique_ptr<LHCb::RawEvent> create_event(const event_traits::record_t& event) const; + evt_data_t create_event(event_traits::record_t record) const; template <typename traits> - void expand_tae(const evt_desc_t& event, bool force, double probability, size_t forced_half_window) const; + size_t output_mdf(const evt_data_t& event, const mask_t& routing_mask, EventOutput& output) const; + /// Output PCIE40 or TELL1 event in MDF format template <typename traits> - void output_mdf(const LHCb::RawEvent& event, const std::vector<int>& routing_mask, EventOutput& output) const; + size_t output_mdf(const typename traits::event_type& event, const mask_t& routing_mask, EventOutput& output) const; + + /// Output PCIE40 or TELL1 event in MDF format + size_t output_mdf(event_traits::record_t record, const mask_t& routing_mask, EventOutput& output) const; + + /// Output PCIE40 event in PCIE40-MEP format + void output_pcie40(const EventAccess::guard_t* guard, EventOutput& output) const; template <typename traits> - void output_tae(const evt_desc_t& event, const std::vector<int>& routing_mask, EventOutput& output) const; + void force_tae(event_traits::record_t event, double probability, size_t half_window) const; template <typename traits> - void output_mep(const evt_desc_t& event, const std::vector<int>& routing_mask, EventOutput& output) const; + void expand_tae(const evt_desc_t& event, bool force, double probability, size_t forced_half_window) const; + + template <typename traits> + void output_tae(const evt_desc_t& event, const mask_t& routing_mask, EventOutput& output) const; + + template <typename traits> + void output_mep(const evt_desc_t& event, const mask_t& routing_mask, EventOutput& output) const; + + /// Add error banks to the data service + evt_data_t get_errors(event_traits::record_t event) const; + public: using Algorithm::Algorithm; diff --git a/Online/GaudiOnline/components/ExpandTAE.cpp b/Online/GaudiOnline/components/ExpandTAE.cpp index 1c7befd5a..681a032b8 100644 --- a/Online/GaudiOnline/components/ExpandTAE.cpp +++ b/Online/GaudiOnline/components/ExpandTAE.cpp @@ -43,7 +43,7 @@ namespace Online { Gaudi::Property<double> m_probability{this, "Probability", 0.0, ""}; /// Data guard location in TES - DataObjectReadHandle<evt_desc_t> m_rawGuard{this, "RawGuard", LHCb::RawEventLocation::Default+"Guard"}; + DataObjectReadHandle<evt_desc_t> m_rawGuard{this, "RawGuard", "DAQ/RawDataGuard"}; public: using EventProcessor::EventProcessor; diff --git a/Online/GaudiOnline/components/ForceTAE.cpp b/Online/GaudiOnline/components/ForceTAE.cpp index fa977ea23..a4a280fdd 100644 --- a/Online/GaudiOnline/components/ForceTAE.cpp +++ b/Online/GaudiOnline/components/ForceTAE.cpp @@ -38,7 +38,7 @@ namespace Online { /// Property: TAE half window size Gaudi::Property<double> m_probability{this, "Probability", 0.0, ""}; /// Data guard location in TES - DataObjectReadHandle<evt_desc_t> m_rawGuard{this, "RawGuard", LHCb::RawEventLocation::Default+"Guard"}; + DataObjectReadHandle<evt_desc_t> m_rawGuard{this, "RawGuard", "DAQ/RawDataGuard"}; public: using EventProcessor::EventProcessor; diff --git a/Online/GaudiOnline/components/IOService.h b/Online/GaudiOnline/components/IOService.h index 94a17e610..450fa5025 100644 --- a/Online/GaudiOnline/components/IOService.h +++ b/Online/GaudiOnline/components/IOService.h @@ -17,7 +17,7 @@ /// Framework include files #include <GaudiKernel/Service.h> -#include <GaudiOnline/EventAccess.h> +#include <EventData/EventAccess.h> /// C/C++ include files diff --git a/Online/GaudiOnline/components/InputAlg.cpp b/Online/GaudiOnline/components/InputAlg.cpp index 98a8a434f..69e9962e7 100644 --- a/Online/GaudiOnline/components/InputAlg.cpp +++ b/Online/GaudiOnline/components/InputAlg.cpp @@ -56,45 +56,31 @@ StatusCode Online::InputAlg::stop() { return StatusCode::SUCCESS; } -/// Add error banks to the data service -Online::InputAlg::daq_error_t -Online::InputAlg::get_errors(const event_traits::record_t& record) const { - daq_error_t banks; - auto* event = record.tell40_event; - for(size_t i=0; i < event->num_bank_collections(); ++i) { - const auto* bc = event->bank_collection(i); - for(const auto* b=bc->begin(); b != bc->end(); b = bc->next(b)) { - if ( b->type() >= RawBank40::DaqErrorBase ) { - banks.emplace_back(b); - } - } - for(const auto* b=bc->special_begin(); b != bc->special_end(); b = bc->next(b)) { - if ( b->type() < b->LastType ) { - banks.emplace_back(b); - } - } - } - return banks; -} - - /// Execute single event StatusCode Online::InputAlg::process(EventContext const& /* ctxt */) const { + using namespace event_traits; EventAccess::event_t e(m_io->pop()); if ( e.second ) { - std::unique_ptr<LHCb::RawEvent> event; - if ( e.second->type() == event_traits::tell1::data_type ) { - event = create_event<event_traits::tell1>(e.second->at(e.first)); - } - else if ( e.second->type() == event_traits::tell40::data_type ) { - const auto& record = e.second->at(e.first); - event = create_event<event_traits::tell40>(record); - if ( m_declareData.value() ) { - m_daqError.put(move(get_errors(record))); + const auto record(e.second->at(e.first)); + auto typ = e.second->type(); + if ( m_declareEvt.value() ) { + if ( typ == tell1::data_type ) { + evt_data_t event = create_event<event_traits::tell1>(record); + m_rawData.put(move(event)); } + else if ( typ == tell40::data_type ) { + evt_data_t event = create_event<event_traits::tell40>(record); + m_rawData.put(move(event)); + } + else { + m_logger->error("+++ Algorithm failed [Invalid event type]."); + return StatusCode::FAILURE; + } + } + if ( typ == tell40::data_type && m_declareErrs.value() ) { + m_daqError.put(get_errors(record)); } if ( m_declareData.value() ) { - m_rawEvent.put(move(event)); m_rawGuard.put(move(e)); } return StatusCode::SUCCESS; diff --git a/Online/GaudiOnline/components/InputAlg.h b/Online/GaudiOnline/components/InputAlg.h index b4a99e369..a108c664c 100644 --- a/Online/GaudiOnline/components/InputAlg.h +++ b/Online/GaudiOnline/components/InputAlg.h @@ -33,26 +33,27 @@ namespace Online { protected: typedef std::vector<const RawBank40*> daq_error_t; - - Gaudi::Property<bool> m_declareData{this,"DeclareData", true, "Declare or drop data"}; - Gaudi::Property<bool> m_enableHalt{this, "EnableHalt", true, "Enable halt the event loop"}; - Gaudi::Property<std::string> m_ioService{this, "IOServiceName","Online::IOService/IOService","Name of IO service"}; + typedef EventProcessor::evt_data_t evt_data_t; - DataObjectWriteHandle<LHCb::RawEvent> m_rawEvent{this, "RawLocation", LHCb::RawEventLocation::Default}; - DataObjectWriteHandle<evt_desc_t> m_rawGuard{this, "RawGuard", LHCb::RawEventLocation::Default+"Guard"}; - DataObjectWriteHandle<daq_error_t> m_daqError{this, "DAQErrors", LHCb::RawEventLocation::Default+"DAQErrors"}; + Gaudi::Property<bool> m_declareData{this, "DeclareData", true, "Declare or drop data"}; + Gaudi::Property<bool> m_declareErrs{this, "DeclareErrors",false, "Declare error data"}; + Gaudi::Property<bool> m_declareEvt {this, "DeclareEvent", false, "Declare raw data"}; + Gaudi::Property<bool> m_enableHalt {this, "EnableHalt", true, "Enable halt the event loop"}; + Gaudi::Property<std::string> m_ioService {this, "IOServiceName","Online::IOService/IOService","Name of IO service"}; + + DataObjectWriteHandle<evt_data_t> m_rawData {this, "RawData", "DAQ/RawData"}; + DataObjectWriteHandle<evt_data_t> m_daqError {this, "DAQErrors", "DAQ/RawDataErrors"}; + DataObjectWriteHandle<evt_desc_t> m_rawGuard {this, "RawGuard", "DAQ/RawDataGuard"}; /// Reference to the I/O service SmartIF<IOService> m_io; /// Halt the algorithm and allow for debugging void halt() const; + /// Allow to escape halt mutable int m_halt; - /// Add error banks to the data service - daq_error_t get_errors(const event_traits::record_t& record) const; - public: using EventProcessor::EventProcessor; /// Initialize the algorithm diff --git a/Online/GaudiOnline/components/OnlineEventApp.cpp b/Online/GaudiOnline/components/OnlineEventApp.cpp index 16cb0324e..b16ab41ca 100644 --- a/Online/GaudiOnline/components/OnlineEventApp.cpp +++ b/Online/GaudiOnline/components/OnlineEventApp.cpp @@ -15,9 +15,9 @@ #include "OnlineEventApp.h" #include "IOService.h" -#include <GaudiOnline/MBMEventAccess.h> -#include <GaudiOnline/NetEventAccess.h> -#include <GaudiOnline/FileEventAccess.h> +#include <EventData/MBMEventAccess.h> +#include <EventData/NetEventAccess.h> +#include <EventData/FileEventAccess.h> #include <Gaudi/PluginService.h> #include <Gaudi/Interfaces/IQueueingEventProcessor.h> diff --git a/Online/GaudiOnline/components/OnlineEventApp.h b/Online/GaudiOnline/components/OnlineEventApp.h index e1dbcfc0d..c8d786071 100644 --- a/Online/GaudiOnline/components/OnlineEventApp.h +++ b/Online/GaudiOnline/components/OnlineEventApp.h @@ -17,13 +17,12 @@ #define GAUDIONLINE_ONLINEEVENTAPP_H /// Framework include files -#include "GaudiOnline/OnlineApplication.h" -#include "IOService.h" -#include <GaudiOnline/EventAccess.h> +#include <GaudiOnline/OnlineApplication.h> #include <GaudiKernel/IHiveWhiteBoard.h> #include <GaudiKernel/EventContext.h> #include <GaudiKernel/IMonitorSvc.h> -#include <GaudiKernel/StatusCode.h> +#include <EventData/EventAccess.h> +#include "IOService.h" /// C/C++ include files #include <memory> diff --git a/Online/GaudiOnline/components/OutputAlg.cpp b/Online/GaudiOnline/components/OutputAlg.cpp index 120e10fc7..dd4163f9d 100644 --- a/Online/GaudiOnline/components/OutputAlg.cpp +++ b/Online/GaudiOnline/components/OutputAlg.cpp @@ -18,8 +18,8 @@ #include <GaudiKernel/IIncidentSvc.h> #include <GaudiKernel/DataObjectHandle.h> #include <GaudiKernel/IIncidentListener.h> -#include <GaudiOnline/FileEventOutput.h> -#include <GaudiOnline/MBMEventOutput.h> +#include <EventData/FileEventOutput.h> +#include <EventData/MBMEventOutput.h> /// Online namespace declaration namespace Online { @@ -46,10 +46,16 @@ namespace Online { /// Property: Forced output trigger mask; if empty will not be applied Gaudi::Property<std::vector<int> > m_routingMask{this, "RoutingMask", {}, ""}; + + /// Property: Use raw event + bool m_useRawData {true}; + /// Property: Output MDF format (instead of PCIE40 MEPs) + bool m_output_mdf {true}; + /// Raw event location in TES - DataObjectReadHandle<LHCb::RawEvent> m_rawEvent{this, "RawLocation", LHCb::RawEventLocation::Default}; + DataObjectReadHandle<evt_data_t> m_rawData{ this, "RawData", "DAQ/RawData"}; /// Data guard location in TES - DataObjectReadHandle<evt_desc_t> m_rawGuard{this, "RawGuard", LHCb::RawEventLocation::Default+"Guard"}; + DataObjectReadHandle<evt_desc_t> m_rawGuard{this, "RawGuard", "DAQ/RawDataGuard"}; /// Reference to output writer std::shared_ptr<EventOutput> m_output; @@ -59,6 +65,10 @@ namespace Online { MBMEventOutput::mbm_config_t mbm; /// FILE output parameters FileEventOutput::file_config_t file; + + void output_mdf_pcie40(const pcie40::event_t& evt, + const std::vector<int>& routing_mask, + EventOutput& output) const; public: /// Default constructor @@ -81,6 +91,8 @@ namespace Online { } /// Framework include files +#include <PCIE40Data/pcie40.h> +#include <PCIE40Data/RawBank40.h> // C/C++ include files @@ -94,6 +106,9 @@ DECLARE_COMPONENT( OutputAlg ) OutputAlg::OutputAlg(const string& nam, ISvcLocator* loc) : extends::extends(nam, loc) { + declareProperty("UseRawData", m_useRawData); + declareProperty("OutputMDF", m_output_mdf); + declareProperty("MBM_allocationSize", mbm.allocation_size); declareProperty("MBM_burstPrintCount", mbm.burstPrintCount); declareProperty("MBM_transactionWaitTMO", mbm.transitionWaitTMO); @@ -193,14 +208,36 @@ StatusCode OutputAlg::stop() { /// Execute single event StatusCode OutputAlg::process(EventContext const& /* ctxt */) const { const EventAccess::event_t* event = m_rawGuard.get(); - const LHCb::RawEvent* evt = m_rawEvent.get(); - if ( event->second->type() == event_traits::tell1::data_type ) - output_mdf<event_traits::tell1>(*evt, m_routingMask.value(), *m_output); - else if ( event->second->type() == event_traits::tell1::burst_type ) - output_mdf<event_traits::tell1>(*evt, m_routingMask.value(), *m_output); - else if ( event->second->type() == event_traits::tell40::data_type ) - output_mdf<event_traits::tell40>(*evt, m_routingMask.value(), *m_output); - else - m_logger->except("+++ Invalid burst data type encountered: %ld",event->second->type()); + if ( m_output_mdf ) { + if ( event->second->type() == event_traits::tell1::data_type ) { + const auto* evt = m_rawData.get(); + output_mdf<event_traits::tell1>(*evt, m_routingMask.value(), *m_output); + } + else if ( event->second->type() == event_traits::tell1::burst_type ) { + const auto* evt = m_rawData.get(); + output_mdf<event_traits::tell1>(*evt, m_routingMask.value(), *m_output); + } + else if ( m_useRawData && event->second->type() == event_traits::tell40::data_type ) { + const auto* evt = m_rawData.get(); + output_mdf<event_traits::tell40>(*evt, m_routingMask.value(), *m_output); + } + else if ( event->second->type() == event_traits::tell40::data_type ) { + EventAccess::record_t rec = event->second->at(event->first); + const pcie40::event_t* evt = rec.tell40_event; + output_mdf<event_traits::tell40>(*evt, m_routingMask.value(), *m_output); + } + else { + m_logger->except("+++ Invalid burst data type encountered: %ld",event->second->type()); + } + } + else { + if ( !m_useRawData && event->second->type() == event_traits::tell40::data_type ) { + output_pcie40(event->second.get(), *m_output); + } + else { + m_logger->except("+++ Invalid PCIE40 output conversion requested!"); + } + } return StatusCode::SUCCESS; } + diff --git a/Online/GaudiOnline/components/Passthrough.cpp b/Online/GaudiOnline/components/Passthrough.cpp index 5393fb9a2..5f3023dff 100644 --- a/Online/GaudiOnline/components/Passthrough.cpp +++ b/Online/GaudiOnline/components/Passthrough.cpp @@ -14,9 +14,7 @@ //========================================================================== /// Framework include files -#include <GaudiKernel/DataObjectHandle.h> -#include <GaudiKernel/Algorithm.h> -#include <Event/RawEvent.h> +#include "EventProcessor.h" /// Online namespace declaration namespace Online { @@ -28,11 +26,10 @@ namespace Online { * @version 1.0 * @date 25/04/2019 */ - class Passthrough: public Gaudi::Algorithm { + class Passthrough : public EventProcessor { private: /// Property: Odin event type to be downscaled - std::vector<int> m_downScale; - + std::vector<int> m_downScale; /// Property: Odin trigger types to be filtered std::vector< int > m_triggerTypesToPass; /// Property: Odin calibration types to be downscaled @@ -56,37 +53,42 @@ namespace Online { unsigned int m_seed; /// Input data handle - DataObjectReadHandle<LHCb::RawEvent> m_rawEvent{this, "RawLocation", LHCb::RawEventLocation::Default}; + DataObjectReadHandle<EventAccess::event_t> m_rawGuard{this, "RawGuard", "DAQ/RawDataGuard"}; protected: - StatusCode _halt(int micro_seconds) const; + void _halt(int micro_seconds) const; + public: /// Standard constructor Passthrough( const std::string& name, ISvcLocator* pSvcLocator ); /// Default Destructor virtual ~Passthrough() = default; /// Initialization - StatusCode initialize() override { - return _halt(m_initDelay*1000); + virtual StatusCode initialize() override { + _halt(m_initDelay*1000); + return this->EventProcessor::initialize(); } /// Algorithm overload: Start the algorithm - StatusCode start() override; + virtual StatusCode start() override; /// Stop - StatusCode stop() override { - return _halt(m_stopDelay*1000); + virtual StatusCode stop() override { + _halt(m_stopDelay*1000); + return this->EventProcessor::stop(); } /// Finalization - StatusCode finalize() override { - return _halt(m_finiDelay*1000); + virtual StatusCode finalize() override { + _halt(m_finiDelay*1000); + return this->EventProcessor::finalize(); } /// Algorithm execution - StatusCode execute(EventContext const& evtCtx) const override; + virtual StatusCode process(EventContext const& evtCtx) const override; }; } /// Framework include files -#include "GaudiKernel/SmartDataPtr.h" #include "GaudiKernel/MsgStream.h" +#include "PCIE40Data/pcie40.h" +#include "PCIE40Data/sodin.h" #include "Tell1Data/MEP.h" #include "RTL/rtl.h" @@ -97,11 +99,11 @@ using namespace Online; DECLARE_COMPONENT( Passthrough ) -Passthrough::Passthrough( const std::string& name, ISvcLocator* pSvcLocator) -: Algorithm(name , pSvcLocator), m_triggerTypesToPass(), m_calibTypesToPass() +/// Initializing constructor +Passthrough::Passthrough( const std::string& name, ISvcLocator* pSvc) : EventProcessor(name , pSvc) { declareProperty("TriggerTypesToPass", m_triggerTypesToPass, - "Trigger types to pass if not calibration trigger inhibit"); + "Trigger types to pass if not calibration trigger inhibit. (Run 1/2 Only)"); declareProperty("CalibTypesToPass" , m_calibTypesToPass, "Calibration trigger inhibits"); @@ -121,85 +123,67 @@ Passthrough::Passthrough( const std::string& name, ISvcLocator* pSvcLocator) declareProperty("MicroDelayTime", m_muDelay=0); } -StatusCode Passthrough::_halt(int micro_seconds) const { +/// Pause function +void Passthrough::_halt(int micro_seconds) const { if ( micro_seconds ) ::lib_rtl_usleep(micro_seconds); - return StatusCode::SUCCESS; } /// Algorithm overload: Start the algorithm StatusCode Passthrough::start() { m_seed = InterfaceID::hash32((RTL::processName()+"@"+RTL::nodeName()).c_str()); ::srand(m_seed); - return _halt(m_startDelay*1000); + _halt(m_startDelay*1000); + return this->EventProcessor::start(); } -StatusCode Passthrough::execute(EventContext const& ctxt) const { +/// Event callback +StatusCode Passthrough::process(EventContext const& ctxt) const { double delay = m_delay*1000+m_muDelay; - auto raw = m_rawEvent.get(); + const auto* raw = m_rawGuard.get(); if ( !raw ) { MsgStream log(msgSvc(), name()); - log << MSG::ERROR << " ==> There is no RawEvent at: " << m_rawEvent << endmsg; - execState(ctxt).setFilterPassed(false); + log << MSG::ERROR << " ==> No raw data at: " << m_rawGuard << endmsg; return StatusCode::FAILURE; } /// If mothing is specified everything will be downscaled + bool downscale = false; if ( m_downScale.empty() && m_triggerTypesToPass.empty() && m_calibTypesToPass.empty() ) { - if ( m_rate < 1.0 ) { - double frac = double(::rand()) / double(RAND_MAX); - if ( m_rate < frac ) { - execState(ctxt).setFilterPassed(false); - return _halt(delay); - } - } - execState(ctxt).setFilterPassed(true); - return _halt(delay); + downscale = (m_rate < 1.0 && m_rate < (double(::rand()) / double(RAND_MAX))); } - - const auto odinBanks = raw->banks(LHCb::RawBank::ODIN); - if ( odinBanks.empty() ) { - execState(ctxt).setFilterPassed(false); - return _halt(delay); - } - - const RunInfo *odin = odinBanks[0]->begin<RunInfo>(); - int trgType = odin->triggerType; - - if ( !m_downScale.empty() ) { /// First downscale certain trigger types if required - if ( std::find(m_downScale.begin(),m_downScale.end(),trgType) != m_downScale.end() ) { - double frac = double(::rand()) / double(RAND_MAX); - if ( m_rate < frac ) { - execState(ctxt).setFilterPassed(false); - return _halt(delay); + else { + EventAccess::record_t record = raw->second->at(raw->first); + if ( raw->second->type() == event_traits::tell1::data_type ) { + const auto *odin = this->get_odin<event_traits::tell1>(record); + if ( !odin ) + downscale = true; + else { + if ( type_found(odin->triggerType, m_downScale) ) + downscale = (m_rate < 1.0 && m_rate < (double(::rand()) / double(RAND_MAX))); + /// Require a certain trigger type + if ( !downscale && !type_found(odin->triggerType, m_triggerTypesToPass) ) + downscale = true; + /// Require a certain calibration type + if ( !downscale && odin->triggerType == 7 && !type_found(odin->readoutType, m_calibTypesToPass) ) + downscale = true; } } - } - - // Check trigger type selection (may be ANDed with calibration type selection - if( !m_triggerTypesToPass.empty() ) { - if( std::find( m_triggerTypesToPass.begin(), m_triggerTypesToPass.end(), trgType) == - m_triggerTypesToPass.end() ) { - execState(ctxt).setFilterPassed(false); - return _halt(delay); - } - if( !m_calibTypesToPass.empty() && (odin->triggerType == 7)) { //calibration type check - if ( std::find( m_calibTypesToPass.begin(), m_calibTypesToPass.end(), odin->readoutType) == - m_calibTypesToPass.end() ) { - execState(ctxt).setFilterPassed(false); - return _halt(delay); - } - } - } - else if( !m_calibTypesToPass.empty() ) { - // Pure calibration type selection - if ( std::find( m_calibTypesToPass.begin(), m_calibTypesToPass.end(), odin->readoutType) == - m_calibTypesToPass.end() ) { - execState(ctxt).setFilterPassed(false); - return _halt(delay); + else if ( raw->second->type() == event_traits::tell40::data_type ) { + const auto *odin = this->get_odin<event_traits::tell40>(record); + if ( !odin ) + downscale = true; + else if ( !type_found(odin->event_type(), m_downScale) ) + downscale = true; + /// Require certain calibration types + if ( !downscale && !type_found(odin->calib_type(), m_calibTypesToPass) ) + downscale = true; } } - - execState(ctxt).setFilterPassed(true); - return _halt(delay); + if ( downscale ) + execState(ctxt).setFilterPassed(false); + else + execState(ctxt).setFilterPassed(true); + _halt(delay); + return StatusCode::SUCCESS; } diff --git a/Online/GaudiOnline/python/GaudiOnline/OnlineApplication.py b/Online/GaudiOnline/python/GaudiOnline/OnlineApplication.py index c7d6642a1..961e64665 100644 --- a/Online/GaudiOnline/python/GaudiOnline/OnlineApplication.py +++ b/Online/GaudiOnline/python/GaudiOnline/OnlineApplication.py @@ -139,6 +139,8 @@ class Application(object): log(MSG_INFO,'Setup event input for REGULAR processing') input = Configurables.Online__InputAlg('EventInput') input.DeclareData = True + input.DeclareEvent = True + input.DeclareErrors = False self.input = input return self.input diff --git a/Online/GaudiOnline/python/GaudiOnline/Passthrough.py b/Online/GaudiOnline/python/GaudiOnline/Passthrough.py index f57c40bcf..4584cc8fe 100644 --- a/Online/GaudiOnline/python/GaudiOnline/Passthrough.py +++ b/Online/GaudiOnline/python/GaudiOnline/Passthrough.py @@ -22,7 +22,8 @@ class Passthrough(Application): input = self.setup_event_input(TAE) self.input = input passThrough = Configurables.Online__Passthrough('Passthrough') - passThrough.RawLocation = '/Event/DAQ/RawEvent' + #passThrough = Configurables.Online__PassthroughPCIE40('Passthrough') + passThrough.RawGuard = '/Event/DAQ/RawDataGuard' passThrough.AcceptRate = acceptRate self.passThrough = passThrough sequence = Gaudi.GaudiSequencer('Output') diff --git a/Online/GaudiOnline/python/GaudiOnline/ReadDatafile.py b/Online/GaudiOnline/python/GaudiOnline/ReadDatafile.py index bddaf741b..2f63295f1 100644 --- a/Online/GaudiOnline/python/GaudiOnline/ReadDatafile.py +++ b/Online/GaudiOnline/python/GaudiOnline/ReadDatafile.py @@ -28,6 +28,7 @@ class ReadDatafile(Application): input = Configurables.Online__InputAlg('EventInput') input.DeclareData = True + input.DeclareEvent = True explorer = Configurables.StoreExplorerAlg('Explorer') explorer.Load = 1 explorer.PrintFreq = 1.0 @@ -35,7 +36,7 @@ class ReadDatafile(Application): ask = Configurables.Online__InteractiveAlg('DumpHandler') ask.Prompt = "Press <ENTER> to dump banks, q or Q to quit :" dump = Configurables.Online__RawEventTestDump('Dump') - dump.RawLocation = '/Event/DAQ/RawEvent' + dump.RawData = 'DAQ/RawData' dump.CheckData = 0 dump.DumpData = 1 dump.FullDump = 1 diff --git a/Online/GaudiOnline/python/GaudiOnline/configureEventConsumer.py b/Online/GaudiOnline/python/GaudiOnline/configureEventConsumer.py index 42b551dd2..eb9227931 100644 --- a/Online/GaudiOnline/python/GaudiOnline/configureEventConsumer.py +++ b/Online/GaudiOnline/python/GaudiOnline/configureEventConsumer.py @@ -9,7 +9,7 @@ def configureConsumer(appMgr, broker, raw, output=False): input = setupAlgorithm('Online__InputAlg', appMgr, broker, instanceName='EventInput', - RawLocation='/Event/DAQ/RawEvent', + RawData='/Event/DAQ/RawData', IOSvc='Online::IOService') application.setup_monitoring() application.setup_mbm_access('Input', True) diff --git a/Online/GaudiOnline/src/Configuration.cpp b/Online/GaudiOnline/src/Configuration.cpp index 369d700b3..b88b7aabc 100644 --- a/Online/GaudiOnline/src/Configuration.cpp +++ b/Online/GaudiOnline/src/Configuration.cpp @@ -14,8 +14,8 @@ //========================================================================== /// Framework include files -#include "GaudiOnline/Configuration.h" -#include "RTL/rtl.h" +#include <GaudiOnline/Configuration.h> +#include <RTL/rtl.h> /// C/C++ include files #include <thread> diff --git a/Online/GaudiOnline/src/OnlineApplication.cpp b/Online/GaudiOnline/src/OnlineApplication.cpp index 1ee91fbf6..0528cb670 100644 --- a/Online/GaudiOnline/src/OnlineApplication.cpp +++ b/Online/GaudiOnline/src/OnlineApplication.cpp @@ -14,7 +14,7 @@ //========================================================================== #define NO_LONGLONG_TYPEDEF -#include "dim/dis.hxx" +#include <dim/dis.hxx> #include <GaudiOnline/OnlineApplication.h> @@ -598,7 +598,7 @@ int OnlineApplication::unload() { setTargetState(ST_OFFLINE); declareState(ST_OFFLINE); ::lib_rtl_sleep(100); - ::exit(0); + ::_exit(0); return 1; } diff --git a/Online/GaudiOnline/src/QueuedFlowManager.cpp b/Online/GaudiOnline/src/QueuedFlowManager.cpp index 8e6951cc8..d797c163a 100644 --- a/Online/GaudiOnline/src/QueuedFlowManager.cpp +++ b/Online/GaudiOnline/src/QueuedFlowManager.cpp @@ -12,6 +12,8 @@ // // Author : Markus Frank //========================================================================== + +/// Framework include files #include <GaudiOnline/QueuedFlowManager.h> #include <thread> diff --git a/Online/GaudiOnlineTests/components/RawEventTestDump.cpp b/Online/GaudiOnlineTests/components/RawEventTestDump.cpp index 6f02fde15..5e30ebf14 100755 --- a/Online/GaudiOnlineTests/components/RawEventTestDump.cpp +++ b/Online/GaudiOnlineTests/components/RawEventTestDump.cpp @@ -16,7 +16,6 @@ // Include files from Gaudi #include <Gaudi/Algorithm.h> #include <GaudiKernel/DataObjectHandle.h> -#include <Event/RawEvent.h> #include <Tell1Data/Tell1Decoder.h> // C/C++ include files @@ -32,11 +31,15 @@ namespace Online { * @date 2005-10-13 */ class RawEventTestDump : public Gaudi::Algorithm { + + typedef Tell1Bank BankHeader; + typedef std::vector<std::pair<const BankHeader*, const void*> > evt_data_t; + Gaudi::Property<bool> m_full{ this, "FullDump", false, "FullDump: If true, full bank contents are dumped"}; Gaudi::Property<bool> m_dump{ this, "DumpData", false, "DumpData: If true, full bank contents are dumped"}; Gaudi::Property<bool> m_check{ this, "CheckData", false, "CheckData: If true, full bank contents are checked"}; Gaudi::Property<int> m_debug{ this, "Debug", 0, "Number of events where all dump flags should be considered true"}; - DataObjectReadHandle<LHCb::RawEvent> m_rawEvent{this, "RawLocation", LHCb::RawEventLocation::Default}; + DataObjectReadHandle<evt_data_t> m_rawData{this, "RawData", "DAQ/RawData"}; mutable std::atomic<long> m_numEvent; ///< Event counter public: @@ -50,23 +53,28 @@ namespace Online { /// Main execution callback StatusCode execute(EventContext const& /* ctxt */) const override { - auto raw = m_rawEvent.get(); MsgStream info(msgSvc(),name()); + const auto* raw = m_rawData.get(); + std::map<int, std::vector<const Tell1Bank*> > banks; bool dmp = m_numEvent<m_debug || m_dump; bool chk = m_numEvent<m_debug || m_check; bool ful = m_numEvent<m_debug || m_full; static int evt = 0; ++evt; info << MSG::INFO; - for(int j=0; j<Tell1Bank::LastType; ++j) { - auto b = raw->banks(LHCb::RawBank::BankType(j)); - int cnt, inc = (j == Tell1Bank::Rich) ? 64 : 32; + for ( const auto& dsc : *raw ) { + banks[dsc.first->type()].push_back(dsc.first); + } + for ( const auto& dsc : banks ) { + const auto& b = dsc.second; + int btyp = dsc.first; + int cnt, inc = (btyp == Tell1Bank::Rich) ? 64 : 32; const int *p; if ( b.size() > 0 ) { if ( dmp ) { info << "Evt No:" << std::left << std::setw(6) << evt - << " has " << b.size() << " bank(s) of type " << j - << " (" << Tell1Printout::bankType(j) << ") " << endmsg; + << " has " << b.size() << " bank(s) of type " << btyp + << " (" << Tell1Printout::bankType(btyp) << ") " << endmsg; } int k = 0; for(const auto* itB : b) { diff --git a/Online/OnlineBase/OnlineBase/RTL/Logger.h b/Online/OnlineBase/OnlineBase/RTL/Logger.h index 11290c1df..3afdd97d1 100644 --- a/Online/OnlineBase/OnlineBase/RTL/Logger.h +++ b/Online/OnlineBase/OnlineBase/RTL/Logger.h @@ -17,6 +17,7 @@ /// C/C++ include files #include <map> +#include <mutex> #include <string> #include <memory> #include <cstdarg> @@ -70,6 +71,8 @@ namespace RTL { }; public: + /// Device lock to protect against interleaved printouts + std::mutex* device_lock {nullptr}; /// Output format std::string format {"%TIME %LEVEL %-16SOURCE"}; /// The processes executable name @@ -225,6 +228,14 @@ namespace RTL { /// Format output string std::string format(const char* format, va_list& args) const; + /// Calls the display action with a given severity level + /** + * @arg severity [int,read-only] Display severity flag (see enum) + * @arg source [string,read-only] Originator name of the message + * @arg msg [string,read-only] Formatted message string + * @return Status code indicating success or failure + */ + size_t printmsg(int severity, const char* source, const char* msg) const; /// Always printout handling void always(const char* format, ...) const; /// Always printout handling diff --git a/Online/OnlineBase/src/RTL/Logger.cpp b/Online/OnlineBase/src/RTL/Logger.cpp index caa75f66d..679c820d7 100644 --- a/Online/OnlineBase/src/RTL/Logger.cpp +++ b/Online/OnlineBase/src/RTL/Logger.cpp @@ -41,7 +41,7 @@ namespace { } /// Default constructor -Logger::LogDevice::LogDevice() { +Logger::LogDevice::LogDevice() : device_lock(new std::mutex()) { char pathName[PATH_MAX] = ""; int len = ::readlink("/proc/self/exe",pathName,sizeof(pathName)-1); if ( len == -1 ) { @@ -58,6 +58,8 @@ Logger::LogDevice::LogDevice() { /// Default destructor Logger::LogDevice::~LogDevice() { + delete device_lock; + device_lock = nullptr; } /// Set global output device @@ -210,9 +212,11 @@ size_t Logger::LogDevice::formatHeader(char* buffer, size_t buff_len, int severi /// Calls the display action with a given severity level size_t Logger::LogDevice::printmsg(int severity, const char* source, const char* msg) const { char header[2048]; - size_t len = formatHeader(header, sizeof(header), severity, source); - len = ::fprintf(stdout, "%s %s\n", header, msg); - ::fflush(stdout); + size_t len = formatHeader(header, sizeof(header), severity, source); { + lock_guard<std::mutex> lock(*device_lock); + len = ::fprintf(stdout, "%s %s\n", header, msg); + ::fflush(stdout); + } return len+1; // '1' from fputc } @@ -235,9 +239,11 @@ size_t Logger::LogDevice::printout(int severity, const char* source, const char* /// Throw exception with message void Logger::LogDevice::exceptmsg(const char* source, const char* msg) const { char header[2048]; - formatHeader(header, sizeof(header), LIB_RTL_ERROR, source); - ::fprintf(stdout, "%s %s\n", header, msg); - ::fflush(stdout); + formatHeader(header, sizeof(header), LIB_RTL_ERROR, source); { + lock_guard<std::mutex> lock(*device_lock); + ::fprintf(stdout, "%s %s\n", header, msg); + ::fflush(stdout); + } throw runtime_error(header+string(" ")+msg); } @@ -347,6 +353,11 @@ string Logger::format(const char* fmt, va_list& args) const { return text; } +/// Calls the display action with a given severity level +size_t Logger::printmsg(int severity, const char* source, const char* msg) const { + return device->printmsg(severity, source, msg); +} + /// Calls the display action with a given severity level size_t Logger::printout(int severity, const char* fmt, va_list& args) const { return device->printout(severity, source.c_str(), fmt, args); diff --git a/Online/PCIE40Data/PCIE40Data/pcie40.h b/Online/PCIE40Data/PCIE40Data/pcie40.h index 46089556d..71ebd03a0 100644 --- a/Online/PCIE40Data/PCIE40Data/pcie40.h +++ b/Online/PCIE40Data/PCIE40Data/pcie40.h @@ -264,8 +264,10 @@ namespace Online { friend class event_collection_t; event_t() = default; public: - MAGIC_WORD - size_t slot; + unsigned long magic = MAGIC_PATTERN; + unsigned long pattern() const { return magic; } + void setPattern() { magic = MAGIC_PATTERN; } + size_t slot; union _flags { unsigned long raw; struct _detail { @@ -277,7 +279,7 @@ namespace Online { } flags; bank_collection_t banks[1]; - static constexpr size_t data_offset = DATA_OFFSET(sizeof(size_t)+sizeof(unsigned long)); + static constexpr size_t data_offset = sizeof(size_t)+2*sizeof(unsigned long); private: /// Disabled move constructor @@ -308,6 +310,7 @@ namespace Online { class event_collection_t { public: MAGIC_WORD + const mep_header_t* mep; size_t length; size_t capacity; event_t events[1]; @@ -331,6 +334,7 @@ namespace Online { size_t size() const { return length; } size_t num_events() const { return length; } size_t max_events() const { return capacity; } + const mep_header_t* mep_header() const { return mep; } const event_t* begin() const; const event_t* end() const; const event_t* next(const event_t* e) const; @@ -662,7 +666,7 @@ namespace Online { len += e->total_length(); return len; } - + } // namespace pcie40 } // namepace Online diff --git a/Online/PCIE40Data/PCIE40Data/pcie40decoder.h b/Online/PCIE40Data/PCIE40Data/pcie40decoder.h index b81a56bcf..d088ea28a 100644 --- a/Online/PCIE40Data/PCIE40Data/pcie40decoder.h +++ b/Online/PCIE40Data/PCIE40Data/pcie40decoder.h @@ -16,6 +16,7 @@ #include <PCIE40Data/pcie40.h> // C/C++ include files +#include <functional> #include <memory> /// Online namespace declaration @@ -46,16 +47,22 @@ namespace Online { */ class decoder_t final { public: + enum { PRINT = 3, ERROR = 5 }; typedef bool (*bank_selector)(unsigned char); typedef bool (*source_selector)(unsigned short); + typedef std::function<void(int, const char* source, const char* msg)> logger_t; + class implementation_t; + static void _prt(int lvl, const char* source, const char* msg); public: /// Debug flag - bool debug {false}; - + bool debug {false}; + implementation_t* implementation {nullptr}; public: /// Default constructor - decoder_t() = default; + decoder_t(); + /// Initializing constructor with printout object + decoder_t(logger_t& l); /// INHIBIT Move constructor decoder_t(decoder_t&& copy) = delete; /// INHIBIT Copy constructor @@ -65,7 +72,7 @@ namespace Online { /// INHIBIT Copy assignment decoder_t& operator=(const decoder_t& copy) = delete; /// Default destructor - ~decoder_t() = default; + ~decoder_t(); /// Initialize user data structure event_collection_t* initialize(std::unique_ptr<event_collection_t>& event_collection, size_t num_events) const; diff --git a/Online/PCIE40Data/src/pcie40decoder.cpp b/Online/PCIE40Data/src/pcie40decoder.cpp index 3afadbde0..0f09bfab7 100644 --- a/Online/PCIE40Data/src/pcie40decoder.cpp +++ b/Online/PCIE40Data/src/pcie40decoder.cpp @@ -30,31 +30,123 @@ using namespace Online::pcie40; namespace { - void _error(const char* ,...) {} - - void _error2(const char* fmt,...) { + void _error(const char* fmt,...) { char text[1024]; va_list args; va_start(args,fmt); - ::vsnprintf(text, sizeof(text),fmt,args); + ::vsnprintf(text, sizeof(text), fmt, args); va_end(args); text[sizeof(text)-1] = 0; ::write(fileno(stdout), text, strlen(text)); } + void _check(const event_collection_t& c) { + size_t num_bank_coll = 0; + size_t num_event = 0; + size_t num_bank = 0; + if ( c.pattern() != MAGIC_PATTERN ) { + _error("Bad magic pattern in event collection!\n"); + } + for(size_t j=0; j < c.capacity; ++j) { + const event_t* e = c.at(j); + if ( e->slot != j || e->pattern() != MAGIC_PATTERN ) { + _error("Bad magic pattern in event!\n"); + } + ++num_event; + for(size_t i=0; i<sizeof(params::collectionOffsets)/sizeof(params::collectionOffsets[0]); ++i) { + const auto* b = add_ptr<bank_collection_t>(e,params::collectionOffsets[i]); + if ( b->capacity != params::collectionCapacity[i] ) { + _error("Bad bank collection!\n"); + } + if ( b->pattern() != MAGIC_PATTERN ) { + _error("Bad magic pattern in bank collection!\n"); + } + ++num_bank_coll; + for(size_t k=0; k < b->capacity; ++k) { + const bank_t* bnk = b->at(k); + if ( bnk->magic() != bank_t::MagicPattern ) { + _error("Bad magic pattern in raw bank!\n"); + } + ++num_bank; + } + } + } + _error("Checked %ld event slots, %ld bank collections and %ld bank slots\n", + num_event, num_bank_coll, num_bank); + } + + void _print(const event_collection_t& c) { + size_t num_bank_coll = 0; + size_t num_event = 0; + size_t num_bank = 0; + _error("Dump PCIE40 event collection @%p capacity:%ld events magic:%p\n", &c, c.capacity, c.pattern()); + for(size_t j=0; j < c.capacity; ++j) { + const event_t* e = c.at(j); + if ( e->slot != j || e->pattern() != MAGIC_PATTERN ) { + _error("Bad magic pattern in event!\n"); + } + ++num_event; + for(size_t i=0; i<sizeof(params::collectionOffsets)/sizeof(params::collectionOffsets[0]); ++i) { + const auto* b = add_ptr<bank_collection_t>(e,params::collectionOffsets[i]); + if ( b->capacity != params::collectionCapacity[i] ) { + _error("Bad bank collection!\n"); + } + if ( b->pattern() != MAGIC_PATTERN ) { + _error("Bad magic pattern in bank collection!\n"); + } + if ( j == 0 ) { + _error("\t bank collection @%p capacity:%3ld banks:%3ld special:%3ld magic:%p offset:%ld\n", + b, b->capacity, b->length, b->specials, b->pattern(), params::collectionOffsets[i]); + } + ++num_bank_coll; + for(size_t k=0; k < b->capacity; ++k) { + const bank_t* bnk = b->at(k); + if ( bnk->magic() != bank_t::MagicPattern ) { + _error("Bad magic pattern in raw bank!\n"); + } + ++num_bank; + } + } + } + _error("Checked %ld event slots, %ld bank collections and %ld bank slots\n", + num_event, num_bank_coll, num_bank); + } + inline bool take_any(unsigned short) { return true; } +} + +class decoder_t::implementation_t { +public: + enum { PRINT = decoder_t::PRINT, ERROR = decoder_t::ERROR }; + decoder_t::logger_t log; + + implementation_t(decoder_t::logger_t& logger) : log(logger) { + } + + inline void print(int lvl, const char* fmt,...) { + if ( lvl > PRINT ) { + char text[1024]; + va_list args; + va_start(args,fmt); + ::vsnprintf(text, sizeof(text), fmt, args); + va_end(args); + text[sizeof(text)-1] = 0; + log(lvl, "pcie40decoder", text); + } + } + inline bank_t* allocate_regular_bank(event_t* e, size_t offset) { bank_collection_writer_t* c = add_ptr<bank_collection_writer_t>(e,offset); ++c->length; #ifdef PCIE40_ENABLE_CHECKS if ( __UNLIKELY(c->pattern() != MAGIC_PATTERN) ) { - _error("Bad magic pattern in bank collection!\n"); + this->print(ERROR, "Bad magic pattern in bank collection!\n"); } if ( __UNLIKELY(c->length + c->specials > c->capacity) ) { - _error("Bank allocation error: Too many banks!\n"); + this->print(ERROR, "Bank allocation error: Too many banks!\n"); } bank_t& b = c->banks[c->length-1]; if ( __UNLIKELY(b.magic() != b.MagicPattern) ) { - _error("Bad magic pattern in bank!\n"); + this->print(ERROR, "Bad magic pattern in bank!\n"); } return &b; #else @@ -67,14 +159,14 @@ namespace { ++c->specials; #ifdef PCIE40_ENABLE_CHECKS if ( c->pattern() != MAGIC_PATTERN ) { - _error("Bad magic pattern in bank collection!\n"); + this->print(ERROR, "+++ Bad magic pattern in bank collection!\n"); } if ( c->length + c->specials > c->capacity ) { - _error("Bank allocation error: Too many banks!\n"); + this->print(ERROR, "+++ Bank allocation error: Too many banks!\n"); } bank_t& b = c->banks[c->capacity - c->specials]; if ( b.magic() != b.MagicPattern ) { - _error("Bad magic pattern in bank!\n"); + this->print(ERROR, "Bad magic pattern in bank!\n"); } return &b; #else @@ -99,10 +191,14 @@ namespace { const uint8_t *typs = mfp->types(); const uint16_t *lens = mfp->sizes(); const frontend_data_t *curr = mfp->data(); - bool run = true; + //bool run = true; + if ( (const char*)typs > ((const char*)mfp)+6*sizeof(int) ) { - _error2("Inconsistent TYPES pointer\n"); - while(run) ::sleep(1); + this->print(ERROR, + "Inconsistent TYPES pointer. Drop MFP Source ID:%3d\n", + int(src_id)); + //while(run) ::sleep(1); + return 0; } for (size_t j=0, cnt=0, n=mfp->packingFactor(); cnt<n; ++cnt, ++typs, ++lens) { auto typx = *typs; @@ -111,24 +207,27 @@ namespace { bank_t* b = allocate_bank(e, src_id, typx); if ( nullptr == b ) { - _error2("ERROR +++ Ignore bank: Unknown bank type:%3d Source ID:%3d Size:%4d vsn:%d j:%ld cnt:%ld n:%ld\n", - int(typx), int(src_id), int(length), int(version), j, cnt, n); - while(run) ::sleep(1); + this->print(ERROR, + "+++ Ignore bank: Unknown bank type:%3d " + "Source ID:%3d Size:%4d vsn:%d j:%ld cnt:%ld n:%ld\n", + int(typx), int(src_id), int(length), int(version), j, cnt, n); + //while(run) ::sleep(1); + continue; } else { #ifdef PCIE40_ENABLE_CHECKS if ( __UNLIKELY(e->magic != MAGIC_PATTERN) ) { - _error("Data corruption in event!\n"); + this->print(ERROR, "Data corruption in event!\n"); } if ( __UNLIKELY(b->magic() != b->MagicPattern) ) { - _error("Data corruption in raw bank!\n"); + this->print(ERROR, "Data corruption in raw bank!\n"); } #endif const void* data = curr->data(); if ( debug ) { - ::printf("+++ Reading bank[%2ld] at %p len:%4d %p typ:%3d src:%3d vsn:%d\n", - cnt, (void*)data, int(length), (void*)((char*)data + length), - typx, src_id, version); + this->print(PRINT, "+++ Reading bank[%2ld] at %p len:%4d %p typ:%3d src:%3d vsn:%d\n", + cnt, (void*)data, int(length), (void*)((char*)data + length), + typx, src_id, version); } //b->setMagic(); b->setSize(length); @@ -136,12 +235,6 @@ namespace { b->setData(data); b->setType((RawBank40::BankType)typx); b->setVersion(version); -#if 0 - int* p1 = (int*)b->data(); - int* p2 = (int*)curr->data(); - for(size_t i=0; i<b->size()/sizeof(int); ++i) - p1[i] = p2[i]; -#endif ++num_bank; } curr = curr->next(length, align); @@ -149,87 +242,16 @@ namespace { } return num_bank; } - - void _check(const event_collection_t& c) { - size_t num_bank_coll = 0; - size_t num_event = 0; - size_t num_bank = 0; - if ( c.pattern() != MAGIC_PATTERN ) { - _error("Bad magic pattern in event collection!\n"); - } - for(size_t j=0; j < c.capacity; ++j) { - const event_t* e = c.at(j); - if ( e->slot != j || e->pattern() != MAGIC_PATTERN ) { - _error("Bad magic pattern in event!\n"); - } - ++num_event; - for(size_t i=0; i<sizeof(params::collectionOffsets)/sizeof(params::collectionOffsets[0]); ++i) { - const auto* b = add_ptr<bank_collection_t>(e,params::collectionOffsets[i]); - if ( b->capacity != params::collectionCapacity[i] ) { - _error("Bad bank collection!\n"); - } - if ( b->pattern() != MAGIC_PATTERN ) { - _error("Bad magic pattern in bank collection!\n"); - } - ++num_bank_coll; - for(size_t k=0; k < b->capacity; ++k) { - const bank_t* bnk = b->at(k); - if ( bnk->magic() != bank_t::MagicPattern ) { - _error("Bad magic pattern in raw bank!\n"); - } - ++num_bank; - } - } - } - _error("Checked %ld event slots, %ld bank collections and %ld bank slots\n", - num_event, num_bank_coll, num_bank); - } - - void _print(const event_collection_t& c) { - size_t num_bank_coll = 0; - size_t num_event = 0; - size_t num_bank = 0; - _error("Dump PCIE40 event collection @%p capacity:%ld events magic:%p\n", &c, c.capacity, c.pattern()); - for(size_t j=0; j < c.capacity; ++j) { - const event_t* e = c.at(j); - if ( e->slot != j || e->pattern() != MAGIC_PATTERN ) { - _error("Bad magic pattern in event!\n"); - } - ++num_event; - for(size_t i=0; i<sizeof(params::collectionOffsets)/sizeof(params::collectionOffsets[0]); ++i) { - const auto* b = add_ptr<bank_collection_t>(e,params::collectionOffsets[i]); - if ( b->capacity != params::collectionCapacity[i] ) { - _error("Bad bank collection!\n"); - } - if ( b->pattern() != MAGIC_PATTERN ) { - _error("Bad magic pattern in bank collection!\n"); - } - if ( j == 0 ) { - _error("\t bank collection @%p capacity:%3ld banks:%3ld special:%3ld magic:%p offset:%ld\n", - b, b->capacity, b->length, b->specials, b->pattern(), params::collectionOffsets[i]); - } - ++num_bank_coll; - for(size_t k=0; k < b->capacity; ++k) { - const bank_t* bnk = b->at(k); - if ( bnk->magic() != bank_t::MagicPattern ) { - _error("Bad magic pattern in raw bank!\n"); - } - ++num_bank; - } - } - } - _error("Checked %ld event slots, %ld bank collections and %ld bank slots\n", - num_event, num_bank_coll, num_bank); - } - + event_collection_t* _allocate(size_t num_events) { size_t len = sizeof(event_collection_t) + num_events*params::eventRecordLength; event_collection_writer_t* c; - void* ptr = ::malloc(len); + void* ptr = ::operator new(len); ::memset(ptr,0,len); c = (event_collection_writer_t*)ptr; c->length = 0; + c->mep = nullptr; c->capacity = num_events; c->setPattern(); for(size_t j=0; j<num_events; ++j) { @@ -254,6 +276,7 @@ namespace { void _initialize(event_collection_writer_t* ec) { ec->length = 0; + ec->mep = nullptr; for(size_t j=0; j < ec->capacity; ++j) { auto* e = ec->at(j); e->flags.raw = 0; @@ -265,8 +288,7 @@ namespace { } } } - inline bool take_any(unsigned short) { return true; } -} +}; void printer_t::print_summary(const std::unique_ptr<event_collection_t>& coll) const { if ( coll ) { @@ -284,22 +306,46 @@ void printer_t::check(const std::unique_ptr<event_collection_t>& coll) const { _error("Invalid event_collection_t instance passed. Cannot check!"); } +void decoder_t::_prt(int, const char* source, const char* msg) { + _error("%s %s", source, msg); +} + +/// Default constructor +decoder_t::decoder_t() { + logger_t log = decoder_t::_prt; + implementation = new implementation_t(log); +} + +/// Initializing constructor with printout object +decoder_t::decoder_t(logger_t& logger) { + implementation = new implementation_t(logger); +} + +/// Default destructor +decoder_t::~decoder_t() { + delete implementation; + implementation = nullptr; +} + +/// Initialize user data structure event_collection_t* decoder_t::initialize(std::unique_ptr<event_collection_t>& c, size_t num_events) const { if ( !c.get() || c->capacity < num_events ) { - if ( c.get() ) ::free(c.release()); - c.reset(_allocate(num_events)); + if ( c.get() ) ::operator delete(c.release()); + c.reset(implementation->_allocate(num_events)); return c.get(); } - _initialize((event_collection_writer_t*)c.get()); + implementation->_initialize((event_collection_writer_t*)c.get()); return c.get(); } +/// Decode MEP fragment void decoder_t::decode(std::unique_ptr<event_collection_t>& coll, const mep_header_t* mep) const { decode(coll, mep, take_any); } +/// Decode MEP fragment with predicate void decoder_t::decode(std::unique_ptr<event_collection_t>& ev_coll, const mep_header_t* mep, source_selector source_selector) const @@ -311,18 +357,21 @@ void decoder_t::decode(std::unique_ptr<event_collection_t>& ev_coll, if ( num_fragments > 0 ) { size_t packing = mep->multi_fragment(0)->packingFactor(); auto ec = (event_collection_writer_t*)initialize(ev_coll, packing); + ec->mep = mep; ec->length = packing; for(uint32_t i=0; i<num_fragments; ++i) { const multi_fragment_t *mfp = mep->multi_fragment(i); const multi_fragment_t::header_t& hdr = mfp->header; uint16_t src_id = hdr.source_id; if ( debug ) { - ::printf("+++ Reading mfp[%2d] at %p len:%6d %p magic:%04X pack:%3d eid:%8ld src:%3d vsn:%d align:%d\n", - i, (void*)mfp, int(hdr.size), - (void*)((char*)mfp + hdr.size), - int(hdr.magic), int(hdr.packing), long(hdr.event_id), - int(src_id), int(hdr.version), - int(hdr.alignment)); + implementation->print(PRINT, + "+++ Reading mfp[%2d] at %p len:%6d %p magic:%04X " + "pack:%3d eid:%8ld src:%3d vsn:%d align:%d\n", + i, (void*)mfp, int(hdr.size), + (void*)((char*)mfp + hdr.size), + int(hdr.magic), int(hdr.packing), long(hdr.event_id), + int(src_id), int(hdr.version), + int(hdr.alignment)); } #ifdef PCIE40_ENABLE_CHECKS if ( mfp->packingFactor() != packing ) { @@ -330,11 +379,13 @@ void decoder_t::decode(std::unique_ptr<event_collection_t>& ev_coll, } #endif if ( !source_selector || source_selector(src_id¶ms::sourceid_TOP5) ) { - num_banks += link_fragment(ec, mfp, debug); + num_banks += implementation->link_fragment(ec, mfp, debug); } } if ( num_err_packing > 0 ) { - _error("Got %ld fragments with inconsistent packing factor!\n", num_err_packing); + implementation->print(ERROR, + "+++ Got %ld fragments with inconsistent packing factor!\n", + num_err_packing); } return; } diff --git a/Online/SmiController/src/SmiController.cpp b/Online/SmiController/src/SmiController.cpp index a8bc7b043..a5768a76e 100644 --- a/Online/SmiController/src/SmiController.cpp +++ b/Online/SmiController/src/SmiController.cpp @@ -1287,7 +1287,9 @@ void SmiController::run() { if ( handle ) { Function fun(GETPROC(handle, "main")); if ( fun.function ) { - thread* thr = new thread([] () { IocSensor::instance().run(); }); + thread* thr = new thread([] () { + IocSensor::instance().run(); + }); if ( thr ) { m_log->info("Executing internal SMI server in thread."); const char* args[] = {"smiSM", "-u", "-t", "-d", config.smi_debug.c_str(), @@ -1296,7 +1298,9 @@ void SmiController::run() { config.smi_domain.c_str(), config.smi_file.c_str(), nullptr}; /// function will never return.... + ::lib_rtl_sleep(2500); (*fun.function)(sizeof(args)/sizeof(args[0])-1, (char**)args); + thr->join(); } } } -- GitLab From e202ff742507e1146de51d9c180e0dd60fb6f138 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Mon, 29 Mar 2021 10:28:41 +0200 Subject: [PATCH 10/26] Fox nightly build errors on dev3 --- Online/Dataflow/dict/dictionary.h | 8 ++++---- Online/RPC/RPC/GUI/MessageBox.h | 1 + Online/TestBeam/TestBeam/gui/MessageBox.h | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Online/Dataflow/dict/dictionary.h b/Online/Dataflow/dict/dictionary.h index a803aa2ff..4f764d9e6 100755 --- a/Online/Dataflow/dict/dictionary.h +++ b/Online/Dataflow/dict/dictionary.h @@ -53,22 +53,22 @@ namespace Online { }; struct PropertyResult { - string data; + std::string data; int status; PropertyResult() : status(0) {} - PropertyResult(const string& d, int s) : data(d), status(s) {} + PropertyResult(const std::string& d, int s) : data(d), status(s) {} PropertyResult(const PropertyResult& c) : data(c.data), status(c.status) {} ~PropertyResult() {} }; class DataflowComponentProperties { public: - static PropertyResult getProperty(DataflowComponent* action, const string& name) { + static PropertyResult getProperty(DataflowComponent* action, const std::string& name) { if ( action->hasProperty(name) ) { return PropertyResult(action->property(name).str(),1); } return PropertyResult("",0); } - static int setProperty(DataflowComponent* action, const string& name, const string& value) { + static int setProperty(DataflowComponent* action, const std::string& name, const std::string& value) { if ( action->hasProperty(name) ) { action->property(name).str(value); return 1; diff --git a/Online/RPC/RPC/GUI/MessageBox.h b/Online/RPC/RPC/GUI/MessageBox.h index f45ff0e07..d149b6da2 100644 --- a/Online/RPC/RPC/GUI/MessageBox.h +++ b/Online/RPC/RPC/GUI/MessageBox.h @@ -17,6 +17,7 @@ /// Framework include files #include "TGMsgBox.h" +#include "TGPicture.h" /// C++ include files diff --git a/Online/TestBeam/TestBeam/gui/MessageBox.h b/Online/TestBeam/TestBeam/gui/MessageBox.h index 964274e21..b8fb04920 100644 --- a/Online/TestBeam/TestBeam/gui/MessageBox.h +++ b/Online/TestBeam/TestBeam/gui/MessageBox.h @@ -17,6 +17,7 @@ /// Framework include files #include "TGMsgBox.h" +#include "TGPicture.h" /// C++ include files -- GitLab From e0c80b481512689d4fae91f68154f273f963b1f7 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Mon, 29 Mar 2021 10:50:55 +0200 Subject: [PATCH 11/26] Fix nightly build errors --- .../python/GaudiOnlineTests/MDFtests.py | 31 ++++++++++--------- .../tests/refs/go_create_mdf.ref | 7 ----- .../mdf.qms/mdf_03_read_mdf.qmt | 0 .../mdf.qms/mdf_04_create_mif.qmt | 0 .../mdf.qms/mdf_05_read_mif.qmt | 0 .../mdf.qms/mdf_08_create_tae.qmt | 0 .../mdf.qms/mdf_11_create_mep.qmt | 0 .../mdf.qms/mdf_12_read_mep.qmt | 0 8 files changed, 17 insertions(+), 21 deletions(-) rename Online/GaudiOnlineTests/tests/{qmtest => retired}/mdf.qms/mdf_03_read_mdf.qmt (100%) rename Online/GaudiOnlineTests/tests/{qmtest => retired}/mdf.qms/mdf_04_create_mif.qmt (100%) rename Online/GaudiOnlineTests/tests/{qmtest => retired}/mdf.qms/mdf_05_read_mif.qmt (100%) rename Online/GaudiOnlineTests/tests/{qmtest => retired}/mdf.qms/mdf_08_create_tae.qmt (100%) rename Online/GaudiOnlineTests/tests/{qmtest => retired}/mdf.qms/mdf_11_create_mep.qmt (100%) rename Online/GaudiOnlineTests/tests/{qmtest => retired}/mdf.qms/mdf_12_read_mep.qmt (100%) diff --git a/Online/GaudiOnlineTests/python/GaudiOnlineTests/MDFtests.py b/Online/GaudiOnlineTests/python/GaudiOnlineTests/MDFtests.py index 4be6e4f2c..077b63fcd 100755 --- a/Online/GaudiOnlineTests/python/GaudiOnlineTests/MDFtests.py +++ b/Online/GaudiOnlineTests/python/GaudiOnlineTests/MDFtests.py @@ -91,20 +91,23 @@ def setupApp(): return app #------------------------------------------------------------------------------------------------ -def mdfCheck(): +def mdfCheck(dump=True, explore=True): app = setupApp() - dmp = Configs.Online__RawEventTestDump('Dump') - dmp.CheckData = 1 - dmp.CheckData = 0 - dmp.DumpData = 0 - dmp.FullDump = 0 - dmp.OutputLevel = 3 - exp = CFG.StoreExplorerAlg('Explorer') - exp.Load = 1 - exp.PrintFreq = 0.00021 - exp.AccessForeign = True - exp.OutputLevel = 3 - app.TopAlg += [dmp,exp] + if dump: + dmp = Configs.Online__RawEventTestDump('Dump') + dmp.CheckData = 1 + dmp.CheckData = 0 + dmp.DumpData = 0 + dmp.FullDump = 0 + dmp.OutputLevel = 3 + app.TopAlg += [dmp] + if explore: + exp = CFG.StoreExplorerAlg('Explorer') + exp.Load = 1 + exp.PrintFreq = 0.00021 + exp.AccessForeign = True + exp.OutputLevel = 3 + app.TopAlg += [exp] return app #------------------------------------------------------------------------------------------------ @@ -153,7 +156,7 @@ def _createMDF(test_castor=None): app = ApplicationMgr() app.EvtSel = 'NONE' app.TopAlg = [Configs.Online__RawEventTestCreator('Creator')] - mdfCheck() + mdfCheck(dmp=False) # To create a MDF file using the standard Gaudi output stream: wr = CFG.OutputStream('Writer_0') wr.Output = "DATAFILE='PFN:root:file://mdfData_0.dat' SVC='LHCb::RawDataCnvSvc' OPT='REC'" diff --git a/Online/GaudiOnlineTests/tests/refs/go_create_mdf.ref b/Online/GaudiOnlineTests/tests/refs/go_create_mdf.ref index b55541d25..fec2f3ef2 100644 --- a/Online/GaudiOnlineTests/tests/refs/go_create_mdf.ref +++ b/Online/GaudiOnlineTests/tests/refs/go_create_mdf.ref @@ -23,16 +23,9 @@ ApplicationMgr INFO Application Manager Configured successfully ], "TopAlg": [ "Online::RawEventTestCreator/Creator", - "Online::RawEventTestDump/Dump", "StoreExplorerAlg/Explorer" ] }, - "Dump": { - "CheckData": false, - "DumpData": false, - "FullDump": false, - "OutputLevel": 3 - }, "EventDataSvc": { "EnableFaultHandler": true, "ForceLeaves": true, diff --git a/Online/GaudiOnlineTests/tests/qmtest/mdf.qms/mdf_03_read_mdf.qmt b/Online/GaudiOnlineTests/tests/retired/mdf.qms/mdf_03_read_mdf.qmt similarity index 100% rename from Online/GaudiOnlineTests/tests/qmtest/mdf.qms/mdf_03_read_mdf.qmt rename to Online/GaudiOnlineTests/tests/retired/mdf.qms/mdf_03_read_mdf.qmt diff --git a/Online/GaudiOnlineTests/tests/qmtest/mdf.qms/mdf_04_create_mif.qmt b/Online/GaudiOnlineTests/tests/retired/mdf.qms/mdf_04_create_mif.qmt similarity index 100% rename from Online/GaudiOnlineTests/tests/qmtest/mdf.qms/mdf_04_create_mif.qmt rename to Online/GaudiOnlineTests/tests/retired/mdf.qms/mdf_04_create_mif.qmt diff --git a/Online/GaudiOnlineTests/tests/qmtest/mdf.qms/mdf_05_read_mif.qmt b/Online/GaudiOnlineTests/tests/retired/mdf.qms/mdf_05_read_mif.qmt similarity index 100% rename from Online/GaudiOnlineTests/tests/qmtest/mdf.qms/mdf_05_read_mif.qmt rename to Online/GaudiOnlineTests/tests/retired/mdf.qms/mdf_05_read_mif.qmt diff --git a/Online/GaudiOnlineTests/tests/qmtest/mdf.qms/mdf_08_create_tae.qmt b/Online/GaudiOnlineTests/tests/retired/mdf.qms/mdf_08_create_tae.qmt similarity index 100% rename from Online/GaudiOnlineTests/tests/qmtest/mdf.qms/mdf_08_create_tae.qmt rename to Online/GaudiOnlineTests/tests/retired/mdf.qms/mdf_08_create_tae.qmt diff --git a/Online/GaudiOnlineTests/tests/qmtest/mdf.qms/mdf_11_create_mep.qmt b/Online/GaudiOnlineTests/tests/retired/mdf.qms/mdf_11_create_mep.qmt similarity index 100% rename from Online/GaudiOnlineTests/tests/qmtest/mdf.qms/mdf_11_create_mep.qmt rename to Online/GaudiOnlineTests/tests/retired/mdf.qms/mdf_11_create_mep.qmt diff --git a/Online/GaudiOnlineTests/tests/qmtest/mdf.qms/mdf_12_read_mep.qmt b/Online/GaudiOnlineTests/tests/retired/mdf.qms/mdf_12_read_mep.qmt similarity index 100% rename from Online/GaudiOnlineTests/tests/qmtest/mdf.qms/mdf_12_read_mep.qmt rename to Online/GaudiOnlineTests/tests/retired/mdf.qms/mdf_12_read_mep.qmt -- GitLab From a2a1def5faa8ba5152ccfc6f52569ebee8eb3e48 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Tue, 30 Mar 2021 09:34:46 +0200 Subject: [PATCH 12/26] Fix nightly build errors --- Online/EventBuilding/options/EB_BU_algo.opts | 6 +++--- Online/EventBuilding/options/EB_RU_algo.opts | 4 ++-- Online/EventBuilding/options/EB_Transport_properties.opts | 4 ++-- Online/EventBuilding/src/RU.cpp | 3 ++- Online/GaudiOnlineTests/python/GaudiOnlineTests/MDFtests.py | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Online/EventBuilding/options/EB_BU_algo.opts b/Online/EventBuilding/options/EB_BU_algo.opts index 59f599362..dff05b7be 100755 --- a/Online/EventBuilding/options/EB_BU_algo.opts +++ b/Online/EventBuilding/options/EB_BU_algo.opts @@ -1,7 +1,7 @@ -BU.MBM_name = { 'Input', 'Input', 'Input', 'Input', 'Input', 'Input' }; +BU.MBM_name = { 'Input', 'Input', 'Input', 'Input', 'Input', 'Input', 'Input', 'Input' }; BU.buffer_size = {}; -BU.buffer_type = { 2, 2, 2, 2, 2, 2 }; +BU.buffer_type = { 2, 2, 2, 2, 2, 2, 2, 2 }; BU.out_file_prefix = ""; BU.shmem_prefix = ""; BU.stop_timeout = 0; -BU.write_to_file = { FALSE, FALSE, FALSE, FALSE, FALSE, FALSE }; +BU.write_to_file = { FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE }; diff --git a/Online/EventBuilding/options/EB_RU_algo.opts b/Online/EventBuilding/options/EB_RU_algo.opts index 793a81403..05fb1ce0c 100755 --- a/Online/EventBuilding/options/EB_RU_algo.opts +++ b/Online/EventBuilding/options/EB_RU_algo.opts @@ -1,9 +1,9 @@ RU.MDF_filename = "/home/fpisani/mc_data/biger_file.mdf"; RU.PCIe40_ids = {}; -RU.PCIe40_names = { 'tdtel022_0', 'tdtel022_1', 'tdtel041_0', 'tdtel041_1', 'tdtel042_0', 'tdtel042_1', 'tdtel043_0', 'tdtel043_1', 'tdtel062_0', 'tdtel062_1', 'tdtel063_0', 'tdtel063_1' }; +RU.PCIe40_names = { 'tdtel071_0', 'tdtel071_1', 'tdtel072_0', 'tdtel072_1', 'tdtel081_0', 'tdtel081_1', 'tdtel082_0', 'tdtel082_1', 'tdtel091_0', 'tdtel091_1', 'tdtel092_0', 'tdtel092_1', 'tdtel101_0', 'tdtel101_1', 'tdtel102_0', 'tdtel102_1' }; RU.PCIe40_timeout = 30; RU.buffer_sizes = {}; -RU.buffer_type = { 3, 3, 3, 3, 3, 3 }; +RU.buffer_type = { 3, 3, 3, 3, 3, 3, 3, 3 }; RU.n_MFPs = 100; RU.n_fragment = 3000; RU.shmem_prefix = "$SHMEM_PREFIX"; diff --git a/Online/EventBuilding/options/EB_Transport_properties.opts b/Online/EventBuilding/options/EB_Transport_properties.opts index d365afbf6..ed1cd38ac 100755 --- a/Online/EventBuilding/options/EB_Transport_properties.opts +++ b/Online/EventBuilding/options/EB_Transport_properties.opts @@ -2,7 +2,7 @@ EB_transport.BU_ranks = {}; EB_transport.MPI_errors_return = FALSE; EB_transport.RU_ranks = {}; EB_transport.RUs_per_nic = {}; -EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.FV2976.json"; +EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.qq3067.json"; EB_transport.n_par_mess = 0; -EB_transport.n_sources_per_ru = { 1, 1, 3, 3, 2, 2 }; +EB_transport.n_sources_per_ru = { 2, 2, 2, 2, 2, 2, 2, 2 }; EB_transport.src_ids = {}; diff --git a/Online/EventBuilding/src/RU.cpp b/Online/EventBuilding/src/RU.cpp index e436c23f2..98f1dedeb 100644 --- a/Online/EventBuilding/src/RU.cpp +++ b/Online/EventBuilding/src/RU.cpp @@ -189,7 +189,7 @@ int EB::RU::start() int EB::RU::stop() { int ret_val = DF_SUCCESS; - logger.info << "stop" << std::flush; + logger.error << "stop" << std::flush; // TODO if start never finishes this lock may never be release probably a timeout is needed std::unique_lock<std::timed_mutex> start_lock(_start_lock, std::chrono::duration<int>(_stop_timeout)); _send_MEPs = false; @@ -217,6 +217,7 @@ int EB::RU::stop() int EB::RU::cancel() { // TODO implement this + logger.error << " cancel triggered" << std::flush; return DF_SUCCESS; } diff --git a/Online/GaudiOnlineTests/python/GaudiOnlineTests/MDFtests.py b/Online/GaudiOnlineTests/python/GaudiOnlineTests/MDFtests.py index 077b63fcd..2363a6c97 100755 --- a/Online/GaudiOnlineTests/python/GaudiOnlineTests/MDFtests.py +++ b/Online/GaudiOnlineTests/python/GaudiOnlineTests/MDFtests.py @@ -156,7 +156,7 @@ def _createMDF(test_castor=None): app = ApplicationMgr() app.EvtSel = 'NONE' app.TopAlg = [Configs.Online__RawEventTestCreator('Creator')] - mdfCheck(dmp=False) + mdfCheck(dump=False) # To create a MDF file using the standard Gaudi output stream: wr = CFG.OutputStream('Writer_0') wr.Output = "DATAFILE='PFN:root:file://mdfData_0.dat' SVC='LHCb::RawDataCnvSvc' OPT='REC'" -- GitLab From ebc475039d01090d7d36f884b0a1587c5c191128 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Wed, 31 Mar 2021 12:00:55 +0200 Subject: [PATCH 13/26] Fixes for event builder debugging --- Online/EventBuilding/options/EB_BU_algo.opts | 2 +- Online/EventBuilding/options/EB_RU_algo.opts | 4 +- .../options/EB_Transport_properties.opts | 4 +- Online/FarmConfig/job/Controller.sh | 6 +++ Online/OnlineBase/src/RTL/Logger.cpp | 22 +++++++- Online/PcSrv/CMakeLists.txt | 5 +- Online/PcSrv/PcSrv/TaskCheck.h | 3 +- Online/PcSrv/src/TaskCheck.cpp | 46 +++++++++------- Online/ROLogger/kafka/OutputListener.cpp | 52 ++++++++++++++----- 9 files changed, 102 insertions(+), 42 deletions(-) diff --git a/Online/EventBuilding/options/EB_BU_algo.opts b/Online/EventBuilding/options/EB_BU_algo.opts index dff05b7be..5d9bea577 100755 --- a/Online/EventBuilding/options/EB_BU_algo.opts +++ b/Online/EventBuilding/options/EB_BU_algo.opts @@ -3,5 +3,5 @@ BU.buffer_size = {}; BU.buffer_type = { 2, 2, 2, 2, 2, 2, 2, 2 }; BU.out_file_prefix = ""; BU.shmem_prefix = ""; -BU.stop_timeout = 0; +BU.stop_timeout = 30; BU.write_to_file = { FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE }; diff --git a/Online/EventBuilding/options/EB_RU_algo.opts b/Online/EventBuilding/options/EB_RU_algo.opts index 05fb1ce0c..e7fd3dd19 100755 --- a/Online/EventBuilding/options/EB_RU_algo.opts +++ b/Online/EventBuilding/options/EB_RU_algo.opts @@ -1,10 +1,10 @@ RU.MDF_filename = "/home/fpisani/mc_data/biger_file.mdf"; RU.PCIe40_ids = {}; -RU.PCIe40_names = { 'tdtel071_0', 'tdtel071_1', 'tdtel072_0', 'tdtel072_1', 'tdtel081_0', 'tdtel081_1', 'tdtel082_0', 'tdtel082_1', 'tdtel091_0', 'tdtel091_1', 'tdtel092_0', 'tdtel092_1', 'tdtel101_0', 'tdtel101_1', 'tdtel102_0', 'tdtel102_1' }; +RU.PCIe40_names = { 'tdtel041_0', 'tdtel041_1', 'tdtel042_0', 'tdtel043_0', 'tdtel043_1', 'tdtel051_1', 'tdtel052_0', 'tdtel052_1', 'tdtel053_0', 'tdtel053_1', 'tdtel071_0', 'tdtel071_1', 'tdtel072_0', 'tdtel072_1', 'tdtel091_0', 'tdtel091_1', 'tdtel092_0', 'tdtel092_1' }; RU.PCIe40_timeout = 30; RU.buffer_sizes = {}; RU.buffer_type = { 3, 3, 3, 3, 3, 3, 3, 3 }; RU.n_MFPs = 100; RU.n_fragment = 3000; RU.shmem_prefix = "$SHMEM_PREFIX"; -RU.stop_timeout = 0; +RU.stop_timeout = 30; diff --git a/Online/EventBuilding/options/EB_Transport_properties.opts b/Online/EventBuilding/options/EB_Transport_properties.opts index ed1cd38ac..908c942e1 100755 --- a/Online/EventBuilding/options/EB_Transport_properties.opts +++ b/Online/EventBuilding/options/EB_Transport_properties.opts @@ -2,7 +2,7 @@ EB_transport.BU_ranks = {}; EB_transport.MPI_errors_return = FALSE; EB_transport.RU_ranks = {}; EB_transport.RUs_per_nic = {}; -EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.qq3067.json"; +EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.V23532.json"; EB_transport.n_par_mess = 0; -EB_transport.n_sources_per_ru = { 2, 2, 2, 2, 2, 2, 2, 2 }; +EB_transport.n_sources_per_ru = { 3, 2, 3, 2, 2, 2, 2, 2 }; EB_transport.src_ids = {}; diff --git a/Online/FarmConfig/job/Controller.sh b/Online/FarmConfig/job/Controller.sh index 548fde6fe..28305cd5b 100755 --- a/Online/FarmConfig/job/Controller.sh +++ b/Online/FarmConfig/job/Controller.sh @@ -20,6 +20,12 @@ echo "ERROR DIM_DNS_NODE: ${DIM_DNS_NODE}" # Check the existence of various arguments. # Otherwise use default values. # +if test -z "${LOGFIFO}"; then + export LOGFIFO=/run/fmc/logSrv.fifo; +fi; +if test "${LOGFIFO}" = "/run/fmc/logSrv.fifo" -a -e "/dev/shm/logs.dev"; then + export LOGFIFO=/dev/shm/logs.dev; +fi; if test -z "${TMS_DNS}"; then export TMS_DNS=${HOST_LONG}; fi; diff --git a/Online/OnlineBase/src/RTL/Logger.cpp b/Online/OnlineBase/src/RTL/Logger.cpp index 679c820d7..175b47352 100644 --- a/Online/OnlineBase/src/RTL/Logger.cpp +++ b/Online/OnlineBase/src/RTL/Logger.cpp @@ -131,30 +131,50 @@ pair<size_t,string> Logger::LogDevice::extract_format(const string& tag) { format = RTL::str_replace(format, tag, "s"); start = format.rfind('%',idx); for(end = idx + 1; format[end] && format[end]!='%';) ++end; - return make_pair(start,format.substr(start,end-start)); + return make_pair(idx,format.substr(start,end-start)); } return make_pair(string::npos,format); } /// Compile the format string void Logger::LogDevice::compileFormat(const string& fmt) { + string tmp; pair<size_t,string> item; if ( !fmt.empty() ) format = fmt; formatItems.clear(); + + // First order format statements according to their occurrence + tmp = format; while( (item=extract_format("TIME")).first != string::npos ) formatItems[item.first] = make_pair(TIME_FORMAT, item.second); + format = tmp; while( (item=extract_format("NODE")).first != string::npos ) formatItems[item.first] = make_pair(NODE_FORMAT, item.second); + format = tmp; while( (item=extract_format("LEVEL")).first != string::npos ) formatItems[item.first] = make_pair(LEVEL_FORMAT, item.second); + format = tmp; while( (item=extract_format("PROCESS")).first != string::npos ) formatItems[item.first] = make_pair(PROCESS_FORMAT, item.second); + format = tmp; while( (item=extract_format("UTGID")).first != string::npos ) formatItems[item.first] = make_pair(PROCESS_FORMAT, item.second); + format = tmp; while( (item=extract_format("SOURCE")).first != string::npos ) formatItems[item.first] = make_pair(SOURCE_FORMAT, item.second); + format = tmp; while( (item=extract_format("EXEC")).first != string::npos ) formatItems[item.first] = make_pair(EXEC_FORMAT, item.second); + + // Now reformat the formatting string + format = tmp; + while( (item=extract_format("TIME")).first != string::npos ); + while( (item=extract_format("NODE")).first != string::npos ); + while( (item=extract_format("LEVEL")).first != string::npos ); + while( (item=extract_format("PROCESS")).first != string::npos ); + while( (item=extract_format("UTGID")).first != string::npos ); + while( (item=extract_format("SOURCE")).first != string::npos ); + while( (item=extract_format("EXEC")).first != string::npos ); } /// Format header string according to precompiled format diff --git a/Online/PcSrv/CMakeLists.txt b/Online/PcSrv/CMakeLists.txt index 5e812a666..68c0524ee 100755 --- a/Online/PcSrv/CMakeLists.txt +++ b/Online/PcSrv/CMakeLists.txt @@ -19,6 +19,7 @@ gaudi_subdir(PcSrv v0r1) ################################################################################ gaudi_depends_on_subdirs(Online/dim Online/RPC + Online/HTTP Online/Parsers Online/OnlineBase) # @@ -38,9 +39,9 @@ include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) # --------------------------------------------------------------------------------------- gaudi_add_library(PcSrvLib src/*.cpp - INCLUDE_DIRS + INCLUDE_DIRS RPC HTTP PUBLIC_HEADERS PcSrv - LINK_LIBRARIES RPC Options Parsers Properties OnlineBase dim) + LINK_LIBRARIES RPC HTTP Options Parsers Properties OnlineBase dim) # Do not depend on ROOT target_compile_definitions(PcSrvLib PRIVATE -DDD4HEP_PARSERS_NO_ROOT=1) # Use boost::spirit for the properties diff --git a/Online/PcSrv/PcSrv/TaskCheck.h b/Online/PcSrv/PcSrv/TaskCheck.h index ac6dd3c06..d1101c823 100644 --- a/Online/PcSrv/PcSrv/TaskCheck.h +++ b/Online/PcSrv/PcSrv/TaskCheck.h @@ -89,6 +89,7 @@ namespace taskdb { std::string m_dns; std::string m_node; std::string m_tmName; + long m_tmDicDns = -1; int m_tmInfo = -1; bool m_inited = false; bool m_info_changed = false; @@ -112,7 +113,7 @@ namespace taskdb { public: /// Default constructor - TaskCheck(const std::string& dns, const std::string& node); + TaskCheck(const std::string& dns, const std::string& tms_dns, const std::string& node); /// Default destructor virtual ~TaskCheck(); /// Access the task container from the tmSrv service diff --git a/Online/PcSrv/src/TaskCheck.cpp b/Online/PcSrv/src/TaskCheck.cpp index 706363ff5..a2834bada 100644 --- a/Online/PcSrv/src/TaskCheck.cpp +++ b/Online/PcSrv/src/TaskCheck.cpp @@ -19,8 +19,9 @@ #include "CPP/IocSensor.h" #include "RPC/HttpRpcClient.h" #include "RPC/XMLRPC.h" -#include "RTL/rtl.h" +#include "RTL/Logger.h" #include "RTL/strdef.h" +#include "RTL/rtl.h" #include "dim/dic.h" #include "dim/dis.h" @@ -50,10 +51,11 @@ enum Commands { typedef void* pvoid; /// Default constructor -TaskCheck::TaskCheck(const string& dns, const string& node) { +TaskCheck::TaskCheck(const string& dns, const std::string& tms_dns, const string& node) { m_dns = dns; m_node = node; m_connectTime = system_clock::from_time_t(0); + m_tmDicDns = ::dic_add_dns(tms_dns.c_str(), ::dim_get_dns_port()); } /// Default destructor @@ -75,9 +77,10 @@ bool TaskCheck::setStrictMatching(bool new_value) { /// Initalization callback bool TaskCheck::initialize() { if ( !m_inited ) { - m_tmName = "/FMC/"+RTL::str_upper(m_node)+"/task_manager/"; - m_tmInfo = ::dic_info_service((m_tmName+"longList").c_str(), MONITORED, - 0, 0, 0, tmSrvInfoHandler, long(this), 0, 0); + m_tmName = "/FMC/"+RTL::str_upper(m_node)+"/task_manager/"; + m_tmInfo = ::dic_info_service_dns(m_tmDicDns, + (m_tmName+"longList").c_str(), MONITORED, + 0, 0, 0, tmSrvInfoHandler, long(this), 0, 0); TimeSensor::instance().add(this, 1, pvoid(TASKCHECK_LOAD_DB)); TimeSensor::instance().add(this, m_dbCheckUpdateTmo, pvoid(TASKCHECK_CHECK_DB)); TimeSensor::instance().add(this, 1, pvoid(TASKCHECK_CHECK_TM)); @@ -274,7 +277,7 @@ bool TaskCheck::start_task(const Task& task) { command = cmd.str(); for(const auto& r : m_replacements) command = RTL::str_replace(command,r.first,r.second); - int ret = ::dic_cmnd_service(svc.c_str(), (void*)command.c_str(), command.length()+1); + int ret = ::dic_cmnd_service_dns(m_tmDicDns, svc.c_str(), (void*)command.c_str(), command.length()+1); if ( ret == 1 ) { ::lib_rtl_output(LIB_RTL_INFO, "======= Successfully STARTED %s [%s].",task.utgid.c_str(),command.c_str()); } @@ -405,11 +408,11 @@ void TaskCheck::handleTimer(const Event& event) { } extern "C" int run_task_check(int argc, char** argv) { - string node, dns; - string port = "3500"; - string mount = "/TDBDATA/XMLRPC"; - string server; - string tm_dns; + string node, dns, server; + string port = "3500"; + string tm_dns = "ecstms01"; + string mount = "/TDBDATA/XMLRPC"; + string logger = ::getenv("LOGFIFO") ? ::getenv("LOGFIFO") : "/proc/self/1"; int print_level = LIB_RTL_INFO; RTL::CLI cli(argc, argv, [] (int, char**) { cout << " run_task_check -opt [-opt] \n" @@ -428,17 +431,21 @@ extern "C" int run_task_check(int argc, char** argv) { cli.getopt("server", 3, server); cli.getopt("mount", 3, mount); cli.getopt("tm_dns", 3, tm_dns); + cli.getopt("logger", 3, logger); cli.getopt("print", 3, print_level); - - TaskCheck chk(dns, node); - ::lib_rtl_set_log_level(print_level); - ::lib_rtl_install_printer(print_msg2, 0); - lib_rtl_output(LIB_RTL_ALWAYS,"-%s- Domain RPC service of type: TaskCheck started. DNS:%s TM:%s %s%s", - node.c_str(), dns.c_str(), tm_dns.c_str(), server.empty() ? "" : "Closest Server:", - server.empty() ? "" : server.c_str()); + { + RTL::Logger::log_args args(print_level); + RTL::Logger::install_rtl_printer(args.level); + RTL::Logger::getGlobalDevice()->compileFormat("%-36PROCESS %-8LEVEL %-20SOURCE"); + RTL::Logger::getGlobalDevice()->set_io_buffering(RTL::Logger::LINE_BUFFERING); + } + TaskCheck chk(dns, tm_dns, node); + ::lib_rtl_output(LIB_RTL_ALWAYS,"Domain RPC service of type: TaskCheck started. DNS:%s TM:%s %s%s", + node.c_str(), dns.c_str(), tm_dns.c_str(), server.empty() ? "" : "Closest Server:", + server.empty() ? "" : server.c_str()); if ( !server.empty() ) chk.servers.emplace_back(make_unique<HttpRpcClient>(server, mount, port)); - chk.servers.emplace_back(make_unique<HttpRpcClient>("slavetasks.service.consul.lhcb.cern.ch", mount, port)); chk.servers.emplace_back(make_unique<HttpRpcClient>("ecs03.lbdaq.cern.ch", mount, "3501")); + chk.servers.emplace_back(make_unique<HttpRpcClient>("slavetasks.service.consul.lhcb.cern.ch", mount, port)); chk.servers.emplace_back(make_unique<HttpRpcClient>("ecs03.lbdaq.cern.ch", mount, "3500")); chk.replaceString("<NODE>", node); chk.replaceString("<DIM_DNS>", dns); @@ -449,7 +456,6 @@ extern "C" int run_task_check(int argc, char** argv) { server = '/' + RTL::str_upper(RTL::nodeNameShort()) + '/' + RTL::processName(); ::dis_set_dns_node(dns.c_str()); ::dis_start_serving(server.c_str()); - //::dic_set_dns_node(tm_dns.c_str()); IocSensor::instance().run(); return 1; } diff --git a/Online/ROLogger/kafka/OutputListener.cpp b/Online/ROLogger/kafka/OutputListener.cpp index 9052b4007..d4b2b3612 100644 --- a/Online/ROLogger/kafka/OutputListener.cpp +++ b/Online/ROLogger/kafka/OutputListener.cpp @@ -138,6 +138,7 @@ void OutputListener::print_msg(const char* time_stamp, ::printf("\n"); } } +#include <unistd.h> /// Handler for all messages void OutputListener::handle_payload(const char* /* topic */, @@ -160,21 +161,39 @@ void OutputListener::handle_payload(const char* /* topic */, *(time_stamp+10) = ' '; *(time_stamp+19) = 0; char* message = mark_start(time_stamp+20, MESSAGE_TAG, m_tag_len); - char* end = mark_end(message); - size_t len = end-message-3; + char* msg_end = mark_end(message); + size_t len = msg_end-message-3; if ( message && len > plen ) { len = strlen(message); } - char* physical = mark_start(end, PHYSICAL_TAG, p_tag_len); - end = mark_end(physical); - char* host = mark_start(end, HOST_TAG, h_tag_len); - end = mark_end(host); - char* utgid = mark_start(end, UTGID_TAG, u_tag_len); - end = mark_end(utgid); - char* systag = mark_start(end, SYSTAG_TAG, s_tag_len); - end = mark_end(systag); - char* counter = mark_start(end, COUNTER_TAG, s_cnt_len); - end = mark_end(counter); + char* physical = mark_start(msg_end, PHYSICAL_TAG, p_tag_len); + char* phys_end = mark_end(physical); + char* host = mark_start(phys_end, HOST_TAG, h_tag_len); + char* host_end = mark_end(host); + char* utgid = mark_start(host_end, UTGID_TAG, u_tag_len); + char* utgid_end = mark_end(utgid); + char* systag = mark_start(utgid_end, SYSTAG_TAG, s_tag_len); + char* sys_end = mark_end(systag); + char* counter = mark_start(sys_end, COUNTER_TAG, s_cnt_len); + char* cnt_end = mark_end(counter); + +#if 0 + if ( message && message[0] == '{' ) { + bool wait = true; + ::printf("Havoc!!!!\n\n"); + while (wait) ::sleep(1); + } + if ( utgid && utgid[0] == '{' ) { + bool wait = true; + ::printf("Havoc!!!!\n\n"); + while (wait) ::sleep(1); + } + if ( systag && systag[0] == '{' ) { + bool wait = true; + ::printf("Havoc!!!!\n\n"); + while (wait) ::sleep(1); + } +#endif if ( !message ) { return; @@ -215,8 +234,15 @@ void OutputListener::handle_payload(const char* /* topic */, } if ( match_utgid && match_host ) { print_msg(time_stamp, message, systag, utgid, host, physical, counter); - return; } + if ( msg_end ) *msg_end = '\"'; + if ( phys_end ) *phys_end = '\"'; + if ( host_end ) *host_end = '\"'; + if ( utgid_end ) *utgid_end = '\"'; + if ( sys_end ) *sys_end = '\"'; + if ( cnt_end ) *cnt_end = '\"'; + if ( time_stamp ) *(time_stamp+10) = 'T'; + if ( time_stamp ) *(time_stamp+19) = ' '; } else { ::printf("%s %-8s '%s'\n", "PAYLOAD", "ERROR: Actual value is ", payload); -- GitLab From 1e5fdb5f3310556d873d697c8097aff5ceb8e4d6 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Thu, 8 Apr 2021 11:07:29 +0200 Subject: [PATCH 14/26] Improvement of fifo based output logging --- Online/Dataflow/src/Storage/StorageWriter.cpp | 3 +- Online/EventBuilding/options/EB_BU_algo.opts | 8 +- Online/EventBuilding/options/EB_RU_algo.opts | 6 +- .../options/EB_Transport_properties.opts | 4 +- .../EventBuilding/src/infiniband_net/sock.cpp | 19 +- Online/EventData/EventData/EventAccess.h | 8 +- Online/EventData/src/EventAccess.cpp | 27 +- Online/FarmConfig/job/EBPass.sh | 6 +- Online/OnlineBase/CMakeLists.txt | 4 +- Online/OnlineBase/OnlineBase/LOG/FifoLog.h | 6 +- Online/OnlineBase/src/LOG/FifoLog.cpp | 1002 ++++++++++++----- Online/ROLogger/kafka/OutputListener.cpp | 23 +- 12 files changed, 804 insertions(+), 312 deletions(-) diff --git a/Online/Dataflow/src/Storage/StorageWriter.cpp b/Online/Dataflow/src/Storage/StorageWriter.cpp index e5851dd8d..9a542480a 100644 --- a/Online/Dataflow/src/Storage/StorageWriter.cpp +++ b/Online/Dataflow/src/Storage/StorageWriter.cpp @@ -476,8 +476,7 @@ int StorageWriter::writeBuffer(const Buffer& buff) { error("writeBuffer: Exception while sending data: UNKNOWN Exception"); } ::lib_rtl_sleep(m_write_error_sleep); - process = (m_cancelled == 0); - process |= (::time(0) - m_cancelled < m_cancel_tmo); + process = (m_cancelled == 0) || (::time(0) - m_cancelled < m_cancel_tmo); process &= (--num_retries < 0); } break; diff --git a/Online/EventBuilding/options/EB_BU_algo.opts b/Online/EventBuilding/options/EB_BU_algo.opts index 5d9bea577..1fd2b0b01 100755 --- a/Online/EventBuilding/options/EB_BU_algo.opts +++ b/Online/EventBuilding/options/EB_BU_algo.opts @@ -1,7 +1,7 @@ -BU.MBM_name = { 'Input', 'Input', 'Input', 'Input', 'Input', 'Input', 'Input', 'Input' }; +BU.MBM_name = { 'Input', 'Input', 'Input', 'Input' }; BU.buffer_size = {}; -BU.buffer_type = { 2, 2, 2, 2, 2, 2, 2, 2 }; +BU.buffer_type = { 2, 2, 2, 2 }; BU.out_file_prefix = ""; BU.shmem_prefix = ""; -BU.stop_timeout = 30; -BU.write_to_file = { FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE }; +BU.stop_timeout = 60; +BU.write_to_file = { FALSE, FALSE, FALSE, FALSE }; diff --git a/Online/EventBuilding/options/EB_RU_algo.opts b/Online/EventBuilding/options/EB_RU_algo.opts index e7fd3dd19..2f9cdb1b7 100755 --- a/Online/EventBuilding/options/EB_RU_algo.opts +++ b/Online/EventBuilding/options/EB_RU_algo.opts @@ -1,10 +1,10 @@ RU.MDF_filename = "/home/fpisani/mc_data/biger_file.mdf"; RU.PCIe40_ids = {}; -RU.PCIe40_names = { 'tdtel041_0', 'tdtel041_1', 'tdtel042_0', 'tdtel043_0', 'tdtel043_1', 'tdtel051_1', 'tdtel052_0', 'tdtel052_1', 'tdtel053_0', 'tdtel053_1', 'tdtel071_0', 'tdtel071_1', 'tdtel072_0', 'tdtel072_1', 'tdtel091_0', 'tdtel091_1', 'tdtel092_0', 'tdtel092_1' }; +RU.PCIe40_names = { 'tdtel041_0', 'tdtel041_1', 'tdtel042_0', 'tdtel043_0', 'tdtel043_1', 'tdtel071_0', 'tdtel071_1', 'tdtel072_0', 'tdtel072_1' }; RU.PCIe40_timeout = 30; RU.buffer_sizes = {}; -RU.buffer_type = { 3, 3, 3, 3, 3, 3, 3, 3 }; +RU.buffer_type = { 3, 3, 3, 3 }; RU.n_MFPs = 100; RU.n_fragment = 3000; RU.shmem_prefix = "$SHMEM_PREFIX"; -RU.stop_timeout = 30; +RU.stop_timeout = 60; diff --git a/Online/EventBuilding/options/EB_Transport_properties.opts b/Online/EventBuilding/options/EB_Transport_properties.opts index 908c942e1..25b1f8781 100755 --- a/Online/EventBuilding/options/EB_Transport_properties.opts +++ b/Online/EventBuilding/options/EB_Transport_properties.opts @@ -2,7 +2,7 @@ EB_transport.BU_ranks = {}; EB_transport.MPI_errors_return = FALSE; EB_transport.RU_ranks = {}; EB_transport.RUs_per_nic = {}; -EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.V23532.json"; +EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.J31513.json"; EB_transport.n_par_mess = 0; -EB_transport.n_sources_per_ru = { 3, 2, 3, 2, 2, 2, 2, 2 }; +EB_transport.n_sources_per_ru = { 3, 2, 2, 2 }; EB_transport.src_ids = {}; diff --git a/Online/EventBuilding/src/infiniband_net/sock.cpp b/Online/EventBuilding/src/infiniband_net/sock.cpp index 716a59d23..6ca6c083c 100644 --- a/Online/EventBuilding/src/infiniband_net/sock.cpp +++ b/Online/EventBuilding/src/infiniband_net/sock.cpp @@ -19,6 +19,23 @@ void Sock::recvQPs(const int rankid, const std::vector<Proc>& k, std::atomic<int { vector<Proc>& hosts = const_cast<vector<Proc>&>(k); int connfd = socket(AF_INET, SOCK_STREAM, 0); + // Preventing the OS from keeping the socket open after close() + struct linger sl; + sl.l_onoff = 1; /* non-zero value enables linger option in kernel */ + sl.l_linger = 0; /* timeout interval in seconds */ + if(setsockopt(connfd, SOL_SOCKET, SO_LINGER, &sl, sizeof(sl)) < 0){ + ret = -20; + return ; + } + + + + int reuse_addr = 1; + if(setsockopt(connfd, SOL_SOCKET, SO_REUSEADDR, &reuse_addr, sizeof(reuse_addr)) < 0){ + ret = -21; + return ; + } + sockaddr_in server; server.sin_family = AF_INET; server.sin_port = hosts.at(rankid).sockAddr.sin_port; @@ -77,4 +94,4 @@ int Sock::sendQP(const int rankid, Proc& rem) send(connfd, (char*) &tmp_qp_info, sizeof(QPInfo), 0); close(connfd); return 0; -} \ No newline at end of file +} diff --git a/Online/EventData/EventData/EventAccess.h b/Online/EventData/EventData/EventAccess.h index 278d9df23..d2dfd04a6 100644 --- a/Online/EventData/EventData/EventAccess.h +++ b/Online/EventData/EventData/EventAccess.h @@ -151,6 +151,10 @@ namespace Online { std::size_t eventsOut = 0; /// Monitoring item: Count of events which are currently buffered std::size_t eventsBuffered = 0; + /// Monitoring item: Count the events with no data banks + std::size_t eventsNoBanks = 0; + /// Monitoring item: Count the events with more banks than allowed + std::size_t eventsMaxBanks = 0; } monitor; /// Property: poll timeout to check if producers have delivered an event [milliseconds] @@ -162,10 +166,10 @@ namespace Online { bank_collection_t convertMDF(datapointer_t start, std::size_t len) const; /// The event is a MDF with multiple events, which must be decoded - event_collection_t convertMultiMDF(datapointer_t start, std::size_t len) const; + event_collection_t convertMultiMDF(datapointer_t start, std::size_t len); /// The event is a PCIE40 MEP structure with multiple events, which must be decoded - std::unique_ptr<pcie40::event_collection_t> convertPCIE40MEP(datapointer_t start, std::size_t len) const; + std::unique_ptr<pcie40::event_collection_t> convertPCIE40MEP(datapointer_t start, std::size_t len); public: /// Default constructors and destructors / inhibits diff --git a/Online/EventData/src/EventAccess.cpp b/Online/EventData/src/EventAccess.cpp index c1c1eea2e..7f74ee4bc 100644 --- a/Online/EventData/src/EventAccess.cpp +++ b/Online/EventData/src/EventAccess.cpp @@ -239,7 +239,7 @@ namespace { /// The event is a PCIE40 MEP structure with multiple events, which must be decoded std::unique_ptr<pcie40::event_collection_t> -EventAccess::convertPCIE40MEP(datapointer_t start, size_t len) const { +EventAccess::convertPCIE40MEP(datapointer_t start, size_t len) { struct _printer { RTL::Logger& logger; _printer(RTL::Logger& e) : logger(e) {} @@ -303,10 +303,12 @@ EventAccess::convertPCIE40MEP(datapointer_t start, size_t len) const { } } if ( num_bank == 0 ) { - m_logger->error("Event with ZERO banks encountered"); + ++monitor.eventsNoBanks; + m_logger->info("Event with ZERO banks encountered"); } else if ( num_bank > max_bank ) { - m_logger->error("Event with TOO MANY banks encountered"); + ++monitor.eventsMaxBanks; + m_logger->info("Event with TOO MANY banks encountered"); } } return events; @@ -318,7 +320,7 @@ EventAccess::convertPCIE40MEP(datapointer_t start, size_t len) const { /// The event is a MDF with multiple events, which must be decoded EventAccess::event_collection_t -EventAccess::convertMultiMDF(datapointer_t start, size_t len) const { +EventAccess::convertMultiMDF(datapointer_t start, size_t len) { event_collection_t events; datapointer_t begin = start, end = start + len; if ( end > start ) { @@ -349,15 +351,20 @@ EventAccess::convertMultiMDF(datapointer_t start, size_t len) const { ++num_banks; } if ( num_banks == 0 ) { + ++monitor.eventsNoBanks; err_msg = "Event with ZERO banks encountered"; } else if ( num_banks > max_banks ) { + ++monitor.eventsMaxBanks; err_msg = "Event with TOO MANY banks encountered"; stop = true; } } if ( err_msg || err_bank ) { - m_logger->error(" SKIPPING EVENT:"); + m_logger->error("SKIPPING EVENT: Buffer:%p Start:%p End:%p Size:(%d,%d,%d) HdrVsn:%u %s", + (void*)begin, (void*)start, (void*)evt_end, header->size0(), + header->size1(), header->size2(), header->headerVersion(), + stop ? "---> STOP decoding" : ""); if ( err_msg ) { m_logger->error(" %s",err_msg); } @@ -365,15 +372,9 @@ EventAccess::convertMultiMDF(datapointer_t start, size_t len) const { m_logger->error(" Bank: %p -> %s", (void*)err_bank, Tell1Printout::bankHeader(err_bank).c_str()); } - m_logger->error(" Start: %p", (void*)start); - m_logger->error(" End: %p", (void*)evt_end); - m_logger->error(" Size0: %d", header->size0()); - m_logger->error(" Size1: %d", header->size1()); - m_logger->error(" Size2: %d", header->size2()); - m_logger->error(" Hdr.Vsn: %u", header->headerVersion()); if ( stop ) { - m_logger->error(" ---> STOP decoding after %ld out of %ld bytes [%ld events]", - start-begin, end-begin, events.size()); + //m_logger->error(" ---> STOP decoding after %ld out of %ld bytes [%ld events]", + // start-begin, end-begin, events.size()); return events; } } diff --git a/Online/FarmConfig/job/EBPass.sh b/Online/FarmConfig/job/EBPass.sh index f220525a2..a6be5bff6 100755 --- a/Online/FarmConfig/job/EBPass.sh +++ b/Online/FarmConfig/job/EBPass.sh @@ -47,9 +47,9 @@ application.config.numEventThreads = 1 application.config.MBM_numConnections = 7 application.config.MBM_numEventThreads = 6 application.config.MBM_requests = [ - 'EvType=3;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0', - 'EvType=2;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0', - 'EvType=1;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0' + 'EvType=3;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=20.0', + 'EvType=2;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=20.0', + 'EvType=1;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=20.0' ] /EOF # diff --git a/Online/OnlineBase/CMakeLists.txt b/Online/OnlineBase/CMakeLists.txt index 51692de1b..6e788c0bb 100755 --- a/Online/OnlineBase/CMakeLists.txt +++ b/Online/OnlineBase/CMakeLists.txt @@ -50,12 +50,12 @@ gaudi_add_library(FifoLog src/LOG/FifoLog_preload.cpp src/LOG/FifoLog.cpp NO_PUBLIC_HEADERS - LINK_LIBRARIES -lrt) + LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt) # #=============================================================================== gaudi_add_python_module(_fifo_log src/LOG/FifoLog_module.cpp - LINK_LIBRARIES OnlineBase) + LINK_LIBRARIES OnlineBase ${CMAKE_DL_LIBS} -lrt) # # Need to cast out warnings created by Python.h target_compile_options(_fifo_log PRIVATE -Wno-register) diff --git a/Online/OnlineBase/OnlineBase/LOG/FifoLog.h b/Online/OnlineBase/OnlineBase/LOG/FifoLog.h index 76e9fcb08..f44809669 100644 --- a/Online/OnlineBase/OnlineBase/LOG/FifoLog.h +++ b/Online/OnlineBase/OnlineBase/LOG/FifoLog.h @@ -22,7 +22,7 @@ #include <memory> /// FIFLOLOG namespace declaration -namespace fiflolog { +namespace fifolog { /// Logger class capturing stdout and stderr. /** @@ -39,7 +39,7 @@ namespace fiflolog { /// Initializing constructor Logger(const std::string& fifo, const std::string& utgid, const std::string& tag); /// Default destructor - ~Logger() = default; + ~Logger(); /// Run the logger in a separate thread void run(); /// Stop processing and shutdown @@ -51,7 +51,7 @@ extern "C" { /// Initialize the logger unit void fifolog_initialize_logger(); /// Set fifo value (Only possible BEFORE startup (return=1), otherwise returns NULL) - int fifolog_set_fifo(const char* fifo); + int fifolog_set_fifo(const char* fifo); /// Set new utgid value void fifolog_set_utgid(const char* new_utgid); /// Set new tag value diff --git a/Online/OnlineBase/src/LOG/FifoLog.cpp b/Online/OnlineBase/src/LOG/FifoLog.cpp index b58b7279c..719c4c082 100644 --- a/Online/OnlineBase/src/LOG/FifoLog.cpp +++ b/Online/OnlineBase/src/LOG/FifoLog.cpp @@ -15,71 +15,259 @@ // //========================================================================== -#include "LOG/FifoLog.h" +#include "LOG/FifoLog.h" +#include <cstdarg> +#include <atomic> #include <thread> -#include <memory> #include <mutex> -#include <atomic> -#include <cerrno> -#include <cstdio> -#include <cstdlib> -#include <cstring> -#include <string> -#include <sstream> -#include <iostream> -#include <stdexcept> - -#include <unistd.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/epoll.h> -#include <fcntl.h> -using namespace fiflolog; -using namespace std; +#define LOGGER_VA_LIST va_list -class Logger::Implementation { +class fifolog::Logger::Implementation { public: - typedef std::unique_ptr<std::thread> thread_t; + enum { STDOUT = 0, STDERR = 1 }; + static std::string utgid; static std::string systag; static std::string fifo_name; - std::string real_host; - std::string physical_host; - std::string stdout_buffer; - std::string stderr_buffer; - std::string output_trailer; + std::string real_host; + std::string physical_host; + std::string output_trailer; + std::string output[2]; + std::mutex output_mutex; std::atomic<unsigned long> msg_count; - std::mutex rebuild_lock; - thread_t runner; - int fifo_fd = -1; - int epoll_fd = -1; - int stdout_fd[2] = {-1, -1}; - int stderr_fd[2] = {-1, -1}; - bool done = false; - Implementation() = default; - ~Implementation() = default; + int fifo_fd = -1; + + /// Dump data buffer to file descriptor + void dump_buffer(int fd, const char* buf, int len); + + /// Build output record and dump record to output device + void flush_line(int stream_id); + + /// Append (unlocked) single character to output buffer + void append(int stream_id, char c) { + switch(c) { + case '\n': + flush_line(stream_id); + break; + default: + output[stream_id].push_back(c); + break; + } + } + + /// Append (locked) single character to output buffer + int push(int stream_id, char c) { + std::lock_guard<std::mutex> lock(output_mutex); + append(stream_id,c); + return c; + } + + /// Append (locked) opaque buffer to output buffer + int push(int stream_id, const void* text, size_t len) { + const char* p = (const char*)text; + std::lock_guard<std::mutex> lock(output_mutex); + for(const char* end=p+len; p<end; ++p) + this->append(stream_id, *p); + return len; + } + + /// Append (locked) string to output buffer + int push(int stream_id, const char* text) { + const char* p = text; + std::lock_guard<std::mutex> lock(output_mutex); + for(; *p; ++p) + this->append(stream_id, *p); + return p-text; + } + + /// Append (locked) formatted string to output buffer + int push(int stream_id, const char* format, LOGGER_VA_LIST args) { + char text[4096]; + int len = ::vsnprintf(text, sizeof(text), format, args); + text[sizeof(text)-1] = 0; + va_end(args); + this->push(stream_id, text); + return len; + } + + /// Append (locked) formatted string to output buffer + int push(int stream_id, const wchar_t* format, LOGGER_VA_LIST args) { + wchar_t text[4096]; + int len = ::vswprintf(text, sizeof(text), format, args); + text[sizeof(text)-1] = 0; + va_end(args); + this->push(stream_id, text, len*sizeof(wchar_t)); + return len; + } + +public: + + /// Default constructor + Implementation(); + + /// Default destructor + virtual ~Implementation(); + /// Object initialization - void init(const string& fifo, const string& utgid, const string& tag); + virtual void init(const std::string& fifo, const std::string& utgid, const std::string& tag); + /// Rebuild trailer - void rebuild_trailer(); - /// Read single data frame - void read_data(int fd, string& data); - /// Poll data from stdout/stderr pipes - void poll(); + virtual void rebuild_trailer(); + /// Stop processing and shutdown - void stop(); - /// Flush pending output buffers - void flush_buffer(string& data); + virtual void stop(); + + /// Run the instance + virtual void run(); }; -std::string Logger::Implementation::fifo_name; -std::string Logger::Implementation::systag; -std::string Logger::Implementation::utgid; +namespace fifolog { + + /// Class to capture all libc output routines to overload stdout/stderr + /** + * \author M.Frank + * \version 1.0 + */ + class FuncPtr { + public: + typedef ssize_t (*write_t) (int fd, const void *buf, size_t count); + typedef size_t (*putchar_t) (int c); + typedef int (*putc_t) (int c, FILE* stream); + typedef int (*puts_t) (const char *format); + + typedef size_t (*fputc_t) (int c, FILE *stream); + typedef size_t (*fputs_t) (const char *ptr, FILE *stream); + typedef size_t (*fwrite_t) (const void *ptr, size_t size, size_t nmemb, FILE *stream); + + typedef int (*printf_t) (const char *format, ...); + typedef int (*vprintf_t) (const char *format, LOGGER_VA_LIST args); + + typedef int (*wprintf_t) (const wchar_t *format, ...); + typedef int (*vwprintf_t) (const wchar_t *format, LOGGER_VA_LIST args); + + typedef int (*dprintf_t) (int fd, const char *format, ...); + typedef int (*vdprintf_t) (int fd, const char *format, LOGGER_VA_LIST args); + + typedef int (*fprintf_t) (FILE* stream, const char *format, ...); + typedef int (*vfprintf_t) (FILE* stream, const char *format, LOGGER_VA_LIST args); + typedef int (*vfwprintf_t)(FILE* stream, const wchar_t *format, LOGGER_VA_LIST args); + + typedef int (*printf_chk_t) (int flag, const char *format, ...); + typedef int (*vprintf_chk_t)(int flag, const char *format, LOGGER_VA_LIST args); + + typedef int (*fprintf_chk_t) (FILE *stream, int flag, const char *format, ...); + typedef int (*vfprintf_chk_t)(FILE *stream, int flag, const char *format, LOGGER_VA_LIST args); + + + public: + write_t real_write { nullptr }; + putchar_t real_putchar { nullptr }; + putc_t real_putc { nullptr }; + puts_t real_puts { nullptr }; + + fputc_t real_fputc { nullptr }; + fputs_t real_fputs { nullptr }; + fwrite_t real_fwrite { nullptr }; + + printf_t real_printf { nullptr }; + vprintf_t real_vprintf { nullptr }; + + wprintf_t real_wprintf { nullptr }; + vwprintf_t real_vwprintf { nullptr }; + + dprintf_t real_dprintf { nullptr }; + vdprintf_t real_vdprintf { nullptr }; + + fprintf_t real_fprintf { nullptr }; + vfprintf_t real_vfprintf { nullptr }; + vfwprintf_t real_vfwprintf{ nullptr }; + + printf_chk_t real_printf_chk { nullptr }; + vprintf_chk_t real_vprintf_chk { nullptr }; + fprintf_chk_t real_fprintf_chk { nullptr }; + vfprintf_chk_t real_vfprintf_chk { nullptr }; + + public: + /// Default constructor + FuncPtr(int); + /// Default destructor + ~FuncPtr() = default; + }; + + /// Logger class uniquely overloading output routines from libc + /** + * \author M.Frank + * \version 1.0 + */ + class LibCLogger : public fifolog::Logger::Implementation { + public: + /// Object initialization + virtual void init(const std::string& fifo, const std::string& utgid, const std::string& tag) override; + + /// Default constructor + LibCLogger(); + + /// Default destructor + virtual ~LibCLogger(); + }; + + /// Logger class overloading output routines from libc and polling on stdout/stderr + /** + * \author M.Frank + * \version 1.0 + */ + class PollLogger : public fifolog::Logger::Implementation { + protected: + typedef std::unique_ptr<std::thread> thread_t; + int epoll_fd = -1; + int std_fd[2] = {-1, -1}; + thread_t runner; + bool done = false; + + public: + /// Object initialization + virtual void init(const std::string& fifo, const std::string& utgid, const std::string& tag) override; + /// Poll data from stdout/stderr pipes + void poll(); + + /// Default constructor + PollLogger(); + + /// Default destructor + virtual ~PollLogger(); + /// Run the logger in a separate thread + void run() override; + /// Stop processing and shutdown + void stop() override; + }; +} + +#include <stdexcept> +#include <iostream> +#include <sstream> +#include <cstring> +#include <cstdarg> +#include <cstdio> + +#include <dlfcn.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/epoll.h> + +using namespace std; +using namespace fifolog; +typedef Logger::Implementation logger_imp; + +string logger_imp::fifo_name; +string logger_imp::systag; +string logger_imp::utgid; /// Anonymous namespace for helpers -namespace { +namespace { /// Translate errno into error string string error(int err) { @@ -88,12 +276,31 @@ namespace { return e; } - std::unique_ptr<Logger>& logger(bool create=true) { - static std::unique_ptr<Logger> log; - if ( create && !log.get() ) { - string tag = Logger::Implementation::systag; - string utgid = Logger::Implementation::utgid; - string fifo = Logger::Implementation::fifo_name; + template <typename FUNC_TYPE> FUNC_TYPE get_sym(const char* name) { + FUNC_TYPE fun = FUNC_TYPE( ::dlsym(RTLD_NEXT, name) ); + if ( nullptr == fun ) { + throw runtime_error("Failed to initialize logger function "+string(name)); + } + return fun; + } + + inline logger_imp& log() { + //static LibCLogger s_log; + static PollLogger s_log; + return s_log; + } + + inline FuncPtr& funcs() { + static FuncPtr s_funcs(0); + return s_funcs; + } + + unique_ptr<Logger>& logger(bool create=true) { + static unique_ptr<Logger> s_log; + if ( create && !s_log.get() ) { + string tag = logger_imp::systag; + string utgid = logger_imp::utgid; + string fifo = logger_imp::fifo_name; if ( utgid.empty() ) { utgid = ::getenv("UTGID") ? ::getenv("UTGID") : ""; } @@ -118,124 +325,119 @@ namespace { bool run = false; while(!run) ::sleep(1); } - log.reset(new Logger(fifo, utgid, tag)); + s_log.reset(new Logger(fifo, utgid, tag)); } - return log; + return s_log; } } +/// Default constructor +FuncPtr::FuncPtr(int) { +#define SYM(x) if ( FuncPtr::real_##x == nullptr ) \ + FuncPtr::real_##x = get_sym< FuncPtr::x##_t > ( #x ) + SYM(write); + SYM(putchar); + SYM(putc); + SYM(puts); + + SYM(fputc); + SYM(fputs); + SYM(fwrite); + + SYM(printf); + SYM(vprintf); + + SYM(wprintf); + SYM(vwprintf); + + SYM(dprintf); + SYM(vdprintf); + + SYM(fprintf); + SYM(vfprintf); + SYM(vfwprintf); +#undef SYM +#define SYM(x) if ( FuncPtr::real_##x == nullptr ) \ + FuncPtr::real_##x = get_sym< FuncPtr::x##_t > ( "__" #x ) + SYM(printf_chk); + SYM(vprintf_chk); + SYM(fprintf_chk); + SYM(vfprintf_chk); +#undef SYM +} + +/// Default constructor +logger_imp::Implementation() { + output[0].reserve(4096); + output[1].reserve(4096); + funcs(); +} + +/// Default destructor +logger_imp::~Implementation() { +} + /// Object initialization -void Logger::Implementation::init(const string& f, const string& u, const string& t) { +void logger_imp::init(const string& f, const string& u, const string& t) { char text[1024]; - int fd, ret = 0; struct stat statBuf; - string& fifo = Logger::Implementation::fifo_name; - string& utgid = Logger::Implementation::utgid; - string& tag = Logger::Implementation::systag; - - if ( !f.empty() ) fifo = f; - if ( !u.empty() ) utgid = u; - if ( !t.empty() ) tag = t; - this->msg_count = 0; - this->epoll_fd = ::epoll_create(10); - - // check if fifo is writable - if( ::access(fifo.c_str(),W_OK) == -1) { // write access to fifo OK - ::snprintf(text,sizeof(text),"+++ Failed to access output: %s [%s]\n", - fifo.c_str(), error(errno).c_str()); - throw runtime_error(text); - } - // get fifo information - if( ::stat(fifo.c_str(),&statBuf) == -1) { // fifo information got - ::snprintf(text,sizeof(text),"+++ Output: Canno stat device %s! [%s]\n", - fifo.c_str(), error(errno).c_str()); - throw runtime_error(text); - } - // check if fifo is a FIFO - if( !S_ISFIFO(statBuf.st_mode)) { // fifo is a FIFO - ::snprintf(text,sizeof(text),"+++ Output: %s is no fifo!\n", fifo.c_str()); - throw runtime_error(text); - } - // open fifo - this->fifo_fd = ::open(fifo.c_str(), O_RDWR|O_NONBLOCK|O_APPEND); - if( this->fifo_fd == -1 ) { /* fifo open() failed */ - ::snprintf(text,sizeof(text),"+++ Failed to open output: %s! [%s]\n", - fifo.c_str(), error(errno).c_str()); - throw runtime_error(text); - } - /* Now we are sure that another process has the FIFO open for reading. */ - /* We unset now the O_NONBLOCK bit to have blocking write (no-drop */ - /* behaviour). */ - int status = ::fcntl(this->fifo_fd, F_GETFL); - if ( status < 0 ) { - ::close(this->fifo_fd); - ::snprintf(text,sizeof(text),"+++ Cannot fcntl fifo path %s. [%s]", - fifo.c_str(), error(errno).c_str()); - throw runtime_error(text); - } - status &= ~O_NONBLOCK; /* unset O_NONBLOCK bit */ - if ( ::fcntl(this->fifo_fd, F_SETFL,status) == -1 ) { - ::close(this->fifo_fd); - ::snprintf(text,sizeof(text),"+++ Cannot set O_NONBLOCK bit of fifo %s. [%s]", - fifo.c_str(), error(errno).c_str()); - throw runtime_error(text); - } - /// - ret = ::pipe(this->stdout_fd); - if ( ret != 0 ) { - ::snprintf(text,sizeof(text),"+++ Failed to create stdout pipe for fifo: %s. [%s]\n", - fifo.c_str(), error(errno).c_str()); - ::close(this->fifo_fd); - throw runtime_error(text); - } - fd = ::fileno(stdout); - ret = ::dup2(this->stdout_fd[1],fd); - if ( ret == -1 ) { - ::snprintf(text,sizeof(text),"+++ Failed to dup stdout for fifo: %s. [%s]\n", - fifo.c_str(), error(errno).c_str()); - ::close(this->fifo_fd); - throw runtime_error(text); - } - /// - ret = ::pipe(this->stderr_fd); - if ( ret != 0 ) { - ::snprintf(text,sizeof(text),"+++ Failed to create stderr pipe for fifo: %s. [%s]\n", - fifo.c_str(), error(errno).c_str()); - ::close(this->fifo_fd); - throw runtime_error(text); - } - fd = ::fileno(stderr); - ret = ::dup2(this->stderr_fd[1],fd); - if ( ret == -1 ) { - ::snprintf(text,sizeof(text),"+++ Failed to dup stderr for fifo: %s. [%s]\n", - fifo.c_str(), error(errno).c_str()); - ::close(this->fifo_fd); - throw runtime_error(text); + if ( this->fifo_fd < 0 ) { + string& fifo = Logger::Implementation::fifo_name; + if ( !f.empty() ) fifo = f; + + if ( !fifo.empty() ) { + string& utgid = Logger::Implementation::utgid; + string& tag = Logger::Implementation::systag; + + if ( !u.empty() ) utgid = u; + if ( !t.empty() ) tag = t; + + this->msg_count = 0; + + // check if fifo is writable + if( ::access(fifo.c_str(),W_OK) == -1) { // write access to fifo OK + ::snprintf(text,sizeof(text),"+++ Failed to access output: %s [%s]\n", + fifo.c_str(), error(errno).c_str()); + throw runtime_error(text); + } + // get fifo information + if( ::stat(fifo.c_str(),&statBuf) == -1) { // fifo information got + ::snprintf(text,sizeof(text),"+++ Output: Canno stat device %s! [%s]\n", + fifo.c_str(), error(errno).c_str()); + throw runtime_error(text); + } + // check if fifo is a FIFO + if( !S_ISFIFO(statBuf.st_mode)) { // fifo is a FIFO + ::snprintf(text,sizeof(text),"+++ Output: %s is no fifo!\n", fifo.c_str()); + throw runtime_error(text); + } + // open fifo + this->fifo_fd = ::open(fifo.c_str(), O_RDWR|O_NONBLOCK|O_APPEND); + if( this->fifo_fd == -1 ) { /* fifo open() failed */ + ::snprintf(text,sizeof(text),"+++ Failed to open output: %s! [%s]\n", + fifo.c_str(), error(errno).c_str()); + throw runtime_error(text); + } + // Now we are sure that another process has the FIFO open for reading. + // We unset now the O_NONBLOCK bit to have blocking write (no-drop behaviour). + int status = ::fcntl(this->fifo_fd, F_GETFL); + if ( status < 0 ) { + ::close(this->fifo_fd); + ::snprintf(text,sizeof(text),"+++ Cannot fcntl fifo path %s. [%s]", + fifo.c_str(), error(errno).c_str()); + throw runtime_error(text); + } + status &= ~O_NONBLOCK; /* unset O_NONBLOCK bit */ + if ( ::fcntl(this->fifo_fd, F_SETFL,status) == -1 ) { + ::close(this->fifo_fd); + ::snprintf(text,sizeof(text),"+++ Cannot set O_NONBLOCK bit of fifo %s. [%s]", + fifo.c_str(), error(errno).c_str()); + throw runtime_error(text); + } + } } /// - struct epoll_event ev; - ev.events = EPOLLIN | EPOLLRDHUP | EPOLLERR | EPOLLHUP | EPOLLET; - /// - ev.data.fd = this->stdout_fd[0]; - ::fcntl(this->stdout_fd[0], F_SETFL, O_NONBLOCK); - ret = ::epoll_ctl(this->epoll_fd, EPOLL_CTL_ADD, this->stdout_fd[0], &ev); - if ( ret != 0 ) { - ::snprintf(text,sizeof(text),"+++ Failed to add FD %d to epoll structure for fifo: %s. [%s]\n", - this->stdout_fd[0], fifo.c_str(), error(errno).c_str()); - ::close(this->fifo_fd); - throw runtime_error(text); - } - ev.data.fd = this->stderr_fd[0]; - ::fcntl(this->stderr_fd[0], F_SETFL, O_NONBLOCK); - ret = ::epoll_ctl(this->epoll_fd, EPOLL_CTL_ADD, this->stderr_fd[0], &ev); - if ( ret != 0 ) { - ::snprintf(text,sizeof(text),"+++ Failed to add FD %d to epoll structure for fifo: %s. [%s]\n", - this->stdout_fd[0], fifo.c_str(), error(errno).c_str()); - ::close(this->epoll_fd); - ::close(this->fifo_fd); - throw runtime_error(text); - } char host[128]; ::gethostname(host,sizeof(host)); this->physical_host = host; @@ -250,148 +452,262 @@ void Logger::Implementation::init(const string& f, const string& u, const string this->rebuild_trailer(); } +/// Run the instance +void logger_imp::run() { +} + +/// Stop processing and shutdown +void logger_imp::stop() { + ::setvbuf(stdout, nullptr, _IONBF, 0); + ::setvbuf(stderr, nullptr, _IONBF, 0); + this->flush_line(STDOUT); + this->flush_line(STDERR); + if ( this->fifo_fd >= 0 ) { + ::syncfs(this->fifo_fd); + ::close(this->fifo_fd); + } + this->fifo_fd = 0; +} + /// Rebuild trailer -void Logger::Implementation::rebuild_trailer() { - std::stringstream str; - std::lock_guard<std::mutex> lock(this->rebuild_lock); - str << ::getpid(); - this->output_trailer = "\",\"physical_host\":\""; - this->output_trailer += this->physical_host; - this->output_trailer += "\",\"host\":\""; - this->output_trailer += this->real_host; - this->output_trailer += "\",\"pid\":\""; - this->output_trailer += str.str(); - this->output_trailer += "\",\"utgid\":\""; - this->output_trailer += Logger::Implementation::utgid; - this->output_trailer += "\",\"systag\":\""; - this->output_trailer += Logger::Implementation::systag; - this->output_trailer += "\"}\n"; -} - -void Logger::Implementation::flush_buffer(string& data) { - if ( !data.empty() ) { - struct tm tim; - char time_str[256]; - time_t now = ::time(0); - struct timespec delay = {0,1000000}; /* 0.001 s */ - string output_buffer; - ::localtime_r(&now, &tim); - ::strftime(time_str,sizeof(time_str),"{\"@timestamp\":\"%Y-%m-%dT%H:%M:%S.000000%z\",\"message\":\"",&tim); - output_buffer.reserve(512); - output_buffer = time_str; - output_buffer += data; - output_buffer += this->output_trailer; - ++this->msg_count; - - int tryC = 0, numRetry = 100; - int len = output_buffer.length(); - for ( tryC=0; tryC < numRetry; tryC++ ) { - int written = write(this->fifo_fd, output_buffer.c_str(), len); - if ( written > 0 ) - len -= written; - else if ( written == -1 ) - break; - else if ( errno != EAGAIN ) - break; - - if ( 0 == len ) break; - nanosleep(&delay,NULL); - delay.tv_sec *= 2; - delay.tv_nsec *= 2; - if(delay.tv_nsec>999999999){ - delay.tv_sec+=1; - delay.tv_nsec-=1000000000; - } +void logger_imp::rebuild_trailer() { + stringstream str; + str << "\",\"physical_host\":\"" << this->physical_host + << "\",\"host\":\"" << this->real_host + << "\",\"pid\":\"" << ::getpid() + << "\",\"utgid\":\"" << logger_imp::utgid + << "\",\"systag\":\"" << logger_imp::systag + << "\"}\n"; + { + lock_guard<mutex> lock(this->output_mutex); + this->output_trailer = str.str(); + } +} + +/// Dump data buffer to file descriptor +void logger_imp::dump_buffer(int fd, const char* buf, int len) { + struct timespec delay = {0,1000000}; /* 0.001 s */ + int tryC = 0, numRetry = 100, done=0; + for ( tryC=0; tryC < numRetry; tryC++ ) { + int written = funcs().real_write(fd, buf+done, len-done); + if ( written > 0 ) + done += written; + else if ( written == -1 ) + break; + else if ( errno != EAGAIN ) + break; + + if ( done == len ) break; + nanosleep(&delay,NULL); + delay.tv_sec *= 2; + delay.tv_nsec *= 2; + if(delay.tv_nsec>999999999){ + delay.tv_sec+=1; + delay.tv_nsec-=1000000000; } } } -/// Read single data frame -void Logger::Implementation::read_data(int fd, string& data) { - char text[512]; - ssize_t nbyt = ::read(fd, text, sizeof(text)-2); - if ( nbyt <= 0 ) { +/// Build output record and dump record to output device +void logger_imp::flush_line(int stream_id) { + string& data = output[stream_id]; + if ( this->fifo_fd >= 0 ) { + if ( !data.empty() ) { + struct tm tim; + char time_str[256]; + time_t now = ::time(0); + string output_buffer; + ::localtime_r(&now, &tim); + ::strftime(time_str,sizeof(time_str),"{\"@timestamp\":\"%Y-%m-%dT%H:%M:%S.000000%z\",\"message\":\"",&tim); + output_buffer.reserve(200+data.length()); + output_buffer = time_str; + output_buffer += data; + output_buffer += this->output_trailer; + data = ""; + ++this->msg_count; + dump_buffer(this->fifo_fd, output_buffer.c_str(), output_buffer.length()); + } return; } - text[nbyt] = 0; - for(char* p = text; *p; ++p) { - switch(*p) { - case '\0': - case '\n': { - if ( !data.empty() ) { - flush_buffer(data); - data = ""; - data.reserve(512); - } - break; + data += '\n'; + if ( stream_id == STDOUT ) { + dump_buffer(STDOUT_FILENO, data.c_str(), data.length()); + data = ""; + return; + } + dump_buffer(STDERR_FILENO, data.c_str(), data.length()); + data = ""; +} + +/// Default constructor +LibCLogger::LibCLogger() { +} + +/// Default destructor +LibCLogger::~LibCLogger() { +} + +/// Object initialization +void LibCLogger::init(const string& f, const string& u, const string& t) { + if ( this->fifo_fd < 0 ) { + this->logger_imp::init(f, u, t); + char text[1024]; + string& fifo = Logger::Implementation::fifo_name; + int fd = ::fileno(stdout); + int ret = ::dup2(this->fifo_fd, fd); + if ( ret == -1 ) { + ::snprintf(text,sizeof(text),"+++ Failed to dup stdout for fifo: %s. [%s]\n", + fifo.c_str(), error(errno).c_str()); + ::close(this->fifo_fd); + throw runtime_error(text); } - default: - data += *p; - break; + fd = ::fileno(stderr); + ret = ::dup2(this->fifo_fd, fd); + if ( ret == -1 ) { + ::snprintf(text,sizeof(text),"+++ Failed to dup stderr for fifo: %s. [%s]\n", + fifo.c_str(), error(errno).c_str()); + ::close(this->fifo_fd); + throw runtime_error(text); + } + this->push(STDOUT,"LIBC logger initialized!!!\n"); + } +} + +/// Default constructor +PollLogger::PollLogger() { +} + +/// Default destructor +PollLogger::~PollLogger() { + this->done = true; + if ( this->runner.get() ) { + try { + this->runner->join(); + this->runner.reset(); + } + catch(...) { + } + } +} + +/// Object initialization +void PollLogger::init(const string& f, const string& u, const string& t) { + if ( this->fifo_fd < 0 ) { + this->logger_imp::init(f, u, t); + char text[1024]; + string& fifo = Logger::Implementation::fifo_name; + int fd, ret; + /// + ret = ::pipe(this->std_fd); + if ( ret != 0 ) { + ::snprintf(text,sizeof(text),"+++ Failed to create stdout pipe for fifo: %s. [%s]\n", + fifo.c_str(), error(errno).c_str()); + ::close(this->fifo_fd); + this->fifo_fd = -1; + throw runtime_error(text); + } + fd = ::fileno(stdout); + ret = ::dup2(this->std_fd[1],fd); + if ( ret == -1 ) { + ::snprintf(text,sizeof(text),"+++ Failed to dup stdout for fifo: %s. [%s]\n", + fifo.c_str(), error(errno).c_str()); + ::close(this->fifo_fd); + this->fifo_fd = -1; + throw runtime_error(text); + } + fd = ::fileno(stderr); + ret = ::dup2(this->std_fd[1],fd); + if ( ret == -1 ) { + ::snprintf(text,sizeof(text),"+++ Failed to dup stderr for fifo: %s. [%s]\n", + fifo.c_str(), error(errno).c_str()); + ::close(this->fifo_fd); + this->fifo_fd = -1; + throw runtime_error(text); } + /// + struct epoll_event ev; + this->epoll_fd = ::epoll_create(10); + ev.events = EPOLLIN | EPOLLRDHUP | EPOLLERR | EPOLLHUP | EPOLLET; + /// + ev.data.fd = this->std_fd[0]; + ::fcntl(this->std_fd[0], F_SETFL, O_NONBLOCK); + ret = ::epoll_ctl(this->epoll_fd, EPOLL_CTL_ADD, this->std_fd[0], &ev); + if ( ret != 0 ) { + ::snprintf(text,sizeof(text),"+++ Failed to add FD %d to epoll structure for fifo: %s. [%s]\n", + this->std_fd[0], fifo.c_str(), error(errno).c_str()); + ::close(this->epoll_fd); + ::close(this->fifo_fd); + throw runtime_error(text); + } + //this->push(STDOUT,"POLL logger initialized!!!\n"); } } +/// Run the logger in a separate thread +void PollLogger::run() { + if ( !this->runner.get() ) { + this->runner = make_unique<thread>([this]() { this->poll(); }); + } +} + +/// Stop processing and shutdown +void PollLogger::stop() { + this->done = true; + this->runner->join(); + this->Logger::Implementation::stop(); +} + /// Poll data from stdout/stderr pipes -void Logger::Implementation::poll() { +void PollLogger::poll() { + int count = 0; + // this->push(STDOUT, "POLL thread is starting now....\n"); while ( 1 ) { struct epoll_event ep; int nc = ::epoll_wait(this->epoll_fd, &ep, 1, 100); if ( nc > 0 ) { - for( int i=0; i < nc; ++i ) { - if ( ep.events&EPOLLIN ) { - if ( ep.data.fd == this->stdout_fd[0] ) - this->read_data(ep.data.fd, this->stdout_buffer); - else if ( ep.data.fd == this->stderr_fd[0] ) - this->read_data(ep.data.fd, this->stderr_buffer); - else - this->read_data(ep.data.fd, this->stdout_buffer); + if ( ep.events&EPOLLIN ) { + int fd = ep.data.fd; + char text[512]; + ssize_t nbyt = ::read(fd, text, sizeof(text)-2); + if ( nbyt <= 0 ) { + return; } + text[nbyt] = 0; + this->push(STDOUT, text, nbyt); } } - if ( this->done ) { + if ( count > 5 ) break; - } + else if ( this->done ) + ++count; } -} - -/// Stop processing and shutdown -void Logger::Implementation::stop() { - ::setvbuf(stdout, nullptr, _IONBF, 0); - ::setvbuf(stderr, nullptr, _IONBF, 0); - this->done = true; - this->runner->join(); - this->flush_buffer(this->stdout_buffer); - this->flush_buffer(this->stderr_buffer); - if ( this->fifo_fd ) { - ::syncfs(this->fifo_fd); - ::close(this->fifo_fd); - } - this->fifo_fd = 0; + // this->push(STDOUT, "POLL thread is exiting now....\n"); } /// Initializing constructor Logger::Logger(const string& fifo_name, const string& utgid, const string& tag) { - this->imp = make_unique<Implementation>(); + this->imp.reset(&log()); this->imp->init(fifo_name, utgid, tag); } +/// Default destructor +Logger::~Logger() { + if ( this->imp.get() ) this->imp.release(); +} + /// Run the logger in a separate thread void Logger::run() { - if ( !this->imp->runner.get() ) { - this->imp->runner = make_unique<thread>([this]() { this->imp->poll(); }); - } + if ( this->imp.get() ) this->imp->run(); } /// Stop processing and shutdown void Logger::stop() { - this->imp->stop(); + if ( this->imp.get() ) this->imp->stop(); } /// Initialize the logger unit extern "C" void fifolog_initialize_logger() { - auto& log = logger(); - log->run(); + logger(true)->run(); } /// Set new utgid value @@ -406,14 +722,14 @@ extern "C" void fifolog_set_utgid(const char* new_utgid) { ::snprintf(text,sizeof(text),"P%06d",::getpid()); utgid = text; } - Logger::Implementation::utgid = new_utgid ? new_utgid : text; + logger_imp::utgid = new_utgid ? new_utgid : text; if ( log.get() ) log->imp->rebuild_trailer(); } /// Set new tag value extern "C" void fifolog_set_tag(const char* new_tag) { auto& log = logger(false); - Logger::Implementation::systag = new_tag ? new_tag : "SYSTEM"; + logger_imp::systag = new_tag ? new_tag : "SYSTEM"; if ( log.get() ) log->imp->rebuild_trailer(); } @@ -421,7 +737,7 @@ extern "C" void fifolog_set_tag(const char* new_tag) { extern "C" int fifolog_set_fifo(const char* fifo) { auto& log = logger(false); if ( fifo && !log.get() ) { - Logger::Implementation::fifo_name = fifo; + logger_imp::fifo_name = fifo; return 1; } ::fprintf(stderr,"Failed to set output fifo: %s [%s] logger %s initialized.\n", @@ -434,8 +750,156 @@ extern "C" int fifolog_set_fifo(const char* fifo) { extern "C" void fifolog_finalize_logger() { auto& log = logger(false); if ( log.get() ) { - log->imp->stop(); + log->stop(); log.reset(); ::fprintf(stderr, "\nFinalizing dim logger....\n"); } } + +extern "C" int fputc(int c, FILE *stream) { + if ( logger(false).get() ) { + if ( stream == stdout ) + return log().push(logger_imp::STDOUT, c); + else if ( stream == stderr ) + return log().push(logger_imp::STDERR, c); + } + return funcs().real_fputc(c, stream); +} + +extern "C" int putc(int c, FILE *stream) { + return ::fputc(c, stream); +} + +extern "C" int putchar(int c) { + return ::fputc(c, stdout); +} + +extern "C" int fputs(const char *str, FILE *stream) { + if ( logger(false).get() ) { + if ( stream == stdout ) + return log().push(logger_imp::STDOUT, str); + else if ( stream == stderr ) + return log().push(logger_imp::STDERR, str); + } + return funcs().real_fputs(str, stream); +} + +extern "C" int puts(const char* str) { + return ::fputs(str, stdout); +} + +extern "C" ssize_t write(int fd, const void *buf, size_t count) { + if ( logger(false).get() ) { + switch(fd) { + case STDOUT_FILENO: return log().push(logger_imp::STDOUT, buf, count); + case STDERR_FILENO: return log().push(logger_imp::STDERR, buf, count); + default: break; + } + } + return funcs().real_write(fd, buf, count); +} + +extern "C" size_t fwrite(const void *str, size_t size, size_t nmemb, FILE *stream) { + if ( logger(false).get() ) { + if ( stream == stdout ) + return log().push(logger_imp::STDOUT, str, size*nmemb); + else if ( stream == stderr ) + return log().push(logger_imp::STDERR, str, size*nmemb); + } + return funcs().real_fwrite(str, size, nmemb, stream); +} + +extern "C" int vprintf(const char *format, LOGGER_VA_LIST args) { + return logger(false).get() + ? log().push(logger_imp::STDOUT, format, args) + : funcs().real_vprintf(format, args); +} + +extern "C" int vwprintf(const wchar_t *format, LOGGER_VA_LIST args) { + return logger(false).get() + ? log().push(logger_imp::STDOUT, format, args) + : funcs().real_vwprintf(format, args); +} + +extern "C" int vdprintf(int fd, const char *format, LOGGER_VA_LIST args) { + if ( logger(false).get() ) { + switch(fd) { + case STDOUT_FILENO: return log().push(logger_imp::STDOUT, format, args); + case STDERR_FILENO: return log().push(logger_imp::STDERR, format, args); + default: break; + } + } + return funcs().real_vdprintf(fd, format, args); +} + +extern "C" int vfprintf(FILE* stream, const char *format, LOGGER_VA_LIST args) { + if ( logger(false).get() ) { + if ( stream == stdout ) + return log().push(logger_imp::STDOUT, format, args); + else if ( stream == stderr ) + return log().push(logger_imp::STDERR, format, args); + } + return funcs().real_vfprintf(stream, format, args); +} + +extern "C" int __vfprintf_chk(FILE* stream, int flag, const char *format, LOGGER_VA_LIST args) { + if ( logger(false).get() ) { + if ( stream == stdout ) + return log().push(logger_imp::STDOUT, format, args); + else if ( stream == stderr ) + return log().push(logger_imp::STDERR, format, args); + } + return funcs().real_vfprintf_chk(stream, flag, format, args); +} + +extern "C" int __vprintf_chk(int flag, const char *format, LOGGER_VA_LIST args) { + return __vfprintf_chk(stdout, flag, format, args); +} + +extern "C" int printf(const char *format, ...) { + LOGGER_VA_LIST args; + va_start(args, format); + int ret = ::vprintf(format, args); + va_end(args); + return ret; +} + +extern "C" int wprintf(const wchar_t *format, ...) { + LOGGER_VA_LIST args; + va_start(args, format); + int ret = ::vwprintf(format, args); + va_end(args); + return ret; +} + +extern "C" int dprintf(int fd, const char *format, ...) { + LOGGER_VA_LIST args; + va_start(args, format); + int ret = ::vdprintf(fd, format, args); + va_end(args); + return ret; +} + +extern "C" int fprintf(FILE* stream, const char *format, ...) { + LOGGER_VA_LIST args; + va_start(args, format); + int ret = ::vfprintf(stream, format, args); + va_end(args); + return ret; +} + +extern "C" int __printf_chk(int flag, const char *format, ...) { + LOGGER_VA_LIST args; + va_start(args, format); + int ret = __vprintf_chk(flag, format, args); + va_end(args); + return ret; +} + +extern "C" int __fprintf_chk(FILE* stream, int flag, const char *format, ...) { + LOGGER_VA_LIST args; + va_start(args, format); + int ret = __vfprintf_chk(stream, flag, format, args); + va_end(args); + return ret; +} diff --git a/Online/ROLogger/kafka/OutputListener.cpp b/Online/ROLogger/kafka/OutputListener.cpp index d4b2b3612..7500b4346 100644 --- a/Online/ROLogger/kafka/OutputListener.cpp +++ b/Online/ROLogger/kafka/OutputListener.cpp @@ -121,21 +121,28 @@ void OutputListener::print_msg(const char* time_stamp, break; } } + size_t len = 0; + char text[1024]; + text[0] = 0; if ( !simple_msg ) { - if ( time_stamp ) ::printf("%s ",time_stamp); - if ( host ) ::printf("%-8s ",host); - if ( print_counter && counter ) ::printf("%8s ",counter); - ::printf("%-6s ", tag); + len += ::snprintf(text+len,sizeof(text)-len,"%s %-9s", + time_stamp ? time_stamp : "", + host ? host : ""); + if ( counter ) + len += ::snprintf(text+len,sizeof(text)-len,"%9s%-7s",counter,tag); + else + len += ::snprintf(text+len,sizeof(text)-len,"%-7s",tag); } - if ( utgid ) ::printf("%-36s ",utgid); - if ( message ) ::printf("%s ",message); - + len += ::snprintf(text+len,sizeof(text)-len,"%-25s%s\n", + utgid ? utgid : "", + message ? message : ""); if ( have_colors ) { graphics::white(); graphics::plain(); graphics::bg_black(); } - ::printf("\n"); + ::fputs(text,stdout); + ::fflush(stdout); } } #include <unistd.h> -- GitLab From 06ce301cc7984c617a9cea714b46ae2e9dcfb2a3 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Fri, 9 Apr 2021 18:12:58 +0200 Subject: [PATCH 15/26] Add monitoring to storage servers --- Online/Dataflow/src/Storage/StorageWriter.cpp | 13 +- .../options/EB_Transport_properties.opts | 3 +- .../python/GaudiOnline/Passthrough.py | 1 - Online/ROLogger/kafka/OutputListener.cpp | 5 +- .../SmiController/SmiController.h | 36 +++-- Online/SmiController/scripts/MBMDistBox.sh | 1 + Online/SmiController/scripts/SNDMulti.sh | 1 + Online/SmiController/scripts/SNDSingle.sh | 2 +- Online/SmiController/scripts/TestFileProd.sh | 9 ++ .../scripts/TestStorageReader.sh | 3 +- .../scripts/TestStorageWriter.sh | 3 +- Online/SmiController/scripts/configure.sh | 32 ++-- Online/SmiController/src/SmiController.cpp | 49 ++++--- Online/Storage/CMakeLists.txt | 7 +- Online/Storage/Storage/fdb_dbase.h | 43 ++++++ Online/Storage/Storage/fdb_server.h | 16 +- Online/Storage/src/server/fdb_SQLite.cpp | 10 +- Online/Storage/src/server/fdb_SQLite_bind.cpp | 137 ------------------ .../Storage/src/server/fdb_SQLite_direct.cpp | 0 Online/Storage/src/server/fdb_db_server.cpp | 91 ++++++++++-- Online/Storage/src/server/fdb_fs_server.cpp | 103 +++++++++++-- 21 files changed, 331 insertions(+), 234 deletions(-) delete mode 100644 Online/Storage/src/server/fdb_SQLite_bind.cpp delete mode 100644 Online/Storage/src/server/fdb_SQLite_direct.cpp diff --git a/Online/Dataflow/src/Storage/StorageWriter.cpp b/Online/Dataflow/src/Storage/StorageWriter.cpp index 9a542480a..ec97f8b65 100644 --- a/Online/Dataflow/src/Storage/StorageWriter.cpp +++ b/Online/Dataflow/src/Storage/StorageWriter.cpp @@ -279,17 +279,18 @@ int StorageWriter::save_pcie40_as_mdf(Buffer& buff, const uint8_t* start, long l /// Save MDF frame or MDF burst int StorageWriter::save_mdf_buffer(Buffer& buff, const uint8_t* start, long len) { - /// Auto detect data type: now check for MDF data type - for( const uint8_t* end = start + len; start < end; ) { + for( const uint8_t *begin=start, *end = start + len; start < end; ) { auto* header = (EventHeader*)start; + if ( !header->is_mdf() ) { + warning("save_mdf: Encountered invalid MDF header: (%d %d %d). Skip %ld bytes of data.", + header->size0(), header->size1(), header->size2(), start-begin); + return DF_SUCCESS; + } auto* hdr = header->subHeader().H1; long length = header->size0(); long run = hdr->runNumber(); - if ( start+length > end ) { - break; - } - else if ( 0 == m_curr_run ) { + if ( 0 == m_curr_run ) { /// First event in this buffer: store the current run number m_curr_run = run; m_curr_orbit = hdr->orbitNumber(); diff --git a/Online/EventBuilding/options/EB_Transport_properties.opts b/Online/EventBuilding/options/EB_Transport_properties.opts index 6cfdb061a..3d06991a2 100755 --- a/Online/EventBuilding/options/EB_Transport_properties.opts +++ b/Online/EventBuilding/options/EB_Transport_properties.opts @@ -2,6 +2,7 @@ EB_transport.BU_ranks = {}; EB_transport.MPI_errors_return = FALSE; EB_transport.RU_ranks = {}; EB_transport.RUs_per_nic = {}; -EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.J31513.json"; +EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.n18450.json"; EB_transport.n_par_mess = 0; EB_transport.n_sources_per_ru = { 3, 2, 2, 2 }; +EB_transport.src_ids = {}; diff --git a/Online/GaudiOnline/python/GaudiOnline/Passthrough.py b/Online/GaudiOnline/python/GaudiOnline/Passthrough.py index 4584cc8fe..907b738ed 100644 --- a/Online/GaudiOnline/python/GaudiOnline/Passthrough.py +++ b/Online/GaudiOnline/python/GaudiOnline/Passthrough.py @@ -22,7 +22,6 @@ class Passthrough(Application): input = self.setup_event_input(TAE) self.input = input passThrough = Configurables.Online__Passthrough('Passthrough') - #passThrough = Configurables.Online__PassthroughPCIE40('Passthrough') passThrough.RawGuard = '/Event/DAQ/RawDataGuard' passThrough.AcceptRate = acceptRate self.passThrough = passThrough diff --git a/Online/ROLogger/kafka/OutputListener.cpp b/Online/ROLogger/kafka/OutputListener.cpp index 7500b4346..1eea5d071 100644 --- a/Online/ROLogger/kafka/OutputListener.cpp +++ b/Online/ROLogger/kafka/OutputListener.cpp @@ -133,9 +133,8 @@ void OutputListener::print_msg(const char* time_stamp, else len += ::snprintf(text+len,sizeof(text)-len,"%-7s",tag); } - len += ::snprintf(text+len,sizeof(text)-len,"%-25s%s\n", - utgid ? utgid : "", - message ? message : ""); + len += ::snprintf(text+len,sizeof(text)-len,"%-34s%s\n", + utgid ? utgid : "", message ? message : ""); if ( have_colors ) { graphics::white(); graphics::plain(); diff --git a/Online/SmiController/SmiController/SmiController.h b/Online/SmiController/SmiController/SmiController.h index 149f4bcec..95e4771a1 100644 --- a/Online/SmiController/SmiController/SmiController.h +++ b/Online/SmiController/SmiController/SmiController.h @@ -178,32 +178,36 @@ namespace FiniteStateMachine { /// Variable to publish the instance information std::string m_instance_info; - long m_tms_dic_dns_ID = 0; - long m_smi_dic_dns_ID = 0; - long m_smi_dis_dns_ID = 0; - long m_local_dic_dns_ID = 0; - long m_local_dis_dns_ID = 0; + long m_tms_dic_dns_ID = 0; + long m_smi_dic_dns_ID = 0; + long m_smi_dis_dns_ID = 0; + long m_local_dic_dns_ID = 0; + long m_local_dis_dns_ID = 0; /// Pointer to the dim service receiving commands - int m_command_ID = 0; + int m_command_ID = 0; /// Pointer to the secondary dim service receiving commands - int m_userCmd_ID = 0; - /// Pointer to the dim service serving the FSM status - int m_status_ID = 0; - int m_state_ID = 0; + int m_userCmd_ID = 0; + /// Pointer to the dim service serving the FSM state + int m_status_ID = 0; + /// Pointer to the dim service serving the FSM state + int m_state_ID = 0; /// Pointer to the dim service serving the FSM sub status information - int m_sub_status_ID = 0; + int m_sub_status_ID = 0; /// Pointer to the dim service publishing the instance information - int m_fsm_tags_ID = 0; + int m_fsm_tags_ID = 0; /// Pointer to the dim service publishing the task information - int m_fsm_tasks_ID = 0; + int m_fsm_tasks_ID = 0; /// Pointer to dim client with maximum instance tag - int m_num_worker_ID = 0; + int m_num_worker_ID = 0; /// Pointer to dim client with node state - int m_node_state_ID = 0; + int m_node_state_ID = 0; /// Maximal number of active threads on the node - int m_num_worker_threads = 1; + int m_num_worker_threads = 1; + /// Variable to check initialization + bool m_inited = false; + /// Basic controller configuration: chains initialize() and start_smi() void configure(); /// Controller configuration diff --git a/Online/SmiController/scripts/MBMDistBox.sh b/Online/SmiController/scripts/MBMDistBox.sh index ec1c05e34..2b7d65a24 100755 --- a/Online/SmiController/scripts/MBMDistBox.sh +++ b/Online/SmiController/scripts/MBMDistBox.sh @@ -8,4 +8,5 @@ # Date: 20/05/2013 # # ========================================================================= +# `dataflow_task Class0` `dataflow_default_options ${TASK_TYPE}` diff --git a/Online/SmiController/scripts/SNDMulti.sh b/Online/SmiController/scripts/SNDMulti.sh index 6d1cb2f46..391e499e7 100755 --- a/Online/SmiController/scripts/SNDMulti.sh +++ b/Online/SmiController/scripts/SNDMulti.sh @@ -8,4 +8,5 @@ # Date: 20/05/2013 # # ========================================================================= +# `dataflow_task Class2` `dataflow_default_options ${TASK_TYPE}`; diff --git a/Online/SmiController/scripts/SNDSingle.sh b/Online/SmiController/scripts/SNDSingle.sh index 7f96a3fc6..f2eec598a 100755 --- a/Online/SmiController/scripts/SNDSingle.sh +++ b/Online/SmiController/scripts/SNDSingle.sh @@ -8,5 +8,5 @@ # Date: 20/05/2013 # # ========================================================================= +# `dataflow_task Class2` `dataflow_default_options ${TASK_TYPE}` - diff --git a/Online/SmiController/scripts/TestFileProd.sh b/Online/SmiController/scripts/TestFileProd.sh index bbdf2fc63..93da36a77 100644 --- a/Online/SmiController/scripts/TestFileProd.sh +++ b/Online/SmiController/scripts/TestFileProd.sh @@ -1,4 +1,13 @@ #!/bin/bash +# ========================================================================= +# +# Generic farm task startup script +# +# Author M.Frank +# Version: 1.0 +# Date: 20/05/2013 +# +# ========================================================================= # export LOGFIFO=/run/fmc/logSrv.fifo; `dataflow_task Class2` `dataflow_default_options ${TASK_TYPE}` diff --git a/Online/SmiController/scripts/TestStorageReader.sh b/Online/SmiController/scripts/TestStorageReader.sh index a12662bb2..83c7de473 100755 --- a/Online/SmiController/scripts/TestStorageReader.sh +++ b/Online/SmiController/scripts/TestStorageReader.sh @@ -8,7 +8,6 @@ # Date: 20/05/2013 # # ========================================================================= -export INFO_OPTIONS=/group/online/dataflow/options/LHCb/MONITORING/OnlineEnv.opts; -echo "ERROR `dataflow_task Class1` `dataflow_default_options ${TASK_TYPE}`"; +# `dataflow_task Class1` `dataflow_default_options ${TASK_TYPE}`; diff --git a/Online/SmiController/scripts/TestStorageWriter.sh b/Online/SmiController/scripts/TestStorageWriter.sh index a12662bb2..83c7de473 100755 --- a/Online/SmiController/scripts/TestStorageWriter.sh +++ b/Online/SmiController/scripts/TestStorageWriter.sh @@ -8,7 +8,6 @@ # Date: 20/05/2013 # # ========================================================================= -export INFO_OPTIONS=/group/online/dataflow/options/LHCb/MONITORING/OnlineEnv.opts; -echo "ERROR `dataflow_task Class1` `dataflow_default_options ${TASK_TYPE}`"; +# `dataflow_task Class1` `dataflow_default_options ${TASK_TYPE}`; diff --git a/Online/SmiController/scripts/configure.sh b/Online/SmiController/scripts/configure.sh index 198d93ffc..b71326ef3 100755 --- a/Online/SmiController/scripts/configure.sh +++ b/Online/SmiController/scripts/configure.sh @@ -13,23 +13,25 @@ export HOST=`echo ${DIM_DNS_HOST} | tr a-z A-Z`; export PREAMBLE_OPTS=`pwd`/../options/Empty.opts; # # -export ONLINETASKS=/group/online/dataflow/templates; -export NON_DYNAMIC_OPTS=/group/online/dataflow/options/${PARTITION_NAME}; +HOST_TYPE=`hostname -s | tr a-z A-Z`; +if test "`echo ${HOST_TYPE} | cut -b 1-3`" = "MON"; then + export NON_DYNAMIC_OPTS=/group/online/dataflow/options/${PARTITION_NAME}/MONITORING; +elif test "`echo ${HOST_TYPE} | cut -b 3-4`" = "EB"; then + export NON_DYNAMIC_OPTS=/group/online/dataflow/options/${PARTITION_NAME}; +else + export NON_DYNAMIC_OPTS=/group/online/dataflow/options/${PARTITION_NAME}; +fi; +# +# export DYNAMIC_OPTS=${NON_DYNAMIC_OPTS}; +export PYTHONPATH=${NON_DYNAMIC_OPTS}:${PYTHONPATH}; +export INFO_OPTIONS=${NON_DYNAMIC_OPTS}/OnlineEnv.opts; +# +export ONLINETASKS=/group/online/dataflow/templates; export DATAINTERFACE=`python /group/online/dataflow/scripts/getDataInterface.py`; -export INFO_OPTIONS=${DYNAMIC_OPTS}/${PARTITION_NAME}_Info.opts; -export OPTS=/group/online/dataflow/templates/options -export msg_svc=MessageSvc -export msg_svc=LHCb::FmcMessageSvc -export gaudi_task="GaudiOnlineExe.exe libGaudiOnline.so OnlineTask -msgsvc=$msg_svc " -export Class0_task="$gaudi_task -tasktype=LHCb::Class0Task " -export Class1_task="$gaudi_task -tasktype=LHCb::Class1Task -main=$OPTS/Main.opts " -export Class2_task="$gaudi_task -tasktype=LHCb::Class2Task -main=$OPTS/Main.opts " -export Checkpoint_task="GaudiCheckpoint.exe libGaudiOnline.so OnlineTask -msgsvc=$msg_svc -tasktype=LHCb::Class1Task -main=$OPTS/Main.opts " -## export DATAFLOW_task="gentest.exe libDataflow.so dataflow_run_task -msg=Dataflow_FmcLogger -mon=Dataflow_DIMMonitoring"; export DATAFLOW_task="gentest.exe libDataflow.so dataflow_run_task -msg=fifo -mon=Dataflow_DIMMonitoring"; # -export PYTHONPATH=${DYNAMIC_OPTS}/MONITORING:${PYTHONPATH}; +# # dataflow_specialized_options() { @@ -39,12 +41,14 @@ dataflow_specialized_options() echo "-opts=/group/online/dataflow/options/${PARTITION_NAME}/MONITORING/${UTGID}.opts"; fi; } +# dataflow_default_options() { echo "-opts=/group/online/dataflow/options/${PARTITION_NAME}/MONITORING/${UTGID}.opts"; } +# dataflow_task() { #echo "exec -a ${UTGID} gentest.exe libDataflow.so dataflow_run_task -msg=Dataflow_FmcLogger -mon=Dataflow_DIMMonitoring -class=$1"; - echo "exec -a ${UTGID} gentest.exe libDataflow.so dataflow_run_task -msg=fifo -mon=Dataflow_DIMMonitoring -class=$1"; + echo "exec -a ${UTGID} gentest.exe libDataflow.so dataflow_run_task -msg=fifo -mon=Dataflow_DIMMonitoring -class=$1"; } diff --git a/Online/SmiController/src/SmiController.cpp b/Online/SmiController/src/SmiController.cpp index a5768a76e..c04dd6391 100644 --- a/Online/SmiController/src/SmiController.cpp +++ b/Online/SmiController/src/SmiController.cpp @@ -85,10 +85,12 @@ public: public: /// Initializing constructor SmiControllerObject(SmiController* ctrl, const std::string& nam) - : SmiObject(ctrl->config.smi_dns.c_str(), ctrl->config.smiDomain().c_str(), nam.c_str()), - controller(ctrl) - { - } + : SmiObject(ctrl->config.smiDomain().c_str(), nam.c_str()), controller(ctrl) + { } + /// Initializing constructor + SmiControllerObject(SmiController* ctrl, const std::string& dns, const std::string& nam) + : SmiObject(dns.c_str(), ctrl->config.smiDomain().c_str(), nam.c_str()), controller(ctrl) + { } /// Default destructor virtual ~SmiControllerObject() = default; virtual void smiStateChangeHandler() override final {} @@ -767,6 +769,14 @@ SmiController::SmiController(const config_t& cfg) m_log.reset(new RTL::Logger(RTL::Logger::LogDevice::getGlobalDevice(), "Controller", cfg.output_level)); m_num_worker_threads = config.num_workers; m_node_state = State::UNKNOWN; + + m_local_dic_dns_ID = ::dic_add_dns(config.dns.c_str(), ::dim_get_dns_port()); + m_tms_dic_dns_ID = ::dic_add_dns(config.tms_dns.c_str(), ::dim_get_dns_port()); + m_smi_dic_dns_ID = ::dic_add_dns(config.smi_dns.c_str(), ::dim_get_dns_port()); + m_local_dis_dns_ID = 0;//::dis_add_dns(config.dns.c_str(), ::dim_get_dns_port()); + m_smi_dis_dns_ID = ::dis_add_dns(config.smi_dns.c_str(), ::dim_get_dns_port()); + + //::dis_start_serving_dns(m_local_dis_dns_ID, config.name.c_str()); } /// Default destructor @@ -801,12 +811,6 @@ void SmiController::initialize() { m_monitor.partitionID = -1; m_monitor.pad = 0; - m_local_dic_dns_ID = ::dic_add_dns(config.dns.c_str(), ::dim_get_dns_port()); - m_tms_dic_dns_ID = ::dic_add_dns(config.tms_dns.c_str(), ::dim_get_dns_port()); - m_smi_dic_dns_ID = ::dic_add_dns(config.smi_dns.c_str(), ::dim_get_dns_port()); - m_local_dis_dns_ID = 0;//::dis_add_dns(config.dns.c_str(), ::dim_get_dns_port()); - m_smi_dis_dns_ID = ::dis_add_dns(config.smi_dns.c_str(), ::dim_get_dns_port()); - m_command_ID = ::dis_add_cmnd_dns( m_local_dis_dns_ID, config.name.c_str(), "C", feed_transition, (long)this); @@ -842,18 +846,20 @@ void SmiController::initialize() { (long)this,0,0); } string slice = config.standalone ? string("Manager") : config.smiDomain() + "_Manager"; - m_taskset = make_unique<TaskSet>(this, slice.c_str()); + m_taskset = make_unique<TaskSet>(this, config.smi_dns, slice.c_str()); if ( config.standalone ) { string node_state = RTL::str_upper("SMI/"+config.smi_domain+"/PROCESSINGNODE"); bool print = config.output_level <= LIB_RTL_INFO; m_self = make_unique<CtrlProxy> (this, "Controller"); - m_nodeProxy = make_unique<NodeProxy> (this, "ProcessingNode"); + m_nodeProxy = make_unique<NodeProxy> (this, config.smi_dns, "ProcessingNode"); m_defTask = make_unique<DfTaskProxy>(this, "DefTask"); m_vipTask = make_unique<DfTaskProxy>(this, "VipTask"); m_node_state_ID = ::dic_info_service_dns(m_local_dic_dns_ID, node_state.c_str(), - MONITORED,0,0,0,node_state_handler,(long)this,0,0); + MONITORED, 0, 0, 0, + node_state_handler, + (long)this, 0, 0); print ? m_self->setPrintOn() : m_self->setPrintOff(); print ? m_defTask->setPrintOn() : m_defTask->setPrintOff(); print ? m_vipTask->setPrintOn() : m_vipTask->setPrintOff(); @@ -862,8 +868,8 @@ void SmiController::initialize() { m_vipTask->setState(State::OFFLINE); } ::dis_start_serving_dns(m_local_dis_dns_ID, config.name.c_str()); - //IocSensor::instance().send(this,CONTROLLER_REMOVE_TASK,new string("DEFTASK")); - //IocSensor::instance().send(this,CONTROLLER_REMOVE_VIPTASK,new string("VIPTASK")); + this_thread::sleep_for(chrono::milliseconds(200)); + m_inited = true; } //============================================================================== @@ -920,7 +926,7 @@ void SmiController::load_tasks() { } } m_log->debug("Sleep 2 seconds after ADDTASKS."); - ::lib_rtl_sleep(2000); + this_thread::sleep_for(chrono::milliseconds(2000)); IocSensor::instance().send(this, CONTROLLER_SETSTATE, nullptr); } @@ -1033,7 +1039,7 @@ bool SmiController::attach_tasks(std::map<std::string, SmiTask*>&& tasks) { } } - ::lib_rtl_sleep(200); + this_thread::sleep_for(chrono::milliseconds(200)); for( const auto& task : tasks) { const auto& nam = task.second->utgid; m_log->info("Adding dependent task: %s", nam.c_str()); @@ -1053,7 +1059,7 @@ bool SmiController::attach_tasks(std::map<std::string, SmiTask*>&& tasks) { proxy->task->add_fmc_args("-DDIM_DNS_HOST="+RTL::str_upper(RTL::nodeNameShort())); } ::dis_start_serving_dns(m_local_dis_dns_ID, config.name.c_str()); - ::lib_rtl_sleep(200); + this_thread::sleep_for(chrono::milliseconds(200)); return true; } @@ -1133,7 +1139,7 @@ void SmiController::handle(const CPP::Event& event) { if ( !config.smi_utgid.empty() ) { TaskManager::instance(RTL::nodeNameShort(), m_tms_dic_dns_ID).kill(config.smi_utgid,SIGKILL); } - ::lib_rtl_sleep(1000); + this_thread::sleep_for(chrono::milliseconds(1000)); _exit(0); return; @@ -1297,8 +1303,11 @@ void SmiController::run() { "-dns", config.smi_dns.c_str(), config.smi_domain.c_str(), config.smi_file.c_str(), nullptr}; + /// Wait with server start until controller is initialized + while ( !this->m_inited ) + this_thread::sleep_for(chrono::milliseconds(100)); + this_thread::sleep_for(chrono::milliseconds(1000)); /// function will never return.... - ::lib_rtl_sleep(2500); (*fun.function)(sizeof(args)/sizeof(args[0])-1, (char**)args); thr->join(); } diff --git a/Online/Storage/CMakeLists.txt b/Online/Storage/CMakeLists.txt index f2496c2cc..c8151ec63 100644 --- a/Online/Storage/CMakeLists.txt +++ b/Online/Storage/CMakeLists.txt @@ -16,7 +16,8 @@ gaudi_subdir(Storage v0r1) ################################################################################ # gaudi_depends_on_subdirs(Online/OnlineBase - Online/HTTP) + Online/HTTP + Online/dim) # find_package(SQLite3 REQUIRED) # @@ -32,9 +33,9 @@ gaudi_add_library(StorageClient src/client/*.cpp target_compile_definitions(StorageClient PRIVATE -DSTORAGECLIENT=1) # gaudi_add_library(StorageServer src/server/*.cpp - INCLUDE_DIRS Boost HTTP SQLite3 + INCLUDE_DIRS Boost HTTP SQLite3 dim NO_PUBLIC_HEADERS - LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt z Boost HTTP SQLite3 StorageClient) + LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt z dim Boost HTTP SQLite3 StorageClient) # target_compile_definitions(StorageServer PRIVATE -DSTORAGECLIENT=1) # diff --git a/Online/Storage/Storage/fdb_dbase.h b/Online/Storage/Storage/fdb_dbase.h index f4c2415d7..c21bad757 100644 --- a/Online/Storage/Storage/fdb_dbase.h +++ b/Online/Storage/Storage/fdb_dbase.h @@ -186,6 +186,49 @@ namespace Online { virtual boost::system::error_code update_object_state(const std::string& object_name, const std::string& state); }; + + + /// Monitor object to publish values + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.02.2021 + */ + class fdb_dbase_monitor_t { + public: + struct Data { + int num_get_success {0}; + int num_get_errors {0}; + int num_get_not_found {0}; + int num_get_unauthorized {0}; + + int num_del_success {0}; + int num_del_errors {0}; + int num_del_not_found {0}; + int num_del_unauthorized {0}; + + int num_put_success {0}; + int num_put_errors {0}; + int num_put_bad_request {0}; + int num_put_unauthorized {0}; + + int num_upda_success {0}; + int num_upda_errors {0}; + int num_upda_bad_request {0}; + } data; + + std::vector<int> services; + + void add(int svc) { services.emplace_back(svc); } + + public: + /// Standard constructor + fdb_dbase_monitor_t(); + /// Default destructor + ~fdb_dbase_monitor_t(); + }; + } // End namespace storage } // End namespace Online #endif // ONLINE_STORAGE_FDB_DBASE_H diff --git a/Online/Storage/Storage/fdb_server.h b/Online/Storage/Storage/fdb_server.h index 4df72257b..64714b397 100644 --- a/Online/Storage/Storage/fdb_server.h +++ b/Online/Storage/Storage/fdb_server.h @@ -35,12 +35,22 @@ namespace http { */ template <typename T> class basic_http_server<T>::handler_t : public HttpRequestHandler { + public: - typedef typename T::traits traits; - typedef typename traits::dbase_t dbase_t; + typedef typename T::traits traits; + typedef typename traits::dbase_t dbase_t; + typedef typename traits::monitor_t monitor_t; + public: - std::unique_ptr<dbase_t> dbase; + + /// Reference to the database instance + std::unique_ptr<dbase_t> dbase; + + /// Reference to the monitoring instance + std::unique_ptr<monitor_t> monitor; + public: + /// Standard constructor explicit handler_t() = default; /// Standard destructor diff --git a/Online/Storage/src/server/fdb_SQLite.cpp b/Online/Storage/src/server/fdb_SQLite.cpp index 3cfee40e0..41da23c57 100644 --- a/Online/Storage/src/server/fdb_SQLite.cpp +++ b/Online/Storage/src/server/fdb_SQLite.cpp @@ -15,6 +15,7 @@ // Framework include files #include "fdb_SQLite.h" +#include <RTL/strdef.h> #include <RTL/rtl.h> // C/C++ files @@ -172,20 +173,25 @@ SQLite_handler_t::set (const std::string& object_name, int value) { return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); } +#include <dim/dis.h> #include <Storage/fdb_server.h> + class Online::storage::db::traits { public: - typedef fdb_dbase_t dbase_t; + typedef fdb_dbase_t dbase_t; + typedef fdb_dbase_monitor_t monitor_t; }; extern "C" int fdb_sqlite_server(int argc, char** argv) { + std::string srv = "/"+RTL::str_upper(RTL::nodeNameShort())+"/"+RTL::processName(); Online::storage::SrvRun<Online::storage::db> s; s.create(argc, argv); auto* sql = new Online::storage::fdb_dbase_t(s.server); sql->_debug = s.application_debug; sql->_engine.reset(new SQLite_handler_t(s.dbase, s.application_debug)); s.ptr->implementation->dbase.reset(sql); + s.ptr->implementation->monitor.reset(new fdb_dbase_monitor_t()); + ::dis_start_serving(srv.c_str()); s.start(); return 0; } - diff --git a/Online/Storage/src/server/fdb_SQLite_bind.cpp b/Online/Storage/src/server/fdb_SQLite_bind.cpp deleted file mode 100644 index 0c136cbaa..000000000 --- a/Online/Storage/src/server/fdb_SQLite_bind.cpp +++ /dev/null @@ -1,137 +0,0 @@ -//========================================================================== -// LHCb Online software suite -//-------------------------------------------------------------------------- -// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) -// All rights reserved. -// -// For the licensing terms see OnlineSys/LICENSE. -// -//-------------------------------------------------------------------------- -// -// Package : RPC -// -// Author : Markus Frank -//========================================================================== -#if 0 - -// Framework include files -#include "fdb_SQLite.h" -#include <RTL/rtl.h> - -// C/C++ files -using namespace boost; - -namespace { - - class handler_t : public Online::storage::fdb_SQLite::handler_t { - public: - - /// Standard constructor - handler_t(const std::string& p, int dbg) - : Online::storage::fdb_SQLite::handler_t(dbg), path(p) - { - this->init(path); - } - - /// Default destructor - virtual ~handler_t() { - this->fini(); - } - - /// Check the existence of a given object in the database - virtual system::error_code query_file(const std::string& object_name, File& file, int state) override { - this->query_record.reset(); - this->query_record.bind(1, object_name); - this->query_record.bind(2, state); - int ret = this->query_record.execute(); - while ( ret == SQLITE_ROW ) { - file.name = query_record.get<std::string>(0); - file.state = query_record.get<std::string>(1); - file.size = query_record.get<std::string>(2); - file.date = query_record.get<std::string>(3); - file.host = query_record.get<std::string>(4); - return system::error_code(system::errc::success, system::system_category()); - } - return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); - } - - /// Add a new object to the database - virtual system::error_code add (const std::string& object_name, - const std::string& date, - std::size_t length, - const std::string& host) override { - this->insert_record.reset(); - this->insert_record.bind(1, object_name); - this->insert_record.bind(2, STATE_OPEN); - this->insert_record.bind(3, length); - this->insert_record.bind(4, date); - this->insert_record.bind(5, host); - this->database.begin(); - int ret = this->insert_record.execute(); - this->insert_record.reset(); - this->database.commit(); - if ( ret == SQLITE_DONE ) { - return system::error_code(system::errc::success, system::system_category()); - } - else if ( ret == SQLITE_CONSTRAINT_PRIMARYKEY ) { - return system::error_code(system::errc::file_exists, system::system_category()); - } - else if ( ret == SQLITE_CONSTRAINT_FOREIGNKEY ) { - return system::error_code(system::errc::file_exists, system::system_category()); - } - else if ( ret != SQLITE_OK ) { - return system::error_code(system::errc::file_exists, system::system_category()); - } - return system::error_code(system::errc::success, system::system_category()); - } - - /// Remove an object from the database - virtual system::error_code del (const std::string& object_name) override { - this->delete_record.reset(); - this->delete_record.bind(1, object_name); - this->database.begin(); - int ret = this->delete_record.execute(); - this->delete_record.reset(); - this->database.commit(); - if ( ret == SQLITE_OK ) - return system::error_code(system::errc::success, system::system_category()); - else if ( ret == SQLITE_DONE ) - return system::error_code(system::errc::success, system::system_category()); - return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); - } - - /// lock database record - virtual system::error_code set (const std::string& object_name, int value) override { - this->lock_record.reset(); - this->lock_record.bind(1, value); - this->lock_record.bind(2, object_name); - this->database.begin(); - int ret = this->lock_record.execute(); - this->lock_record.reset(); - this->database.commit(); - if ( ret == SQLITE_DONE ) - return system::error_code(system::errc::success, system::system_category()); - else if ( ret == SQLITE_OK ) - return system::error_code(system::errc::success, system::system_category()); - return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); - } - }; -} - -#include <Storage/fdb_server.h> -class Online::storage::db::traits { -public: - typedef fdb_dbase_t dbase_t; -}; - -extern "C" int fdb_sqlite_bind_server(int argc, char** argv) { - Online::storage::SrvRun<Online::storage::db> s; - s.create(argc, argv); - auto* sql = new Online::storage::fdb_SQLite(s.server); - sql->debug = s.application_debug; - sql->engine.reset(new handler_t(s.dbase, s.application_debug)); - s.ptr->implementation->dbase.reset(sql); - s.start(); - return 0; -} -#endif diff --git a/Online/Storage/src/server/fdb_SQLite_direct.cpp b/Online/Storage/src/server/fdb_SQLite_direct.cpp deleted file mode 100644 index e69de29bb..000000000 diff --git a/Online/Storage/src/server/fdb_db_server.cpp b/Online/Storage/src/server/fdb_db_server.cpp index 4e6dc0a29..33643942a 100644 --- a/Online/Storage/src/server/fdb_db_server.cpp +++ b/Online/Storage/src/server/fdb_db_server.cpp @@ -18,12 +18,16 @@ #include <Storage/fdb_dbase.h> #include <Storage/fdb_server.h> #include <Storage/communication.h> +#include <RTL/strdef.h> +#include <RTL/rtl.h> +#include <dim/dis.h> // C/C++ include files class Online::storage::db::traits { public: - typedef fdb_dbase_t dbase_t; + typedef fdb_dbase_t dbase_t; + typedef fdb_dbase_monitor_t monitor_t; }; using namespace std; @@ -43,6 +47,51 @@ static inline string object_name(const string& obj, string opt="") { return obj; } +/// Standard constructor +fdb_dbase_monitor_t::fdb_dbase_monitor_t() { + std::string nam = "/"+RTL::str_upper(RTL::nodeNameShort())+"/"+RTL::processName(); + add( ::dis_add_service((nam+"/DATA").c_str(),"C",&data,sizeof(data),0,0) ); + add( ::dis_add_service((nam+"/GET").c_str(),"L", + &data.num_get_success,sizeof(data.num_get_success),0,0) ); + add( ::dis_add_service((nam+"/GET_errors").c_str(),"L", + &data.num_get_errors,sizeof(data.num_get_errors),0,0) ); + add( ::dis_add_service((nam+"/GET_not_found").c_str(),"L", + &data.num_get_not_found,sizeof(data.num_get_not_found),0,0) ); + add( ::dis_add_service((nam+"/GET_unauthorized").c_str(),"L", + &data.num_get_unauthorized,sizeof(data.num_get_unauthorized),0,0) ); + + add( ::dis_add_service((nam+"/DEL").c_str(),"L", + &data.num_del_success,sizeof(data.num_del_success),0,0) ); + add( ::dis_add_service((nam+"/DEL_errors").c_str(),"L", + &data.num_del_errors,sizeof(data.num_del_errors),0,0) ); + add( ::dis_add_service((nam+"/DEL_not_found").c_str(),"L", + &data.num_del_not_found,sizeof(data.num_del_not_found),0,0) ); + add( ::dis_add_service((nam+"/DEL_unauthorized").c_str(),"L", + &data.num_del_unauthorized,sizeof(data.num_del_unauthorized),0,0) ); + + add( ::dis_add_service((nam+"/PUT").c_str(),"L", + &data.num_put_success,sizeof(data.num_put_success),0,0) ); + add( ::dis_add_service((nam+"/PUT_errors").c_str(),"L", + &data.num_put_errors,sizeof(data.num_put_errors),0,0) ); + add( ::dis_add_service((nam+"/PUT_bad_request").c_str(),"L", + &data.num_put_bad_request,sizeof(data.num_put_bad_request),0,0) ); + add( ::dis_add_service((nam+"/PUT_unauthorized").c_str(),"L", + &data.num_put_unauthorized,sizeof(data.num_put_unauthorized),0,0) ); + + add( ::dis_add_service((nam+"/UPDA").c_str(),"L", + &data.num_upda_success,sizeof(data.num_upda_success),0,0) ); + add( ::dis_add_service((nam+"/UPDA_errors").c_str(),"L", + &data.num_upda_errors,sizeof(data.num_upda_errors),0,0) ); + add( ::dis_add_service((nam+"/UPDA_bad_request").c_str(),"L", + &data.num_upda_bad_request,sizeof(data.num_upda_bad_request),0,0) ); +} + +/// Default destructor +fdb_dbase_monitor_t::~fdb_dbase_monitor_t() { + for( auto svc : services ) ::dis_remove_service(svc); +} + + /// Standard destructor template <> http::basic_http_server<db>::handler_t::~handler_t() { } @@ -57,19 +106,25 @@ http::basic_http_server<db>::handler_t::handle_get(const request_t& req, reply_t dbase_t::lock_t lock(dbase.get()); ec = dbase->query_object(req.uri, access_name, date, length); } - if ( ec == boost::system::errc::no_such_file_or_directory ) + if ( ec == boost::system::errc::no_such_file_or_directory ) { rep = reply_t::stock_reply(reply_t::not_found); - else if ( ec == boost::system::errc::permission_denied ) + ++monitor->data.num_get_not_found; + } + else if ( ec == boost::system::errc::permission_denied ) { rep = reply_t::stock_reply(reply_t::unauthorized); - else if ( !error_code_ok(ec, this->debug) ) + ++monitor->data.num_get_unauthorized; + } + else if ( !error_code_ok(ec, this->debug) ) { rep = reply_t::stock_reply(reply_t::not_found); - + ++monitor->data.num_get_not_found; + } if ( !error_code_ok(ec, this->debug) ) { header_t h(http::constants::error_cause,"Failed to get "+req.uri+" ["+ec.message()+"]"); ::lib_rtl_output(LIB_RTL_ERROR,"GET: %s %-20s = %s", reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); rep.headers.emplace_back(move(h)); + ++monitor->data.num_get_errors; } else { bool mb = length > 3*MByte; @@ -78,6 +133,7 @@ http::basic_http_server<db>::handler_t::handle_get(const request_t& req, reply_t ::lib_rtl_output(LIB_RTL_INFO,"GET: '%s' %.1f %cB [%s]", access_name.c_str(), double(length)/(mb ? MByte : kByte), mb ? 'M' : 'k', req.remote_address().to_string().c_str()); + ++monitor->data.num_get_success; } return write; } @@ -92,19 +148,25 @@ http::basic_http_server<db>::handler_t::handle_delete(const request_t& req, repl dbase_t::lock_t lock(dbase.get()); ec = dbase->delete_object(obj, access_name, date, length); } - if ( ec == boost::system::errc::no_such_file_or_directory ) + if ( ec == boost::system::errc::no_such_file_or_directory ) { rep = reply_t::stock_reply(reply_t::not_found); - else if ( ec == boost::system::errc::permission_denied ) + ++monitor->data.num_del_not_found; + } + else if ( ec == boost::system::errc::permission_denied ) { rep = reply_t::stock_reply(reply_t::unauthorized); - else if ( !error_code_ok(ec) ) + ++monitor->data.num_del_unauthorized; + } + else if ( !error_code_ok(ec) ) { rep = reply_t::stock_reply(reply_t::not_found); - + ++monitor->data.num_del_not_found; + } if ( !error_code_ok(ec, this->debug) ) { header_t h(http::constants::error_cause,"Failed to delete "+req.uri+" ["+ec.message()+"]"); ::lib_rtl_output(LIB_RTL_ERROR,"DELETE: %s %-20s = %s", reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); rep.headers.emplace_back(move(h)); + ++monitor->data.num_del_errors; } else { bool mb = length > 3*MByte; @@ -113,6 +175,7 @@ http::basic_http_server<db>::handler_t::handle_delete(const request_t& req, repl ::lib_rtl_output(LIB_RTL_INFO,"DELETE:'%s' %.1f %cB [%s]", obj.c_str(), double(length)/(mb ? MByte : kByte), mb ? 'M' : 'k', req.remote_address().to_string().c_str()); + ++monitor->data.num_del_success; } return write; } @@ -126,6 +189,7 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t rep = reply_t::stock_reply(reply_t::bad_request); ::lib_rtl_output(LIB_RTL_ERROR,"PUT: bad_request %-20s = %s", h.name.c_str(), h.value.c_str()); + ++monitor->data.num_put_bad_request; rep.headers.emplace_back(move(h)); } else if ( (hdr_len=req.header(http::constants::content_length)) == nullptr ) { @@ -133,6 +197,7 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t rep = reply_t::stock_reply(reply_t::bad_request); ::lib_rtl_output(LIB_RTL_ERROR,"PUT: bad_request %-20s = %s", h.name.c_str(), h.value.c_str()); + ++monitor->data.num_put_bad_request; rep.headers.emplace_back(move(h)); } else { @@ -149,6 +214,7 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t ::lib_rtl_output(LIB_RTL_ERROR,"PUT: %s %-20s = %s", reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); + ++monitor->data.num_put_unauthorized; rep.headers.emplace_back(move(h)); } else if ( !error_code_ok(ec, this->debug) ) { @@ -158,6 +224,7 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t ::lib_rtl_output(LIB_RTL_ERROR,"PUT: %s %-20s = %s", reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); + ++monitor->data.num_put_bad_request; rep.headers.emplace_back(move(h)); } else { @@ -171,8 +238,11 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t ::lib_rtl_output(LIB_RTL_INFO,"PUT: '%s' %.1f %cB [%s]", req.uri.c_str(), double(len)/(mb ? MByte : kByte), mb ? 'M' : 'k', req.remote_address().to_string().c_str()); + ++monitor->data.num_put_success; + return write; } } + ++monitor->data.num_put_errors; return write; } @@ -185,6 +255,7 @@ http::basic_http_server<db>::handler_t::handle_update(const request_t& req, repl rep = reply_t::stock_reply(reply_t::bad_request); ::lib_rtl_output(LIB_RTL_ERROR,"UPDATE: bad_request %-20s = %s", h.name.c_str(), h.value.c_str()); + ++monitor->data.num_upda_bad_request; rep.headers.emplace_back(move(h)); } else { @@ -199,9 +270,11 @@ http::basic_http_server<db>::handler_t::handle_update(const request_t& req, repl ::lib_rtl_output(LIB_RTL_ERROR,"UPDATE: %s %-20s = %s", reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); + ++monitor->data.num_upda_errors; rep.headers.emplace_back(move(h)); } else { + ++monitor->data.num_upda_success; rep = reply_t::stock_reply(reply_t::ok); rep.content.clear(); rep.headers.clear(); diff --git a/Online/Storage/src/server/fdb_fs_server.cpp b/Online/Storage/src/server/fdb_fs_server.cpp index eb5a0e6b4..d07012fac 100644 --- a/Online/Storage/src/server/fdb_fs_server.cpp +++ b/Online/Storage/src/server/fdb_fs_server.cpp @@ -15,14 +15,17 @@ // Framework include files #include <HTTP/HttpReply.h> -#include <Storage/fdb_server.h> #include <Storage/client.h> +#include <Storage/fdb_server.h> #include <boost/filesystem.hpp> +#include <RTL/strdef.h> +#include <dim/dis.h> // C/C++ include files #include <cstring> #include <unistd.h> +// Put private stuff in anonymous namespace namespace { /// Database object to check fs @@ -44,7 +47,36 @@ namespace { { return file_dir + object_name; } }; - + /// Monitor object to publish values + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.02.2021 + */ + struct fs_monitor { + public: + struct mon_data { + int numGET {0}; + int numPUT {0}; + int numDEL {0}; + int numGET_not_found {0}; + int numDEL_not_found {0}; + int numPUT_error {0}; + long bytesGET {0}; + long bytesPUT {0}; + long bytesDEL {0}; + } data; + + std::vector<int> services; + + void add(int svc) { services.emplace_back(svc); } + public: + /// Standard constructor + fs_monitor(); + /// Default destructor + ~fs_monitor(); + }; /// File I/O object /** @@ -132,11 +164,35 @@ namespace { class Online::storage::fs::traits { public: - typedef fs_db dbase_t; + typedef fs_db dbase_t; + typedef fs_monitor monitor_t; }; using namespace Online::storage; +/// Standard constructor +fs_monitor::fs_monitor() { + std::string nam = "/"+RTL::str_upper(RTL::nodeNameShort())+"/"+RTL::processName(); + add( ::dis_add_service((nam+"/DATA").c_str(),"C",&data,sizeof(data),0,0) ); + add( ::dis_add_service((nam+"/GET").c_str(),"L",&data.numGET,sizeof(data.numGET),0,0) ); + add( ::dis_add_service((nam+"/PUT").c_str(),"L",&data.numPUT,sizeof(data.numPUT),0,0) ); + add( ::dis_add_service((nam+"/DEL").c_str(),"L",&data.numDEL,sizeof(data.numDEL),0,0) ); + add( ::dis_add_service((nam+"/GET_bytes").c_str(),"L",&data.bytesGET,sizeof(data.bytesGET),0,0) ); + add( ::dis_add_service((nam+"/PUT_bytes").c_str(),"L",&data.bytesPUT,sizeof(data.bytesPUT),0,0) ); + add( ::dis_add_service((nam+"/DEL_bytes").c_str(),"L",&data.bytesDEL,sizeof(data.bytesDEL),0,0) ); + add( ::dis_add_service((nam+"/PUT_error").c_str(),"L", + &data.numPUT_error,sizeof(data.numPUT_error),0,0) ); + add( ::dis_add_service((nam+"/GET_not_found").c_str(),"L", + &data.numGET_not_found,sizeof(data.numGET_not_found),0,0) ); + add( ::dis_add_service((nam+"/DEL_not_found").c_str(),"L", + &data.numDEL_not_found,sizeof(data.numDEL_not_found),0,0) ); +} + +/// Default destructor +fs_monitor::~fs_monitor() { + for( auto svc : services ) ::dis_remove_service(svc); +} + /// Standard destructor template <> http::basic_http_server<fs>::handler_t::~handler_t() { } @@ -153,6 +209,7 @@ http::basic_http_server<fs>::handler_t::handle_get(const request_t& req, reply_t if ( -1 == ret_stat ) { rep = reply_t::stock_reply(reply_t::not_found); rep.headers.emplace_back(constants::error_cause,"File '"+fname+"' does not exist"); + ++monitor->data.numGET_not_found; return write; } rep = reply_t::stock_reply(reply_t::ok); @@ -172,6 +229,7 @@ http::basic_http_server<fs>::handler_t::handle_get(const request_t& req, reply_t if ( !error_code_ok(ec, this->debug) ) { rep = reply_t::stock_reply(reply_t::not_found); rep.headers.emplace_back(constants::error_cause,"File '"+fname+"' does not exist"); + ++monitor->data.numGET_not_found; } return write; } @@ -188,12 +246,12 @@ http::basic_http_server<fs>::handler_t::handle_get(const request_t& req, reply_t ctxt->st_data += rd; } else { - ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read data '%s' [%s] only got %ld out of %ld bytes", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read '%s' [%s] only got %ld out of %ld bytes", req.method.c_str(), req.uri.c_str(), ::strerror(errno), rd, len); } if ( ctxt->st_data == ctxt->st_size ) { - ::lib_rtl_output(LIB_RTL_INFO, "+++ %s: Prepared last chunk '%s' %.1f MB", + ::lib_rtl_output(LIB_RTL_INFO, "+++ %s: Last chunk '%s' %.1f MB", req.method.c_str(), req.uri.c_str(), double(ctxt->st_size)/(1024e0*1024e0)); } return write; @@ -207,9 +265,11 @@ http::basic_http_server<fs>::handler_t::handle_get(const request_t& req, reply_t req.method.c_str(), ctxt->st_size, net, mb/std::max(1e-6, double(net)/1e3), wrt, mb/std::max(1e-6, double(wrt)/1e3)); + ++monitor->data.numGET; + monitor->data.bytesGET += ctxt->st_data; return none; } - ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read data '%s' size:%ld / %ld %s [Inconsistemt context]", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read '%s' size:%ld / %ld %s [Inconsistemt context]", req.method.c_str(), req.uri.c_str(), rep.bytes_sent, rep.bytes_total, rep.stock_status(rep.status).c_str()); rep.context.reset(); @@ -229,6 +289,7 @@ http::basic_http_server<fs>::handler_t::handle_delete(const request_t& req, repl if ( -1 == ret_stat ) { rep = reply_t::stock_reply(reply_t::not_found); rep.headers.emplace_back(constants::error_cause,"File '"+fname+"' does not exist"); + ++monitor->data.numDEL_not_found; return write; } rep = reply_t::stock_reply(reply_t::ok); @@ -249,6 +310,7 @@ http::basic_http_server<fs>::handler_t::handle_delete(const request_t& req, repl if ( !error_code_ok(ec, this->debug) ) { rep = reply_t::stock_reply(reply_t::not_found); rep.headers.emplace_back(constants::error_cause,"File '"+fname+"' does not exist"); + ++monitor->data.numDEL_not_found; } return write; } @@ -265,7 +327,7 @@ http::basic_http_server<fs>::handler_t::handle_delete(const request_t& req, repl ctxt->st_data += rd; } else { - ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read data '%s' [%s] only got %ld out of %ld bytes", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read '%s' [%s] only got %ld out of %ld bytes", req.method.c_str(), req.uri.c_str(), ::strerror(errno), rd, len); } if ( ctxt->st_data == ctxt->st_size ) { @@ -274,7 +336,7 @@ http::basic_http_server<fs>::handler_t::handle_delete(const request_t& req, repl ec = ctxt->unlink(fname.c_str()); if ( !error_code_ok(ec, this->debug) ) { } - ::lib_rtl_output(LIB_RTL_INFO, "+++ %s: Prepared last chunk '%s' %.1f MB", + ::lib_rtl_output(LIB_RTL_INFO, "+++ %s: Last chunk '%s' %.1f MB", req.method.c_str(), req.uri.c_str(), double(ctxt->st_size)/(1024e0*1024e0)); } return write; @@ -288,9 +350,11 @@ http::basic_http_server<fs>::handler_t::handle_delete(const request_t& req, repl req.method.c_str(), ctxt->st_size, net, mb/std::max(1e-6, double(net)/1e3), wrt, mb/std::max(1e-6, double(wrt)/1e3)); + ++monitor->data.numDEL; + monitor->data.bytesDEL += ctxt->st_data; return none; } - ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read data '%s' size:%ld / %ld %s [Inconsistemt context]", + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read '%s' size:%ld / %ld %s [Inconsistemt context]", req.method.c_str(), req.uri.c_str(), rep.bytes_sent, rep.bytes_total, rep.stock_status(rep.status).c_str()); rep.context.reset(); @@ -312,16 +376,19 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t if ( 0 == ret_stat ) { rep = reply_t::stock_reply(reply_t::conflict); rep.headers.emplace_back(constants::error_cause,"Target already exists"); + ++monitor->data.numPUT_error; return write; } else if ( (hdr_date=req.header(constants::date)) == nullptr ) { rep = reply_t::stock_reply(reply_t::bad_request); rep.headers.emplace_back(constants::error_cause,"Missing date header [Protocol Violation]"); + ++monitor->data.numPUT_error; return write; } else if ( (hdr_len=req.header(constants::content_length)) == nullptr ) { rep = reply_t::stock_reply(reply_t::bad_request); rep.headers.emplace_back(constants::error_cause,"Missing content length header [Protocol Violation]"); + ++monitor->data.numPUT_error; return write; } boost::filesystem::path p = boost::filesystem::path(fname).parent_path(); @@ -346,6 +413,7 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t else rep = reply_t::stock_reply(reply_t::bad_request); rep.headers.emplace_back(constants::error_cause,err); + ++monitor->data.numPUT_error; return write; } } @@ -365,6 +433,7 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t if ( !error_code_ok(ec, this->debug) ) { rep = reply_t::stock_reply(reply_t::forbidden); rep.headers.emplace_back(constants::error_cause, "Failed to open file "+fname+" ["+ec.message()+"]"); + ++monitor->data.numPUT_error; } if ( req.content.empty() ) { return write; @@ -372,7 +441,6 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t // This should not happen: The client must first receive the continue! ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Received non-empty data on request to write " "'%s' [Protocol Violation]", req.method.c_str(), fname.c_str()); - //req.content.clear(); return write; } else if ( ctxt && ctxt->st_data < ctxt->st_size && req.content.empty() ) { @@ -384,12 +452,12 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t long wrt = ctxt->write(len, &req.content.at(0)); ctxt->st_data += wrt; if ( wrt != len ) { - ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to write data chunk '%s' " + ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to write '%s' " "[%s] only wrote %ld out of %ld bytes", req.method.c_str(), req.uri.c_str(), ::strerror(errno), wrt, len); + ++monitor->data.numPUT_error; } } - //req.content.clear(); return read; } else if ( ctxt && ctxt->st_data >= ctxt->st_size && rep.status != reply_t::created ) { @@ -397,12 +465,15 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t long net = std::chrono::duration_cast<std::chrono::milliseconds>(req.netio-null).count(); long wrt = std::chrono::duration_cast<std::chrono::milliseconds>(req.handling-null).count(); double mb = double(ctxt->st_size)/(1024e0*1024e0); - ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %s: Wrote data '%s' %.0f %cB", req.method.c_str(), + ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %s: Wrote '%s' %.0f %cB", req.method.c_str(), req.uri.c_str(), mb > 10e0 ? mb : mb*1024e0, mb > 10e0 ? 'M' : 'k'); - ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %s: %ld bytes netio: %ld ms %.1f MB/s wrt: %ld ms %.1f MB/s", + ::lib_rtl_output(LIB_RTL_ALWAYS, "+++ %s: %ld bytes netio: %ld ms %.1f MB/s wrt: %ld ms %.1f MB/s", req.method.c_str(), ctxt->st_size, net, mb/std::max(1e-6, double(net)/1e3), wrt, mb/std::max(1e-6, double(wrt)/1e3)); + ++monitor->data.numPUT; + monitor->data.bytesPUT += ctxt->st_size; + rep.context.reset(); rep = reply_t::stock_reply(reply_t::created); rep.headers.clear(); @@ -422,6 +493,7 @@ http::basic_http_server<fs>::handler_t::handle_put(const request_t& req, reply_t return write; } } + ++monitor->data.numPUT_error; ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: FAILED to update '%s' to state WRITTEN!", req.method.c_str(), req.uri.c_str()); return write; @@ -457,9 +529,12 @@ namespace http { } extern "C" int fdb_fs_file_server(int argc, char** argv) { + std::string srv = "/"+RTL::str_upper(RTL::nodeNameShort())+"/"+RTL::processName(); SrvRun<fs> s; s.create(argc, argv); s.ptr->implementation->dbase.reset(new fs_db(s.server, s.file_dir)); + s.ptr->implementation->monitor.reset(new fs_monitor()); + ::dis_start_serving(srv.c_str()); s.start(); return 0; } -- GitLab From 9ba15756ebf154af18e2d510a5cdbb09623b5615 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Wed, 14 Apr 2021 19:14:24 +0200 Subject: [PATCH 16/26] Fix to fifo logger. Fix storage file server --- Online/Dataflow/src/framework/DiskReader.cpp | 16 ++- Online/FarmConfig/job/Controller.sh | 3 +- Online/HTTP/HTTP/HttpServer.h | 19 ++- Online/HTTP/src/HttpServer.cpp | 5 + Online/OnlineBase/CMakeLists.txt | 9 ++ Online/OnlineBase/src/LOG/FifoLog.cpp | 113 +++++++++++------- Online/OnlineBase/src/RTL/Logger.cpp | 24 +++- Online/OnlineKernel/CMakeLists.txt | 6 - Online/OnlineKernel/main/checkpoint.cpp | 21 ---- Online/OnlineKernel/main/gentest.cpp | 52 -------- Online/OnlineKernel/main/mbm_remove.cpp | 28 ----- Online/OnlineKernel/main/wt.cpp | 31 ----- Online/PCIE40Data/src/pcie40decoder.cpp | 29 ++++- Online/ROLogger/kafka/OutputListener.cpp | 19 ++- Online/ROLogger/kafka/OutputListener.h | 1 + Online/ROLogger/kafka/OutputLogger.cpp | 6 + .../SmiController/scripts/EventProcessor.py | 3 +- .../SmiController/scripts/EventProcessor.sh | 3 +- Online/SmiController/scripts/MBMMonMulti.sh | 1 + Online/SmiController/scripts/MBMMonSingle.sh | 1 + Online/SmiController/scripts/RCVMon.sh | 1 + Online/SmiController/scripts/TestMEPProd.sh | 9 ++ Online/SmiController/scripts/configure.sh | 7 +- Online/SmiController/scripts/runDummyTask.sh | 2 +- Online/SmiController/scripts/runTask.sh | 8 +- Online/Storage/CMakeLists.txt | 9 +- Online/Storage/src/server/fdb_db_server.cpp | 55 +++++---- Online/Storage/src/server/fdb_dbase.cpp | 28 ++--- Online/Storage/src/server/fdb_fs_server.cpp | 54 +++++++-- Online/Tell1Data/src/RawFile.cpp | 6 +- Online/TestBeam/job/Controller.sh | 4 +- Online/TestBeam/job/runTask.sh | 6 +- 32 files changed, 317 insertions(+), 262 deletions(-) delete mode 100644 Online/OnlineKernel/main/checkpoint.cpp delete mode 100755 Online/OnlineKernel/main/gentest.cpp delete mode 100755 Online/OnlineKernel/main/mbm_remove.cpp delete mode 100755 Online/OnlineKernel/main/wt.cpp diff --git a/Online/Dataflow/src/framework/DiskReader.cpp b/Online/Dataflow/src/framework/DiskReader.cpp index 8fc1e2685..c61a97b76 100644 --- a/Online/Dataflow/src/framework/DiskReader.cpp +++ b/Online/Dataflow/src/framework/DiskReader.cpp @@ -482,12 +482,24 @@ DiskReader::load_event_data(RawFile::Allocator& allocator, RawFile& file) { return { size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE }; } mdf_hdr = (EventHeader*)data_ptr; - if ( mdf_hdr->recordSize() + data_size > alloc_size ) { + if ( !mdf_hdr->is_mdf() ) { + warning("Corrupted file found: %s at position %ld. Skip %ld bytes.", + file.cname(), position, long(file.data_size())-file.position()); + file.reset(true); + return { size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE }; + } + else if ( mdf_hdr->recordSize() + data_size > alloc_size ) { file.position(position); return { size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE }; } status = file.read(data_ptr + peek, mdf_hdr->recordSize() - peek); - if ( status < mdf_hdr->recordSize() - peek ) { + if ( status < 0 ) { + warning("Corrupted file found: %s at position %ld. Skip %ld bytes.", + file.cname(), position, long(file.data_size())-file.position()); + file.reset(true); + return { size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE }; + } + else if ( status < (mdf_hdr->recordSize() - peek) ) { file.reset(true); return { size_t(data_size), num_evts, RawFile::MDF_INPUT_TYPE }; } diff --git a/Online/FarmConfig/job/Controller.sh b/Online/FarmConfig/job/Controller.sh index 28305cd5b..4e89e04a7 100755 --- a/Online/FarmConfig/job/Controller.sh +++ b/Online/FarmConfig/job/Controller.sh @@ -10,7 +10,7 @@ # ========================================================================= HOST=`hostname -s | tr a-z A-Z`; HOST_LONG=`hostname -f | tr a-z A-Z`; -echo "ERROR DIM_DNS_NODE: ${DIM_DNS_NODE}" +# echo "ERROR DIM_DNS_NODE: ${DIM_DNS_NODE}" # #if test "${HOST}" = "N3080501"; then # export SMI_DNS=devpvsscc7; @@ -42,6 +42,7 @@ if test -z "${DIM_DNS_NODE}"; then export DIM_DNS_NODE=${HOST_LONG}; fi; # +# exec -a ${UTGID} `which gentest.exe` libSmiController.so smi_controller \ -print=4 \ -logger=fifo \ diff --git a/Online/HTTP/HTTP/HttpServer.h b/Online/HTTP/HTTP/HttpServer.h index 196877edf..b54383d2d 100644 --- a/Online/HTTP/HTTP/HttpServer.h +++ b/Online/HTTP/HTTP/HttpServer.h @@ -281,9 +281,10 @@ namespace http { typedef std::vector<std::shared_ptr<std::thread> > Threads; enum continue_action { none, write, read, close }; - - typedef HttpRequest request_t; - typedef HttpReply reply_t; + + typedef std::atomic<long> counter_t; + typedef HttpRequest request_t; + typedef HttpReply reply_t; /// Object Lock to ensure we have no race conditions when editing the call-map std::mutex lock; @@ -299,10 +300,18 @@ namespace http { HttpConnectionManager manager; /// The next socket to be accepted. boost::asio::ip::tcp::socket socket; + /// Number of currently active connections + counter_t num_connections {0}; + /// Number of ever opened connections + counter_t num_connections_opened {0}; + /// Number of ever closed connections + counter_t num_connections_closed {0}; + /// Number of ever stopped connections + counter_t num_connections_stopped {0}; /// Default buffer size of the connection - size_t buffer_size = 16*1024; + size_t buffer_size {16*1024}; /// Enable debugging if necessary - int debug = 0; + int debug {0}; protected: /// Perform an asynchronous accept operation. diff --git a/Online/HTTP/src/HttpServer.cpp b/Online/HTTP/src/HttpServer.cpp index 45c464f6e..89109effd 100644 --- a/Online/HTTP/src/HttpServer.cpp +++ b/Online/HTTP/src/HttpServer.cpp @@ -55,10 +55,14 @@ HttpConnection::HttpConnection(asio::ip::tcp::socket s, request.start = system_clock::now(); request.netio = std::chrono::time_point<system_clock>(system_clock::duration::zero()); request.handling = std::chrono::time_point<system_clock>(system_clock::duration::zero()); + ++handler.num_connections_opened; + ++handler.num_connections; } /// Default destructor HttpConnection::~HttpConnection() { + ++handler.num_connections_closed; + --handler.num_connections; if ( handler.debug ) { std::cout << "HttpConnection: DESTRUCT." << std::endl << std::flush; } @@ -94,6 +98,7 @@ void HttpConnection::stop() { if ( socket.is_open() ) { system::error_code ec; socket.close(ec); + ++handler.num_connections_stopped; if ( ec ) { std::cout << "HttpConnection: STOP: " << ec.message() << std::endl << std::flush; } diff --git a/Online/OnlineBase/CMakeLists.txt b/Online/OnlineBase/CMakeLists.txt index 6e788c0bb..5d8ce0fb5 100755 --- a/Online/OnlineBase/CMakeLists.txt +++ b/Online/OnlineBase/CMakeLists.txt @@ -59,6 +59,15 @@ gaudi_add_python_module(_fifo_log # # Need to cast out warnings created by Python.h target_compile_options(_fifo_log PRIVATE -Wno-register) +#=============================================================================== +gaudi_add_executable(gentest main/gentest.cpp LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt) +gaudi_add_executable(genRunner main/genRunner.cpp LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt) +gaudi_add_executable(genPython main/genPython.cpp LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt ${PYTHON_LIBRARIES} ) +gaudi_add_executable(mbm_remove main/mbm_remove.cpp LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt) +gaudi_add_executable(checkpoint main/checkpoint.cpp LINK_LIBRARIES OnlineBase ${CMAKE_DL_LIBS} -lrt) +# +gaudi_add_executable(onlinekernel_test main/wt.cpp LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt) + #=============================================================================== # gaudi_install_python_modules() diff --git a/Online/OnlineBase/src/LOG/FifoLog.cpp b/Online/OnlineBase/src/LOG/FifoLog.cpp index 719c4c082..2556d3ccd 100644 --- a/Online/OnlineBase/src/LOG/FifoLog.cpp +++ b/Online/OnlineBase/src/LOG/FifoLog.cpp @@ -48,6 +48,8 @@ public: /// Append (unlocked) single character to output buffer void append(int stream_id, char c) { switch(c) { + case '\0': + case '\r': case '\n': flush_line(stream_id); break; @@ -96,7 +98,7 @@ public: int push(int stream_id, const wchar_t* format, LOGGER_VA_LIST args) { wchar_t text[4096]; int len = ::vswprintf(text, sizeof(text), format, args); - text[sizeof(text)-1] = 0; + text[sizeof(text)/sizeof(wchar_t)-1] = 0; va_end(args); this->push(stream_id, text, len*sizeof(wchar_t)); return len; @@ -263,7 +265,7 @@ using namespace fifolog; typedef Logger::Implementation logger_imp; string logger_imp::fifo_name; -string logger_imp::systag; +string logger_imp::systag = "SYSTEM"; string logger_imp::utgid; /// Anonymous namespace for helpers @@ -285,8 +287,8 @@ namespace { } inline logger_imp& log() { - //static LibCLogger s_log; - static PollLogger s_log; + static LibCLogger s_log; + //static PollLogger s_log; return s_log; } @@ -315,8 +317,8 @@ namespace { if ( fifo.empty() ) { fifo = ::getenv("LOGFIFO") ? ::getenv("LOGFIFO") : ""; } - if ( tag.empty() ) { - tag = ::getenv("PARTITION_NAME") ? ::getenv("PARTITION_NAME") : ""; + if ( tag.empty() || tag == "SYSTEM" ) { + tag = ::getenv("PARTITION_NAME") ? ::getenv("PARTITION_NAME") : tag; } if ( tag.empty() ) { tag = "SYSTEM"; @@ -569,7 +571,7 @@ void LibCLogger::init(const string& f, const string& u, const string& t) { ::close(this->fifo_fd); throw runtime_error(text); } - this->push(STDOUT,"LIBC logger initialized!!!\n"); + //this->push(STDOUT,"LIBC logger initialized!!!\n"); } } @@ -729,7 +731,7 @@ extern "C" void fifolog_set_utgid(const char* new_utgid) { /// Set new tag value extern "C" void fifolog_set_tag(const char* new_tag) { auto& log = logger(false); - logger_imp::systag = new_tag ? new_tag : "SYSTEM"; + logger_imp::systag = new_tag && ::strlen(new_tag)>0 ? new_tag : "SYSTEM"; if ( log.get() ) log->imp->rebuild_trailer(); } @@ -740,6 +742,10 @@ extern "C" int fifolog_set_fifo(const char* fifo) { logger_imp::fifo_name = fifo; return 1; } + else if ( log.get() ) { + log->imp->rebuild_trailer(); + return 1; + } ::fprintf(stderr,"Failed to set output fifo: %s [%s] logger %s initialized.\n", fifo, ::strerror(errno), log.get() ? "already" : "NOT"); errno = ENOENT; @@ -756,7 +762,7 @@ extern "C" void fifolog_finalize_logger() { } } -extern "C" int fputc(int c, FILE *stream) { +extern "C" int local__fputc(int c, FILE *stream) { if ( logger(false).get() ) { if ( stream == stdout ) return log().push(logger_imp::STDOUT, c); @@ -766,15 +772,15 @@ extern "C" int fputc(int c, FILE *stream) { return funcs().real_fputc(c, stream); } -extern "C" int putc(int c, FILE *stream) { +extern "C" int local__putc(int c, FILE *stream) { return ::fputc(c, stream); } -extern "C" int putchar(int c) { +extern "C" int local__putchar(int c) { return ::fputc(c, stdout); } -extern "C" int fputs(const char *str, FILE *stream) { +extern "C" int local__fputs(const char *str, FILE *stream) { if ( logger(false).get() ) { if ( stream == stdout ) return log().push(logger_imp::STDOUT, str); @@ -784,11 +790,11 @@ extern "C" int fputs(const char *str, FILE *stream) { return funcs().real_fputs(str, stream); } -extern "C" int puts(const char* str) { +extern "C" int local__puts(const char* str) { return ::fputs(str, stdout); } -extern "C" ssize_t write(int fd, const void *buf, size_t count) { +extern "C" ssize_t local__write(int fd, const void *buf, size_t count) { if ( logger(false).get() ) { switch(fd) { case STDOUT_FILENO: return log().push(logger_imp::STDOUT, buf, count); @@ -799,7 +805,7 @@ extern "C" ssize_t write(int fd, const void *buf, size_t count) { return funcs().real_write(fd, buf, count); } -extern "C" size_t fwrite(const void *str, size_t size, size_t nmemb, FILE *stream) { +extern "C" size_t local__fwrite(const void *str, size_t size, size_t nmemb, FILE *stream) { if ( logger(false).get() ) { if ( stream == stdout ) return log().push(logger_imp::STDOUT, str, size*nmemb); @@ -809,19 +815,19 @@ extern "C" size_t fwrite(const void *str, size_t size, size_t nmemb, FILE *strea return funcs().real_fwrite(str, size, nmemb, stream); } -extern "C" int vprintf(const char *format, LOGGER_VA_LIST args) { +extern "C" int local__vprintf(const char *format, LOGGER_VA_LIST args) { return logger(false).get() ? log().push(logger_imp::STDOUT, format, args) : funcs().real_vprintf(format, args); } -extern "C" int vwprintf(const wchar_t *format, LOGGER_VA_LIST args) { +extern "C" int local__vwprintf(const wchar_t *format, LOGGER_VA_LIST args) { return logger(false).get() ? log().push(logger_imp::STDOUT, format, args) : funcs().real_vwprintf(format, args); } -extern "C" int vdprintf(int fd, const char *format, LOGGER_VA_LIST args) { +extern "C" int local__vdprintf(int fd, const char *format, LOGGER_VA_LIST args) { if ( logger(false).get() ) { switch(fd) { case STDOUT_FILENO: return log().push(logger_imp::STDOUT, format, args); @@ -832,7 +838,7 @@ extern "C" int vdprintf(int fd, const char *format, LOGGER_VA_LIST args) { return funcs().real_vdprintf(fd, format, args); } -extern "C" int vfprintf(FILE* stream, const char *format, LOGGER_VA_LIST args) { +extern "C" int local__vfprintf(FILE* stream, const char *format, LOGGER_VA_LIST args) { if ( logger(false).get() ) { if ( stream == stdout ) return log().push(logger_imp::STDOUT, format, args); @@ -842,21 +848,7 @@ extern "C" int vfprintf(FILE* stream, const char *format, LOGGER_VA_LIST args) return funcs().real_vfprintf(stream, format, args); } -extern "C" int __vfprintf_chk(FILE* stream, int flag, const char *format, LOGGER_VA_LIST args) { - if ( logger(false).get() ) { - if ( stream == stdout ) - return log().push(logger_imp::STDOUT, format, args); - else if ( stream == stderr ) - return log().push(logger_imp::STDERR, format, args); - } - return funcs().real_vfprintf_chk(stream, flag, format, args); -} - -extern "C" int __vprintf_chk(int flag, const char *format, LOGGER_VA_LIST args) { - return __vfprintf_chk(stdout, flag, format, args); -} - -extern "C" int printf(const char *format, ...) { +extern "C" int local__printf(const char *format, ...) { LOGGER_VA_LIST args; va_start(args, format); int ret = ::vprintf(format, args); @@ -864,7 +856,7 @@ extern "C" int printf(const char *format, ...) { return ret; } -extern "C" int wprintf(const wchar_t *format, ...) { +extern "C" int local__wprintf(const wchar_t *format, ...) { LOGGER_VA_LIST args; va_start(args, format); int ret = ::vwprintf(format, args); @@ -872,7 +864,7 @@ extern "C" int wprintf(const wchar_t *format, ...) { return ret; } -extern "C" int dprintf(int fd, const char *format, ...) { +extern "C" int local__dprintf(int fd, const char *format, ...) { LOGGER_VA_LIST args; va_start(args, format); int ret = ::vdprintf(fd, format, args); @@ -880,7 +872,7 @@ extern "C" int dprintf(int fd, const char *format, ...) { return ret; } -extern "C" int fprintf(FILE* stream, const char *format, ...) { +extern "C" int local__fprintf(FILE* stream, const char *format, ...) { LOGGER_VA_LIST args; va_start(args, format); int ret = ::vfprintf(stream, format, args); @@ -888,18 +880,57 @@ extern "C" int fprintf(FILE* stream, const char *format, ...) { return ret; } -extern "C" int __printf_chk(int flag, const char *format, ...) { + +extern "C" int local__vfprintf_chk(FILE* stream, int flag, const char *format, LOGGER_VA_LIST args) { + if ( logger(false).get() ) { + if ( stream == stdout ) + return log().push(logger_imp::STDOUT, format, args); + else if ( stream == stderr ) + return log().push(logger_imp::STDERR, format, args); + } + return funcs().real_vfprintf_chk(stream, flag, format, args); +} + +extern "C" int local__vprintf_chk(int flag, const char *format, LOGGER_VA_LIST args) { + return local__vfprintf_chk(stdout, flag, format, args); +} + +extern "C" int local__printf_chk(int flag, const char *format, ...) { LOGGER_VA_LIST args; va_start(args, format); - int ret = __vprintf_chk(flag, format, args); + int ret = local__vprintf_chk(flag, format, args); va_end(args); return ret; } -extern "C" int __fprintf_chk(FILE* stream, int flag, const char *format, ...) { +extern "C" int local__fprintf_chk(FILE* stream, int flag, const char *format, ...) { LOGGER_VA_LIST args; va_start(args, format); - int ret = __vfprintf_chk(stream, flag, format, args); + int ret = local__vfprintf_chk(stream, flag, format, args); va_end(args); return ret; } + + +__asm__ (".global fputc; .type fputc,@function; fputc = local__fputc"); +//__asm__ (".global putc; .type putc,@function; putc = local__ putc"); +//__asm__ (".global putchar; .type putchar,@function; putchar = local__ putchar"); +__asm__ (".global fputs; .type fputs,@function; fputs = local__fputs"); +__asm__ (".global write; .type write,@function; write = local__write"); +__asm__ (".global fputs; .type fputs,@function; fputs = local__fputs"); +__asm__ (".global puts; .type puts,@function; puts = local__puts"); +__asm__ (".global fwrite; .type fwrite,@function; fwrite = local__fwrite"); + +__asm__ (".global vprintf; .type vprintf,@function; vprintf = local__vprintf"); +__asm__ (".global vwprintf; .type vwprintf,@function; vwprintf = local__vwprintf"); +__asm__ (".global vdprintf; .type vdprintf,@function; vdprintf = local__vdprintf"); +__asm__ (".global vfprintf; .type vfprintf,@function; vfprintf = local__vfprintf"); +__asm__ (".global printf; .type printf,@function; printf = local__printf"); +__asm__ (".global wprintf; .type wprintf,@function; wprintf = local__wprintf"); +__asm__ (".global dprintf; .type dprintf,@function; dprintf = local__dprintf"); +__asm__ (".global fprintf; .type fprintf,@function; fprintf = local__fprintf"); +__asm__ (".global vfprintf_chk; .type vfprintf_chk,@function; vfprintf_chk = local__vfprintf_chk"); +__asm__ (".global __vfprintf_chk; .type __vfprintf_chk,@function; __vfprintf_chk = local__vfprintf_chk"); +__asm__ (".global __printf_chk; .type __printf_chk,@function; __printf_chk = local__printf_chk"); +__asm__ (".global __fprintf_chk; .type __fprintf_chk,@function; __fprintf_chk = local__fprintf_chk"); +__asm__ (".global __vfprintf_chk; .type __vfprintf_chk,@function; __vfprintf_chk = local__vfprintf_chk"); diff --git a/Online/OnlineBase/src/RTL/Logger.cpp b/Online/OnlineBase/src/RTL/Logger.cpp index 175b47352..99d40ed0c 100644 --- a/Online/OnlineBase/src/RTL/Logger.cpp +++ b/Online/OnlineBase/src/RTL/Logger.cpp @@ -231,13 +231,24 @@ size_t Logger::LogDevice::formatHeader(char* buffer, size_t buff_len, int severi /// Calls the display action with a given severity level size_t Logger::LogDevice::printmsg(int severity, const char* source, const char* msg) const { - char header[2048]; - size_t len = formatHeader(header, sizeof(header), severity, source); { - lock_guard<std::mutex> lock(*device_lock); - len = ::fprintf(stdout, "%s %s\n", header, msg); - ::fflush(stdout); + char text[4096], header[2048]; + size_t len1 = formatHeader(header, sizeof(header), severity, source); + size_t len2 = ::strlen(msg); + if ( len1+len2+3 < sizeof(text) ) { + ::memcpy(text, header, len1); + text[len1] = ' '; + ::memcpy(text+len1+1, msg, len2); + size_t len = len1+len2+1; + if ( msg[len2-1] != '\n' && msg[len2-2] != '\n' ) { + text[len] = '\n'; + ++len; + } + { + lock_guard<std::mutex> lock(*device_lock); + return ::write(STDOUT_FILENO, text, len); + } } - return len+1; // '1' from fputc + return 0; // '1' from fputc } /// Calls the display action with a given severity level @@ -326,6 +337,7 @@ void Logger::install_fifolog(const char* fifo) { void Logger::install_fifolog(const char* fifo, const log_args& args) { if ( fifo && fifo[0] ) { if ( 0 == ::fifolog_set_fifo(fifo) ) ::exit(1); + ::fifolog_set_utgid(RTL::processName().c_str()); ::fifolog_initialize_logger(); } if ( args.debug ) diff --git a/Online/OnlineKernel/CMakeLists.txt b/Online/OnlineKernel/CMakeLists.txt index fa40ee047..78d9c995b 100644 --- a/Online/OnlineKernel/CMakeLists.txt +++ b/Online/OnlineKernel/CMakeLists.txt @@ -35,12 +35,6 @@ gaudi_add_dictionary(OnlineKernel dict/dictionary.h dict/dictionary.xml INCLUDE_DIRS PythonLibs LINK_LIBRARIES OnlineKernel PythonLibs) # -gaudi_add_executable(gentest main/gentest.cpp LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt) -gaudi_add_executable(checkpoint main/checkpoint.cpp LINK_LIBRARIES OnlineBase ${CMAKE_DL_LIBS} -lrt) -gaudi_add_executable(mbm_remove main/mbm_remove.cpp LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt) -# -gaudi_add_executable(onlinekernel_test main/wt.cpp LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt) -# # Disable generation of ConfUserDB (must be done before gaudi_install_python_modules) set_directory_properties(PROPERTIES CONFIGURABLE_USER_MODULES None) # diff --git a/Online/OnlineKernel/main/checkpoint.cpp b/Online/OnlineKernel/main/checkpoint.cpp deleted file mode 100644 index a9d494bd5..000000000 --- a/Online/OnlineKernel/main/checkpoint.cpp +++ /dev/null @@ -1,21 +0,0 @@ -//========================================================================== -// LHCb Online software suite -//-------------------------------------------------------------------------- -// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) -// All rights reserved. -// -// For the licensing terms see OnlineSys/LICENSE. -// -//-------------------------------------------------------------------------- -// -// Author : Markus Frank -//========================================================================== -#define main checkpointing_main -#include "gentest.cpp" -#undef main -extern void CheckpointRestoreWrapper__init_instance(int argc, char** argv); - -int main (int argc, char** argv) { - CheckpointRestoreWrapper__init_instance(argc, argv); - return checkpointing_main(argc,argv); -} diff --git a/Online/OnlineKernel/main/gentest.cpp b/Online/OnlineKernel/main/gentest.cpp deleted file mode 100755 index 99988aab4..000000000 --- a/Online/OnlineKernel/main/gentest.cpp +++ /dev/null @@ -1,52 +0,0 @@ -//========================================================================== -// LHCb Online software suite -//-------------------------------------------------------------------------- -// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) -// All rights reserved. -// -// For the licensing terms see OnlineSys/LICENSE. -// -//-------------------------------------------------------------------------- -// -// Author : Markus Frank -//========================================================================== -#include "RTL/DllAccess.h" -#include <iostream> -#include <cerrno> - -/// Generic main program to start any process -/** - * Generic main program to start any process - * by loading a library and executing a - * function within - * - * \author M.Frank - * \version 1.0 - */ -int main (int argc, char** argv) { - if ( argc >= 3 ) { - void* handle = LOAD_LIB( argv[1] ); - if ( handle ) { - Function fun(GETPROC(handle, argv[2])); - if ( fun.function ) { - return (*fun.function)(argc-2, &argv[2]); - } - // The required function is not in the loaded image. - // Try to load it directly from the executable - Function f1(GETPROC(0, argv[2])); - if ( f1.function ) { - std::cout << "[WARNING] Executing function " << argv[2] - << " from exe rather than image " << std::endl; - return (*f1.function)(argc-2, &argv[2]); - } - std::cout << "[ERROR] Failed to access test procedure: '" << argv[2] << "'" << std::endl; - std::cout << "[ERROR] Error: " << DLERROR << std::endl; - return EINVAL; - } - std::cout << "[ERROR] Failed to load test library: '" << argv[1] << "'" << std::endl; - std::cout << "[ERROR] Error: " << DLERROR << std::endl; - return EINVAL; - } - std::cout << "[ERROR] usage: gentest.exe <library-name> <function-name> -arg [-arg]" << std::endl; - return EINVAL; -} diff --git a/Online/OnlineKernel/main/mbm_remove.cpp b/Online/OnlineKernel/main/mbm_remove.cpp deleted file mode 100755 index bbc209c55..000000000 --- a/Online/OnlineKernel/main/mbm_remove.cpp +++ /dev/null @@ -1,28 +0,0 @@ -//========================================================================== -// LHCb Online software suite -//-------------------------------------------------------------------------- -// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) -// All rights reserved. -// -// For the licensing terms see OnlineSys/LICENSE. -// -//-------------------------------------------------------------------------- -// -// Author : Markus Frank -//========================================================================== -#include "RTL/DllAccess.h" -#include <iostream> - -int main (int argc, char** argv) { - void* handle = LOAD_LIB2( "OnlineKernel" ); - if ( 0 != handle ) { - Function fun(GETPROC(handle,"mbm_remove")); - if ( fun.function ) { - return (*fun.function)(argc, argv); - } - std::cout << "Failed to access test procedure!" << std::endl; - } - std::cout << "Failed to load test library!" << std::endl; - std::cout << "Error: " << DLERROR << std::endl; - return 0; -} diff --git a/Online/OnlineKernel/main/wt.cpp b/Online/OnlineKernel/main/wt.cpp deleted file mode 100755 index e6a77bf30..000000000 --- a/Online/OnlineKernel/main/wt.cpp +++ /dev/null @@ -1,31 +0,0 @@ -//========================================================================== -// LHCb Online software suite -//-------------------------------------------------------------------------- -// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) -// All rights reserved. -// -// For the licensing terms see OnlineSys/LICENSE. -// -//-------------------------------------------------------------------------- -// -// Author : Markus Frank -//========================================================================== -#include "RTL/DllAccess.h" -#include <iostream> - -using OnlineBase::FuncPtrCast; - -int main (int argc, char** argv) { - void* handle = LOAD_LIB2("OnlineKernel"); - if ( 0 != handle ) { - typedef int (*func_t)(int, char**); - func_t fun = FuncPtrCast<func_t>(GETPROC(handle, argv[1])); - if ( fun ) { - return (*fun)(argc-1, &argv[1]); - } - std::cout << "Failed to access test procedure!" << std::endl; - } - std::cout << "Failed to load test library!" << std::endl; - std::cout << "Error: " << DLERROR << std::endl; - return 0; -} diff --git a/Online/PCIE40Data/src/pcie40decoder.cpp b/Online/PCIE40Data/src/pcie40decoder.cpp index 0f09bfab7..899a9beb2 100644 --- a/Online/PCIE40Data/src/pcie40decoder.cpp +++ b/Online/PCIE40Data/src/pcie40decoder.cpp @@ -355,14 +355,38 @@ void decoder_t::decode(std::unique_ptr<event_collection_t>& ev_coll, size_t num_err_packing = 0; size_t num_fragments = mep->num_source; if ( num_fragments > 0 ) { + const multi_fragment_t *last_mfp = nullptr; size_t packing = mep->multi_fragment(0)->packingFactor(); auto ec = (event_collection_writer_t*)initialize(ev_coll, packing); ec->mep = mep; ec->length = packing; for(uint32_t i=0; i<num_fragments; ++i) { - const multi_fragment_t *mfp = mep->multi_fragment(i); - const multi_fragment_t::header_t& hdr = mfp->header; + const auto* mfp = mep->multi_fragment(i); + const auto& hdr = mfp->header; uint16_t src_id = hdr.source_id; + if ( !mfp->is_valid() ) { + implementation->print(ERROR, + "+++ SKIP CORRUPTED MFP[%2d] at %p len:%6d %p magic:%04X " + "pack:%3d eid:%8ld src:%3d vsn:%d align:%d\n", + i, (void*)mfp, int(hdr.size), + (void*)((char*)mfp + hdr.size), + int(hdr.magic), int(hdr.packing), long(hdr.event_id), + int(src_id), int(hdr.version), + int(hdr.alignment)); + if ( last_mfp ) { + const auto& lhdr = last_mfp->header; + uint16_t lsrc_id = lhdr.source_id; + implementation->print(ERROR, + "+++ LAST GOOD MFP[%2d] at %p len:%6d %p magic:%04X " + "pack:%3d eid:%8ld src:%3d vsn:%d align:%d\n", + i, (void*)last_mfp, int(lhdr.size), + (void*)((char*)last_mfp + lhdr.size), + int(lhdr.magic), int(lhdr.packing), long(lhdr.event_id), + int(lsrc_id), int(lhdr.version), + int(lhdr.alignment)); + } + continue; + } if ( debug ) { implementation->print(PRINT, "+++ Reading mfp[%2d] at %p len:%6d %p magic:%04X " @@ -381,6 +405,7 @@ void decoder_t::decode(std::unique_ptr<event_collection_t>& ev_coll, if ( !source_selector || source_selector(src_id¶ms::sourceid_TOP5) ) { num_banks += implementation->link_fragment(ec, mfp, debug); } + last_mfp = mfp; } if ( num_err_packing > 0 ) { implementation->print(ERROR, diff --git a/Online/ROLogger/kafka/OutputListener.cpp b/Online/ROLogger/kafka/OutputListener.cpp index 1eea5d071..bea09ee24 100644 --- a/Online/ROLogger/kafka/OutputListener.cpp +++ b/Online/ROLogger/kafka/OutputListener.cpp @@ -129,12 +129,12 @@ void OutputListener::print_msg(const char* time_stamp, time_stamp ? time_stamp : "", host ? host : ""); if ( counter ) - len += ::snprintf(text+len,sizeof(text)-len,"%9s%-7s",counter,tag); + len += ::snprintf(text+len,sizeof(text)-len,"%-9s%-7s",counter,tag); else len += ::snprintf(text+len,sizeof(text)-len,"%-7s",tag); } len += ::snprintf(text+len,sizeof(text)-len,"%-34s%s\n", - utgid ? utgid : "", message ? message : ""); + utgid ? utgid : "", message ? message : ""); if ( have_colors ) { graphics::white(); graphics::plain(); @@ -164,8 +164,15 @@ void OutputListener::handle_payload(const char* /* topic */, int match = *(long*)payload == PAYLOAD_MATCH; if ( match ) { + std::string raw_msg; *(time_stamp+10) = ' '; *(time_stamp+19) = 0; + if ( print_raw ) { + raw_msg.clear(); + raw_msg.reserve(plen+1); + raw_msg.insert(raw_msg.end(), payload, payload+plen); + raw_msg.push_back('\n'); + } char* message = mark_start(time_stamp+20, MESSAGE_TAG, m_tag_len); char* msg_end = mark_end(message); size_t len = msg_end-message-3; @@ -239,7 +246,13 @@ void OutputListener::handle_payload(const char* /* topic */, } } if ( match_utgid && match_host ) { - print_msg(time_stamp, message, systag, utgid, host, physical, counter); + if ( print_raw ) { + ::fwrite(raw_msg.c_str(), 1, raw_msg.length(), stdout); + ::fflush(stdout); + } + else { + print_msg(time_stamp, message, systag, utgid, host, physical, counter); + } } if ( msg_end ) *msg_end = '\"'; if ( phys_end ) *phys_end = '\"'; diff --git a/Online/ROLogger/kafka/OutputListener.h b/Online/ROLogger/kafka/OutputListener.h index cb9c1fcc3..6d0e33346 100644 --- a/Online/ROLogger/kafka/OutputListener.h +++ b/Online/ROLogger/kafka/OutputListener.h @@ -43,6 +43,7 @@ namespace kafka { Matches node_match; Matches utgid_match; Matches message_veto; + bool print_raw = false; bool print_stats = false; bool print_counter = false; bool simple_msg = false; diff --git a/Online/ROLogger/kafka/OutputLogger.cpp b/Online/ROLogger/kafka/OutputLogger.cpp index 848241601..c1a5ae161 100644 --- a/Online/ROLogger/kafka/OutputLogger.cpp +++ b/Online/ROLogger/kafka/OutputLogger.cpp @@ -51,6 +51,7 @@ namespace { " -b <broker> Set DIM message broker. \n" " -O <level> Set RTL output level (1...4). \n" " -P Print statistics \n" + " -R Print RAW messages \n" " -m <reg-ex> Add node matching regular expression. \n" " -V <reg-ex> Veto messages matching regular expression.\n"); return 0; @@ -69,6 +70,7 @@ extern "C" int run_output_logger (int argc, char **argv) { bool print_help = false; bool print_stats = false; bool print_counter = false; + bool print_raw = false; bool simple_msg = false; bool have_colors = false; size_t rows = 0, cols = 0; @@ -105,6 +107,9 @@ extern "C" int run_output_logger (int argc, char **argv) { case 'O': output_level = ::atol(argv[++i]); break; + case 'R': + print_raw = true; + break; case 'V': value = argv[++i]; ::lib_rtl_output(LIB_RTL_ALWAYS,"+++ Adding message veto: %s\n",value.c_str()); @@ -162,6 +167,7 @@ extern "C" int run_output_logger (int argc, char **argv) { listener.message_veto = std::move(message_veto); listener.node_match = std::move(node_matches); listener.utgid_match = std::move(utgid_matches); + listener.print_raw = print_raw; listener.print_stats = print_stats; listener.print_counter = print_counter; listener.have_colors = have_colors; diff --git a/Online/SmiController/scripts/EventProcessor.py b/Online/SmiController/scripts/EventProcessor.py index a8e3bbb57..bee5242bd 100644 --- a/Online/SmiController/scripts/EventProcessor.py +++ b/Online/SmiController/scripts/EventProcessor.py @@ -22,7 +22,8 @@ application = GaudiOnline.Passthrough(outputLevel=OnlineEnvBase.OutputLevel, partitionID=OnlineEnvBase.PartitionID, classType=GaudiOnline.Class1) application.setup_fifolog() -application.log(GaudiOnline.MSG_INFO,'+++ OnlineEnv: %s Level:%d',OnlineEnvBase.__file__,OnlineEnvBase.OutputLevel) +application.log(GaudiOnline.MSG_INFO, '+++ OnlineEnv: %s Level:%d',OnlineEnvBase.__file__,OnlineEnvBase.OutputLevel) +application.log(GaudiOnline.MSG_ERROR,'+++ OnlineEnv: %s Level:%d',OnlineEnvBase.__file__,OnlineEnvBase.OutputLevel) application.setup_mbm_access('Events', True) writer = None writer = application.setup_mbm_output('EventOutput') diff --git a/Online/SmiController/scripts/EventProcessor.sh b/Online/SmiController/scripts/EventProcessor.sh index 5a78fb01d..deb6226a2 100644 --- a/Online/SmiController/scripts/EventProcessor.sh +++ b/Online/SmiController/scripts/EventProcessor.sh @@ -8,4 +8,5 @@ # Date: 20/05/2013 # # ========================================================================= -exec -a ${UTGID} python `which gaudirun.py` EventProcessor.py --application=OnlineEvents +# +exec -a ${UTGID} genPython.exe `which gaudirun.py` EventProcessor.py --application=OnlineEvents; diff --git a/Online/SmiController/scripts/MBMMonMulti.sh b/Online/SmiController/scripts/MBMMonMulti.sh index ec1c05e34..2b7d65a24 100755 --- a/Online/SmiController/scripts/MBMMonMulti.sh +++ b/Online/SmiController/scripts/MBMMonMulti.sh @@ -8,4 +8,5 @@ # Date: 20/05/2013 # # ========================================================================= +# `dataflow_task Class0` `dataflow_default_options ${TASK_TYPE}` diff --git a/Online/SmiController/scripts/MBMMonSingle.sh b/Online/SmiController/scripts/MBMMonSingle.sh index 71f6ece82..56ccd112c 100755 --- a/Online/SmiController/scripts/MBMMonSingle.sh +++ b/Online/SmiController/scripts/MBMMonSingle.sh @@ -8,5 +8,6 @@ # Date: 20/05/2013 # # ========================================================================= +# `dataflow_task Class0` `dataflow_default_options ${TASK_TYPE}` diff --git a/Online/SmiController/scripts/RCVMon.sh b/Online/SmiController/scripts/RCVMon.sh index c3e616d0a..ad3d05210 100755 --- a/Online/SmiController/scripts/RCVMon.sh +++ b/Online/SmiController/scripts/RCVMon.sh @@ -8,4 +8,5 @@ # Date: 20/05/2013 # # ========================================================================= +# `dataflow_task Class1` `dataflow_default_options ${TASK_TYPE}` diff --git a/Online/SmiController/scripts/TestMEPProd.sh b/Online/SmiController/scripts/TestMEPProd.sh index 48b19a903..5875196c4 100644 --- a/Online/SmiController/scripts/TestMEPProd.sh +++ b/Online/SmiController/scripts/TestMEPProd.sh @@ -1,3 +1,12 @@ #!/bin/bash +# ========================================================================= # +# Generic farm task startup script +# +# Author M.Frank +# Version: 1.0 +# Date: 20/05/2013 +# +# ========================================================================= +# `dataflow_task Class2` `dataflow_default_options ${TASK_TYPE}` diff --git a/Online/SmiController/scripts/configure.sh b/Online/SmiController/scripts/configure.sh index b71326ef3..bdaebb756 100755 --- a/Online/SmiController/scripts/configure.sh +++ b/Online/SmiController/scripts/configure.sh @@ -33,6 +33,11 @@ export DATAFLOW_task="gentest.exe libDataflow.so dataflow_run_task -msg=fifo -mo # # # +dataflow_task_setup() +{ + echo "echo bla > /dev/null;"; +} +# dataflow_specialized_options() { if test -n "${1}"; then @@ -50,5 +55,5 @@ dataflow_default_options() dataflow_task() { #echo "exec -a ${UTGID} gentest.exe libDataflow.so dataflow_run_task -msg=Dataflow_FmcLogger -mon=Dataflow_DIMMonitoring -class=$1"; - echo "exec -a ${UTGID} gentest.exe libDataflow.so dataflow_run_task -msg=fifo -mon=Dataflow_DIMMonitoring -class=$1"; + echo "exec -a ${UTGID} genRunner.exe libDataflow.so dataflow_run_task -msg=fifo -mon=Dataflow_DIMMonitoring -class=$1"; } diff --git a/Online/SmiController/scripts/runDummyTask.sh b/Online/SmiController/scripts/runDummyTask.sh index c38ced154..6a6f4e748 100755 --- a/Online/SmiController/scripts/runDummyTask.sh +++ b/Online/SmiController/scripts/runDummyTask.sh @@ -38,6 +38,6 @@ else fi; echo "Warning: ${DIM_DNS_NODE} -- ${DIM_DNS_HOST}" cd ${SMICONTROLLERROOT}/scripts; -python ${SMICONTROLLERROOT}/scripts/DummyTask.py -utgid ${UTGID} -partition 103 -print ${OUTPUT_LEVEL} +exec -a ${UTGID} genPython.exe ${SMICONTROLLERROOT}/scripts/DummyTask.py -utgid ${UTGID} -partition 103 -print ${OUTPUT_LEVEL} # echo "Going asleep....." diff --git a/Online/SmiController/scripts/runTask.sh b/Online/SmiController/scripts/runTask.sh index b025961e4..029b7f499 100755 --- a/Online/SmiController/scripts/runTask.sh +++ b/Online/SmiController/scripts/runTask.sh @@ -25,8 +25,8 @@ cd ${SMICONTROLLERROOT}/scripts; # if test -z "${FARMCONFIGROOT}";then export PARTITION=$PARTITION_NAME; - echo "ERROR FARMCONFIGROOT: ${FARMCONFIGROOT}"; - echo "UTGID:${UTGID} Partition: $PARTITION_NAME $PARTITION Task type:${TASK_TYPE} Log fifo:${LOGFIFO}"; + #echo "ERROR FARMCONFIGROOT: ${FARMCONFIGROOT}"; + #echo "UTGID:${UTGID} Partition: $PARTITION_NAME $PARTITION Task type:${TASK_TYPE} Log fifo:${LOGFIFO}"; else echo "Skip execution of setup.${CMTCONFIG}.vars" > /dev/null; #echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"; @@ -39,11 +39,11 @@ fi; #echo "[ERROR] ${UTGID} Current directory: `pwd`"; # if test -f ./${TASK_TYPE}.sh; then - #echo "[ERROR] ${UTGID} Running specialized task"; + # echo "[ERROR] ${UTGID} Running specialized task"; . ./configure.sh; . ./${TASK_TYPE}.sh $*; elif test -n "`echo ${UTGID} | grep TestEvent`"; then - #echo "[ERROR] ${UTGID} Running event processing task"; + # echo "[ERROR] ${UTGID} Running event processing task"; . ./configure.sh; . ./EventProcessor.sh $*; else diff --git a/Online/Storage/CMakeLists.txt b/Online/Storage/CMakeLists.txt index c8151ec63..9d47debc3 100644 --- a/Online/Storage/CMakeLists.txt +++ b/Online/Storage/CMakeLists.txt @@ -23,6 +23,8 @@ find_package(SQLite3 REQUIRED) # find_package(Boost REQUIRED COMPONENTS system filesystem) # +find_package(MySQL) # REQUIRED) +# include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) # gaudi_add_library(StorageClient src/client/*.cpp @@ -33,11 +35,14 @@ gaudi_add_library(StorageClient src/client/*.cpp target_compile_definitions(StorageClient PRIVATE -DSTORAGECLIENT=1) # gaudi_add_library(StorageServer src/server/*.cpp - INCLUDE_DIRS Boost HTTP SQLite3 dim + INCLUDE_DIRS Boost HTTP SQLite3 MySQL dim NO_PUBLIC_HEADERS - LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt z dim Boost HTTP SQLite3 StorageClient) + LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt z dim SQLite3 Boost HTTP StorageClient) # target_compile_definitions(StorageServer PRIVATE -DSTORAGECLIENT=1) +if (MYSQL_FOUND) + target_compile_definitions(StorageServer PRIVATE -DSTORAGE_HAVE_MYSQL=1) +endif() # # --------------------------------------------------------------------------------------- # Testing diff --git a/Online/Storage/src/server/fdb_db_server.cpp b/Online/Storage/src/server/fdb_db_server.cpp index 33643942a..438d9d4d4 100644 --- a/Online/Storage/src/server/fdb_db_server.cpp +++ b/Online/Storage/src/server/fdb_db_server.cpp @@ -120,8 +120,8 @@ http::basic_http_server<db>::handler_t::handle_get(const request_t& req, reply_t } if ( !error_code_ok(ec, this->debug) ) { header_t h(http::constants::error_cause,"Failed to get "+req.uri+" ["+ec.message()+"]"); - ::lib_rtl_output(LIB_RTL_ERROR,"GET: %s %-20s = %s", - reply_t::stock_status(rep.status).c_str(), + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: %s %-20s = %s", + req.method.c_str(), reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); rep.headers.emplace_back(move(h)); ++monitor->data.num_get_errors; @@ -130,8 +130,9 @@ http::basic_http_server<db>::handler_t::handle_get(const request_t& req, reply_t bool mb = length > 3*MByte; rep = reply_t::stock_reply(reply_t::temp_redirect); rep.headers.emplace_back(http::constants::location,access_name); - ::lib_rtl_output(LIB_RTL_INFO,"GET: '%s' %.1f %cB [%s]", - access_name.c_str(), double(length)/(mb ? MByte : kByte), mb ? 'M' : 'k', + ::lib_rtl_output(LIB_RTL_INFO,"+++ %s: '%s' %.1f %cB [%s]", + req.method.c_str(), access_name.c_str(), + double(length)/(mb ? MByte : kByte), mb ? 'M' : 'k', req.remote_address().to_string().c_str()); ++monitor->data.num_get_success; } @@ -162,8 +163,8 @@ http::basic_http_server<db>::handler_t::handle_delete(const request_t& req, repl } if ( !error_code_ok(ec, this->debug) ) { header_t h(http::constants::error_cause,"Failed to delete "+req.uri+" ["+ec.message()+"]"); - ::lib_rtl_output(LIB_RTL_ERROR,"DELETE: %s %-20s = %s", - reply_t::stock_status(rep.status).c_str(), + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: %s %-20s = %s", + req.method.c_str(), reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); rep.headers.emplace_back(move(h)); ++monitor->data.num_del_errors; @@ -172,8 +173,9 @@ http::basic_http_server<db>::handler_t::handle_delete(const request_t& req, repl bool mb = length > 3*MByte; rep = reply_t::stock_reply(reply_t::temp_redirect); rep.headers.emplace_back(http::constants::location,access_name); - ::lib_rtl_output(LIB_RTL_INFO,"DELETE:'%s' %.1f %cB [%s]", - obj.c_str(), double(length)/(mb ? MByte : kByte), mb ? 'M' : 'k', + ::lib_rtl_output(LIB_RTL_INFO,"+++ %s: '%s' %.1f %cB [%s]", + req.method.c_str(), obj.c_str(), + double(length)/(mb ? MByte : kByte), mb ? 'M' : 'k', req.remote_address().to_string().c_str()); ++monitor->data.num_del_success; } @@ -187,16 +189,16 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t if ( (hdr_date=req.header(http::constants::date)) == nullptr ) { header_t h(http::constants::error_cause,"Missing date header"); rep = reply_t::stock_reply(reply_t::bad_request); - ::lib_rtl_output(LIB_RTL_ERROR,"PUT: bad_request %-20s = %s", - h.name.c_str(), h.value.c_str()); + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: bad_request %-20s = %s", + req.method.c_str(), h.name.c_str(), h.value.c_str()); ++monitor->data.num_put_bad_request; rep.headers.emplace_back(move(h)); } else if ( (hdr_len=req.header(http::constants::content_length)) == nullptr ) { header_t h(http::constants::error_cause,"Missing content length header"); rep = reply_t::stock_reply(reply_t::bad_request); - ::lib_rtl_output(LIB_RTL_ERROR,"PUT: bad_request %-20s = %s", - h.name.c_str(), h.value.c_str()); + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: bad_request %-20s = %s", + req.method.c_str(), h.name.c_str(), h.value.c_str()); ++monitor->data.num_put_bad_request; rep.headers.emplace_back(move(h)); } @@ -211,8 +213,8 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t if ( ec == boost::system::errc::permission_denied ) { header_t h(http::constants::error_cause,"Failed to add "+req.uri+" "+ec.message()); rep = reply_t::stock_reply(reply_t::unauthorized); - ::lib_rtl_output(LIB_RTL_ERROR,"PUT: %s %-20s = %s", - reply_t::stock_status(rep.status).c_str(), + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: %s %-20s = %s", + req.method.c_str(), reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); ++monitor->data.num_put_unauthorized; rep.headers.emplace_back(move(h)); @@ -221,23 +223,30 @@ http::basic_http_server<db>::handler_t::handle_put(const request_t& req, reply_t header_t h(http::constants::error_cause, "Failed to add object "+req.uri+" in dbase: "+ec.message()); rep = reply_t::stock_reply(reply_t::bad_request); - ::lib_rtl_output(LIB_RTL_ERROR,"PUT: %s %-20s = %s", - reply_t::stock_status(rep.status).c_str(), + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: %s %-20s = %s", + req.method.c_str(), reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); ++monitor->data.num_put_bad_request; rep.headers.emplace_back(move(h)); } else { bool mb = len > 3*MByte; + std::string remote = "UNKNOWN-ERROR"; + try { + remote = req.remote_address().to_string(); + } + catch (...) { + } rep = reply_t::stock_reply(reply_t::temp_redirect); rep.content.clear(); rep.headers.clear(); rep.headers.emplace_back(http::constants::location,access_name); rep.headers.emplace_back(http::constants::data_length,len); rep.headers.emplace_back(http::constants::date,date); - ::lib_rtl_output(LIB_RTL_INFO,"PUT: '%s' %.1f %cB [%s]", - req.uri.c_str(), double(len)/(mb ? MByte : kByte), mb ? 'M' : 'k', - req.remote_address().to_string().c_str()); + ::lib_rtl_output(LIB_RTL_INFO,"+++ %s: '%s' %.1f %cB [%s]", + req.method.c_str(), req.uri.c_str(), + double(len)/(mb ? MByte : kByte), mb ? 'M' : 'k', + remote.c_str()); ++monitor->data.num_put_success; return write; } @@ -253,8 +262,8 @@ http::basic_http_server<db>::handler_t::handle_update(const request_t& req, repl if ( (hdr_state=req.header(http::constants::state)) == nullptr ) { header_t h(http::constants::error_cause,"Missing state header"); rep = reply_t::stock_reply(reply_t::bad_request); - ::lib_rtl_output(LIB_RTL_ERROR,"UPDATE: bad_request %-20s = %s", - h.name.c_str(), h.value.c_str()); + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: bad_request %-20s = %s", + req.method.c_str(), h.name.c_str(), h.value.c_str()); ++monitor->data.num_upda_bad_request; rep.headers.emplace_back(move(h)); } @@ -267,8 +276,8 @@ http::basic_http_server<db>::handler_t::handle_update(const request_t& req, repl header_t h(http::constants::error_cause, "Failed to update object "+req.uri+" in dbase: "+ec.message()); rep = reply_t::stock_reply(reply_t::bad_request); - ::lib_rtl_output(LIB_RTL_ERROR,"UPDATE: %s %-20s = %s", - reply_t::stock_status(rep.status).c_str(), + ::lib_rtl_output(LIB_RTL_ERROR,"+++ %s: %s %-20s = %s", + req.method.c_str(), reply_t::stock_status(rep.status).c_str(), h.name.c_str(), h.value.c_str()); ++monitor->data.num_upda_errors; rep.headers.emplace_back(move(h)); diff --git a/Online/Storage/src/server/fdb_dbase.cpp b/Online/Storage/src/server/fdb_dbase.cpp index 83e6ad294..81b563917 100644 --- a/Online/Storage/src/server/fdb_dbase.cpp +++ b/Online/Storage/src/server/fdb_dbase.cpp @@ -49,10 +49,10 @@ fdb_dbase_t::handler_t::query(std::string& object, host = file.host; length = ::atol(file.size.c_str()); date = file.date; - ::lib_rtl_output(_prt(this), "Query '%s'", object.c_str()); + ::lib_rtl_output(_prt(this), "+++ Query '%s'", object.c_str()); return ec; } - ::lib_rtl_output(LIB_RTL_ALWAYS,"Query: file '%s' NOT FOUND!", object.c_str()); + ::lib_rtl_output(LIB_RTL_ERROR,"+++ Query: file '%s' NOT FOUND!", object.c_str()); return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); } @@ -71,12 +71,12 @@ fdb_dbase_t::handler_t::next( std::string& object, date = file.date; ec = this->set(object, handler_t::STATE_READ); if ( ec == system::errc::success ) { - ::lib_rtl_output(_prt(this), "Next '%s'", object.c_str()); + ::lib_rtl_output(_prt(this), "+++ Next '%s'", object.c_str()); return system::error_code(system::errc::success, system::system_category()); } return system::error_code(system::errc::protocol_error, system::system_category()); } - ::lib_rtl_output(LIB_RTL_ALWAYS,"Query: file '%s' NOT FOUND!", object.c_str()); + ::lib_rtl_output(LIB_RTL_ERROR,"+++ Query: file '%s' NOT FOUND!", object.c_str()); return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); } @@ -116,10 +116,10 @@ system::error_code fdb_dbase_t::query_object(const std::string& object, auto ec = this->_engine->query(obj, host, date, length); if ( ec == system::errc::success ) { access = this->network_file(host, obj); - ::lib_rtl_output(_prt(this), "Lookup '%s'", obj.c_str()); + ::lib_rtl_output(_prt(this), "+++ Lookup '%s'", obj.c_str()); return ec; } - ::lib_rtl_output(LIB_RTL_ALWAYS,"FAILED to lookup object: '%s' [%s]", + ::lib_rtl_output(LIB_RTL_ERROR,"+++ FAILED lookup: '%s' [%s]", object.c_str(), ec.message().c_str()); return ec; } @@ -136,11 +136,11 @@ system::error_code fdb_dbase_t::delete_next( const std::string& object, ec = this->_engine->del(obj); access = this->network_file(host, obj); if ( ec == system::errc::success ) { - ::lib_rtl_output(_prt(this), "Lookup '%s'", obj.c_str()); + ::lib_rtl_output(_prt(this), "+++ Lookup '%s'", obj.c_str()); return ec; } } - ::lib_rtl_output(LIB_RTL_ERROR,"FAILED to lookup sequence object: '%s' [%s]", + ::lib_rtl_output(LIB_RTL_ERROR,"+++ FAILED sequence lookup: '%s' [%s]", object.c_str(), ec.message().c_str()); return ec; } @@ -157,11 +157,11 @@ system::error_code fdb_dbase_t::delete_object(const std::string& object, ec = this->_engine->del(obj); access = this->network_file(host, obj); if ( ec == system::errc::success ) { - ::lib_rtl_output(_prt(this), "Remove '%s'", obj.c_str()); + ::lib_rtl_output(_prt(this), "+++ Remove '%s'", obj.c_str()); return ec; } } - ::lib_rtl_output(_prt(this), "FAILED to remove: '%s' [%s]", + ::lib_rtl_output(_prt(this), "+++ FAILED remove: '%s' [%s]", object.c_str(), ec.message().c_str()); return ec; } @@ -178,12 +178,12 @@ fdb_dbase_t::add_object(const std::string& object, auto ec = this->_engine->add(obj, date, length, host); if ( ec == system::errc::success ) { access = this->network_file(host, obj); - ::lib_rtl_output(_prt(this), "Add '%s' %s %s", object.c_str(), + ::lib_rtl_output(_prt(this), "+++ Add '%s' %s %s", object.c_str(), this->_debug > 1 ? "access:" : "", this->_debug > 1 ? access.c_str() : ""); return ec; } - ::lib_rtl_output(LIB_RTL_ERROR,"FAILED to add: '%s' access: %s [%s]", + ::lib_rtl_output(LIB_RTL_ERROR,"+++ FAILED add: '%s' access: %s [%s]", object.c_str(), access.c_str(), ec.message().c_str()); return ec; } @@ -208,10 +208,10 @@ fdb_dbase_t::update_object_state(const std::string& object, const std::string& s } if ( ec == system::errc::success ) { ::lib_rtl_output(this->_debug > 1 ? LIB_RTL_ALWAYS : LIB_RTL_DEBUG, - "Update '%s' NEW state: %s", object.c_str(), state.c_str()); + "+++ Update '%s' state: %s", object.c_str(), state.c_str()); return ec; } - ::lib_rtl_output(LIB_RTL_ERROR,"FAILED update '%s' to state: %s [%s]", + ::lib_rtl_output(LIB_RTL_ERROR,"+++ FAILED update '%s' state: %s [%s]", object.c_str(), state.c_str(), ec.message().c_str()); return ec; } diff --git a/Online/Storage/src/server/fdb_fs_server.cpp b/Online/Storage/src/server/fdb_fs_server.cpp index d07012fac..bf8905ad7 100644 --- a/Online/Storage/src/server/fdb_fs_server.cpp +++ b/Online/Storage/src/server/fdb_fs_server.cpp @@ -56,7 +56,9 @@ namespace { */ struct fs_monitor { public: + typedef http::basic_http_server<Online::storage::fs>::handler_t handler_t; struct mon_data { + typedef std::pair<long, std::atomic<long>* > counter_t; int numGET {0}; int numPUT {0}; int numDEL {0}; @@ -66,14 +68,19 @@ namespace { long bytesGET {0}; long bytesPUT {0}; long bytesDEL {0}; + counter_t num_connections {0,nullptr}; + counter_t num_connections_opened {0,nullptr}; + counter_t num_connections_closed {0,nullptr}; + counter_t num_connections_stopped {0,nullptr}; } data; - std::vector<int> services; + handler_t& handler; + std::vector<int> services; void add(int svc) { services.emplace_back(svc); } public: /// Standard constructor - fs_monitor(); + fs_monitor(handler_t& hdlr); /// Default destructor ~fs_monitor(); }; @@ -160,6 +167,21 @@ namespace { return boost::system::error_code(errno, boost::system::system_category()); } }; + + /// Feed data to DIS when updating state + void counter_handler(void* tag, void** buff, int* size, int* /* first */) { + typedef std::pair<long, std::atomic<long>* > counter_t; + static long defaults = 0; + counter_t* h = *(counter_t**)tag; + if ( h ) { + h->first = h->second->load(std::memory_order_relaxed); + *buff = &h->first; + *size = sizeof(h->first); + return; + } + *buff = &defaults; + *size = sizeof(defaults); + } } class Online::storage::fs::traits { @@ -171,12 +193,26 @@ public: using namespace Online::storage; /// Standard constructor -fs_monitor::fs_monitor() { +fs_monitor::fs_monitor(handler_t& hdlr) : handler(hdlr) +{ std::string nam = "/"+RTL::str_upper(RTL::nodeNameShort())+"/"+RTL::processName(); + data.num_connections.second = &handler.num_connections; + data.num_connections_opened.second = &handler.num_connections_opened; + data.num_connections_closed.second = &handler.num_connections_closed; + data.num_connections_stopped.second = &handler.num_connections_stopped; + + add( ::dis_add_service((nam+"/Connections_active").c_str(), "L", 0, 0, + counter_handler, (long)&data.num_connections) ); + add( ::dis_add_service((nam+"/Connections_opened").c_str(), "L", 0, 0, + counter_handler, (long)&data.num_connections_opened) ); + add( ::dis_add_service((nam+"/Connections_closed").c_str(), "L", 0, 0, + counter_handler, (long)&data.num_connections_closed) ); + add( ::dis_add_service((nam+"/Connections_stopped").c_str(), "L", 0, 0, + counter_handler, (long)&data.num_connections_stopped) ); add( ::dis_add_service((nam+"/DATA").c_str(),"C",&data,sizeof(data),0,0) ); - add( ::dis_add_service((nam+"/GET").c_str(),"L",&data.numGET,sizeof(data.numGET),0,0) ); - add( ::dis_add_service((nam+"/PUT").c_str(),"L",&data.numPUT,sizeof(data.numPUT),0,0) ); - add( ::dis_add_service((nam+"/DEL").c_str(),"L",&data.numDEL,sizeof(data.numDEL),0,0) ); + add( ::dis_add_service((nam+"/GET").c_str(),"L",&data.numGET,sizeof(data.numGET), 0, 0) ); + add( ::dis_add_service((nam+"/PUT").c_str(),"L",&data.numPUT,sizeof(data.numPUT), 0, 0) ); + add( ::dis_add_service((nam+"/DEL").c_str(),"L",&data.numDEL,sizeof(data.numDEL), 0, 0) ); add( ::dis_add_service((nam+"/GET_bytes").c_str(),"L",&data.bytesGET,sizeof(data.bytesGET),0,0) ); add( ::dis_add_service((nam+"/PUT_bytes").c_str(),"L",&data.bytesPUT,sizeof(data.bytesPUT),0,0) ); add( ::dis_add_service((nam+"/DEL_bytes").c_str(),"L",&data.bytesDEL,sizeof(data.bytesDEL),0,0) ); @@ -352,13 +388,13 @@ http::basic_http_server<fs>::handler_t::handle_delete(const request_t& req, repl wrt, mb/std::max(1e-6, double(wrt)/1e3)); ++monitor->data.numDEL; monitor->data.bytesDEL += ctxt->st_data; - return none; + return close; } ::lib_rtl_output(LIB_RTL_ERROR, "+++ %s: Failed to read '%s' size:%ld / %ld %s [Inconsistemt context]", req.method.c_str(), req.uri.c_str(), rep.bytes_sent, rep.bytes_total, rep.stock_status(rep.status).c_str()); rep.context.reset(); - return none; + return close; } /// Specific handler for PUT requests @@ -533,7 +569,7 @@ extern "C" int fdb_fs_file_server(int argc, char** argv) { SrvRun<fs> s; s.create(argc, argv); s.ptr->implementation->dbase.reset(new fs_db(s.server, s.file_dir)); - s.ptr->implementation->monitor.reset(new fs_monitor()); + s.ptr->implementation->monitor.reset(new fs_monitor(*s.ptr->implementation)); ::dis_start_serving(srv.c_str()); s.start(); return 0; diff --git a/Online/Tell1Data/src/RawFile.cpp b/Online/Tell1Data/src/RawFile.cpp index 2327f422a..5c8f8db83 100644 --- a/Online/Tell1Data/src/RawFile.cpp +++ b/Online/Tell1Data/src/RawFile.cpp @@ -225,7 +225,11 @@ long RawFile::write(const void* data, size_t len) { long RawFile::read(void* pointer, int len) { unsigned char* p = (unsigned char*)pointer; - if ( m_begin ) { + if ( len < 0 ) { + errno = EINVAL; + return -1; + } + else if ( m_begin ) { if ( m_ptr + len <= m_end ) { ::memcpy(p, m_ptr, len); m_ptr += len; diff --git a/Online/TestBeam/job/Controller.sh b/Online/TestBeam/job/Controller.sh index fe0409e30..568878b59 100755 --- a/Online/TestBeam/job/Controller.sh +++ b/Online/TestBeam/job/Controller.sh @@ -10,7 +10,7 @@ # ========================================================================= HOST=`hostname -s | tr a-z A-Z`; HOST_LONG=`hostname -f | tr a-z A-Z`; -echo "ERROR DIM_DNS_NODE: ${DIM_DNS_NODE}" +# echo "ERROR DIM_DNS_NODE: ${DIM_DNS_NODE}" # # Check the existence of various arguments. # Otherwise use default values. @@ -31,7 +31,7 @@ if test -z "${DIM_DNS_NODE}"; then export DIM_DNS_NODE=${HOST_LONG}; fi; # -exec -a ${UTGID} `which gentest.exe` libSmiController.so smi_controller \ +exec -a ${UTGID} `which genRunner.exe` libSmiController.so smi_controller \ -print=4 \ -logger=fifo \ -part=${PARTITION_NAME} \ diff --git a/Online/TestBeam/job/runTask.sh b/Online/TestBeam/job/runTask.sh index d11364a06..c5fef55ed 100755 --- a/Online/TestBeam/job/runTask.sh +++ b/Online/TestBeam/job/runTask.sh @@ -9,8 +9,6 @@ # # ========================================================================= # -##export DIM_DNS_NODE=`hostname -s`; -# export SUBFARM=`echo ${DIM_DNS_NODE} | tr a-z A-Z`; export STATIC_OPTS=${FARMCONFIGROOT}/options; if test -z "${DATAINTERFACE}"; then @@ -23,11 +21,9 @@ export INFO_OPTIONS=${OPTIONS_DIR}/OnlineEnv.opts; export MBM_SETUP_OPTIONS=${OPTIONS_DIR}/MBMSetup.opts; export DATAFLOW_task="gentest.exe libDataflow.so dataflow_run_task -msg=fifo -mon=Dataflow_DIMMonitoring -print=${OUTPUT_LEVEL}"; # -##echo `pwd`; -# if test -f ./${TASK_TYPE}.sh; then - echo "RunTask: TASK_TYPE: ${TASK_TYPE} ./${TASK_TYPE}.sh" + #echo "RunTask: TASK_TYPE: ${TASK_TYPE} ./${TASK_TYPE}.sh" . ./${TASK_TYPE}.sh $*; elif test -f ./${PARTITION_NAME}${TASK_TYPE}.sh; then -- GitLab From d115506eca562d88fe924e7df08fb4932dc36afe Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Thu, 15 Apr 2021 15:42:06 +0200 Subject: [PATCH 17/26] Move basic main programs from OnlineKernel to OnlineBase --- Online/OnlineBase/main/checkpoint.cpp | 21 ++++++++ Online/OnlineBase/main/genPython.cpp | 75 +++++++++++++++++++++++++++ Online/OnlineBase/main/genRunner.cpp | 72 +++++++++++++++++++++++++ Online/OnlineBase/main/gentest.cpp | 52 +++++++++++++++++++ Online/OnlineBase/main/mbm_remove.cpp | 28 ++++++++++ Online/OnlineBase/main/wt.cpp | 31 +++++++++++ 6 files changed, 279 insertions(+) create mode 100644 Online/OnlineBase/main/checkpoint.cpp create mode 100644 Online/OnlineBase/main/genPython.cpp create mode 100644 Online/OnlineBase/main/genRunner.cpp create mode 100755 Online/OnlineBase/main/gentest.cpp create mode 100755 Online/OnlineBase/main/mbm_remove.cpp create mode 100755 Online/OnlineBase/main/wt.cpp diff --git a/Online/OnlineBase/main/checkpoint.cpp b/Online/OnlineBase/main/checkpoint.cpp new file mode 100644 index 000000000..a9d494bd5 --- /dev/null +++ b/Online/OnlineBase/main/checkpoint.cpp @@ -0,0 +1,21 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Author : Markus Frank +//========================================================================== +#define main checkpointing_main +#include "gentest.cpp" +#undef main +extern void CheckpointRestoreWrapper__init_instance(int argc, char** argv); + +int main (int argc, char** argv) { + CheckpointRestoreWrapper__init_instance(argc, argv); + return checkpointing_main(argc,argv); +} diff --git a/Online/OnlineBase/main/genPython.cpp b/Online/OnlineBase/main/genPython.cpp new file mode 100644 index 000000000..59bde542c --- /dev/null +++ b/Online/OnlineBase/main/genPython.cpp @@ -0,0 +1,75 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Author : Markus Frank +//========================================================================== +#if defined(__clang__) || defined(__CLING__) +#pragma clang diagnostic ignored "-Wregister" +#elif defined(__GNUC__) +#pragma GCC diagnostic ignored "-Wregister" +#endif + +/// Framework include files +#include "../src/LOG/FifoLog.cpp" + +/// C/C++ include files +#include <iostream> +#include <cerrno> + +// Iniitalize the logger unit +__attribute__((constructor)) void s__fifolog_initialize_logger() { + fifolog_initialize_logger(); +} + +/// Shutdown the loggetr unit +__attribute__((destructor)) void s__fifolog_finalize_logger() { + ::usleep(3000); + fifolog_finalize_logger(); +} + +/// Framework include files +#include <RTL/DllAccess.h> + +// ----------------------------------------------------------------------------- +// Python hacks to avoid warnings if outside dictionaries ..... +// ----------------------------------------------------------------------------- +// --> /usr/include/python2.7/pyconfig.h:1161:0: warning: "_POSIX_C_SOURCE" redefined [enabled by default] +#ifdef _POSIX_C_SOURCE +#undef _POSIX_C_SOURCE +#endif /* _POSIX_C_SOURCE */ +// --> /usr/include/python2.7/pyconfig.h:1183:0: warning: "_XOPEN_SOURCE" redefined [enabled by default] +#ifdef _XOPEN_SOURCE +#undef _XOPEN_SOURCE +#endif /* _XOPEN_SOURCE */ +#include "Python.h" + +/// Start the interpreter in normal mode without hacks like 'python.exe' does. +int main( int argc, char** argv ) { +#if PY_VERSION_HEX < 0x03000000 + return ::Py_Main(argc, argv); +#else + std::vector<wstring> wargs; + std::vector<const wchar_t*> wargv; + for(int i=0; i<argc;++i) { + std::wstring wstr; + if ( argv[i] ) { + const size_t size = ::strlen(argv[i]); + if (size > 0) { + wstr.resize(size+1); + std::mbstowcs(&wstr[0], argv[i], size); + wstr[size] = 0; + } + } + wargs.push_back(wstr); + } + for( auto& s : wargs ) wargv.push_back(s.c_str()); + return ::Py_Main(argc, (wchar_t**)&wargv[0]); +#endif +} diff --git a/Online/OnlineBase/main/genRunner.cpp b/Online/OnlineBase/main/genRunner.cpp new file mode 100644 index 000000000..99ff457b6 --- /dev/null +++ b/Online/OnlineBase/main/genRunner.cpp @@ -0,0 +1,72 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Author : Markus Frank +//========================================================================== + +/// Framework include files +#include "../src/LOG/FifoLog.cpp" + +/// C/C++ include files +#include <unistd.h> +#include <iostream> +#include <cerrno> + +// Iniitalize the logger unit +__attribute__((constructor)) void s__fifolog_initialize_logger() { + fifolog_initialize_logger(); +} + +/// Shutdown the loggetr unit +__attribute__((destructor)) void s__fifolog_finalize_logger() { + ::usleep(3000); + fifolog_finalize_logger(); +} + + +/// Framework include files +#include <RTL/DllAccess.h> + +/// Generic main program to start any process +/** + * Generic main program to start any process + * by loading a library and executing a + * function within + * + * \author M.Frank + * \version 1.0 + */ +int main (int argc, char** argv) { + if ( argc >= 3 ) { + void* handle = LOAD_LIB( argv[1] ); + if ( handle ) { + Function fun(GETPROC(handle, argv[2])); + if ( fun.function ) { + return (*fun.function)(argc-2, &argv[2]); + } + // The required function is not in the loaded image. + // Try to load it directly from the executable + Function f1(GETPROC(0, argv[2])); + if ( f1.function ) { + std::cout << "[WARNING] Executing function " << argv[2] + << " from exe rather than image " << std::endl; + return (*f1.function)(argc-2, &argv[2]); + } + std::cout << "[ERROR] Failed to access test procedure: '" << argv[2] << "'" << std::endl; + std::cout << "[ERROR] Error: " << DLERROR << std::endl; + return EINVAL; + } + std::cout << "[ERROR] Failed to load test library: '" << argv[1] << "'" << std::endl; + std::cout << "[ERROR] Error: " << DLERROR << std::endl; + return EINVAL; + } + std::cout << "[ERROR] usage: gentest.exe <library-name> <function-name> -arg [-arg]" << std::endl; + return EINVAL; +} diff --git a/Online/OnlineBase/main/gentest.cpp b/Online/OnlineBase/main/gentest.cpp new file mode 100755 index 000000000..99988aab4 --- /dev/null +++ b/Online/OnlineBase/main/gentest.cpp @@ -0,0 +1,52 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Author : Markus Frank +//========================================================================== +#include "RTL/DllAccess.h" +#include <iostream> +#include <cerrno> + +/// Generic main program to start any process +/** + * Generic main program to start any process + * by loading a library and executing a + * function within + * + * \author M.Frank + * \version 1.0 + */ +int main (int argc, char** argv) { + if ( argc >= 3 ) { + void* handle = LOAD_LIB( argv[1] ); + if ( handle ) { + Function fun(GETPROC(handle, argv[2])); + if ( fun.function ) { + return (*fun.function)(argc-2, &argv[2]); + } + // The required function is not in the loaded image. + // Try to load it directly from the executable + Function f1(GETPROC(0, argv[2])); + if ( f1.function ) { + std::cout << "[WARNING] Executing function " << argv[2] + << " from exe rather than image " << std::endl; + return (*f1.function)(argc-2, &argv[2]); + } + std::cout << "[ERROR] Failed to access test procedure: '" << argv[2] << "'" << std::endl; + std::cout << "[ERROR] Error: " << DLERROR << std::endl; + return EINVAL; + } + std::cout << "[ERROR] Failed to load test library: '" << argv[1] << "'" << std::endl; + std::cout << "[ERROR] Error: " << DLERROR << std::endl; + return EINVAL; + } + std::cout << "[ERROR] usage: gentest.exe <library-name> <function-name> -arg [-arg]" << std::endl; + return EINVAL; +} diff --git a/Online/OnlineBase/main/mbm_remove.cpp b/Online/OnlineBase/main/mbm_remove.cpp new file mode 100755 index 000000000..bbc209c55 --- /dev/null +++ b/Online/OnlineBase/main/mbm_remove.cpp @@ -0,0 +1,28 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Author : Markus Frank +//========================================================================== +#include "RTL/DllAccess.h" +#include <iostream> + +int main (int argc, char** argv) { + void* handle = LOAD_LIB2( "OnlineKernel" ); + if ( 0 != handle ) { + Function fun(GETPROC(handle,"mbm_remove")); + if ( fun.function ) { + return (*fun.function)(argc, argv); + } + std::cout << "Failed to access test procedure!" << std::endl; + } + std::cout << "Failed to load test library!" << std::endl; + std::cout << "Error: " << DLERROR << std::endl; + return 0; +} diff --git a/Online/OnlineBase/main/wt.cpp b/Online/OnlineBase/main/wt.cpp new file mode 100755 index 000000000..e6a77bf30 --- /dev/null +++ b/Online/OnlineBase/main/wt.cpp @@ -0,0 +1,31 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Author : Markus Frank +//========================================================================== +#include "RTL/DllAccess.h" +#include <iostream> + +using OnlineBase::FuncPtrCast; + +int main (int argc, char** argv) { + void* handle = LOAD_LIB2("OnlineKernel"); + if ( 0 != handle ) { + typedef int (*func_t)(int, char**); + func_t fun = FuncPtrCast<func_t>(GETPROC(handle, argv[1])); + if ( fun ) { + return (*fun)(argc-1, &argv[1]); + } + std::cout << "Failed to access test procedure!" << std::endl; + } + std::cout << "Failed to load test library!" << std::endl; + std::cout << "Error: " << DLERROR << std::endl; + return 0; +} -- GitLab From e504edc7a205389a4536805aa8fcb231405280c8 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Fri, 30 Apr 2021 18:09:26 +0200 Subject: [PATCH 18/26] Save Controller and Storage changes. Merge Flavios commit --- Online/EventBuilding/options/EB_BU_algo.opts | 2 +- Online/EventBuilding/options/EB_RU_algo.opts | 17 +- .../options/EB_Transport_properties.opts | 3 +- .../options/MFP_generator_params.opts | 0 Online/FarmConfig/job/Controller.sh | 32 +- Online/OnlineBase/OnlineBase/CPP/sqlite.h | 524 ----- Online/OnlineBase/OnlineBase/RTL/strdef.h | 29 +- Online/OnlineBase/main/genPython.cpp | 4 +- Online/OnlineBase/main/genRunner.cpp | 4 +- Online/OnlineBase/src/LOG/FifoLog.cpp | 2 + Online/OnlineBase/src/RTL/rtl_strdef.cpp | 38 + .../OnlineKernel/python/fifo_log/__init__.py | 17 +- Online/ROLogger/kafka/DimNodeConsumer.cpp | 2 + Online/ROLogger/kafka/DimPublish.cpp | 10 +- Online/ROLogger/kafka/OutputListener.cpp | 8 +- .../SmiController/SmiController.h | 2 + .../SmiController/SmiTaskConfiguration.h | 25 +- Online/SmiController/SmiController/Tasklist.h | 6 +- Online/SmiController/options/Empty.opts | 1 - .../options/lbDataflowArch_SmiTest.xml | 10 +- Online/SmiController/scripts/DummyTask.py | 2 +- Online/SmiController/scripts/configure.sh | 59 - .../scripts/createEnvironment.sh | 115 - Online/SmiController/scripts/runDummyTask.sh | 43 - Online/SmiController/scripts/runTask.sh | 52 +- Online/SmiController/src/SmiController.cpp | 160 +- .../src/SmiTaskConfiguration.cpp | 63 +- Online/SmiController/src/TasklistHandlers.cpp | 12 +- Online/SmiController/src/smi_controller.cpp | 69 +- Online/Storage/CMakeLists.txt | 7 +- Online/Storage/Storage/fdb_dbase.h | 3 +- Online/Storage/src/server/fdb_dbase.cpp | 4 +- .../server/{fdb_SQLite.cpp => fdb_sqldb.cpp} | 86 +- .../src/server/{fdb_SQLite.h => fdb_sqldb.h} | 24 +- Online/Storage/src/server/sqlite_test.cpp | 181 +- Online/Storage/tests/src/fdb_cli.cpp | 1 - Online/TestBeam/TestBeam/gui/NodeFSMPanel.h | 12 +- Online/TestBeam/TestBeam/gui/TaskManager.h | 70 + Online/TestBeam/TestBeam/gui/config.h | 23 +- Online/TestBeam/job/Controller.sh | 49 - Online/TestBeam/job/Monitor.sh | 36 - Online/TestBeam/job/PassThrough.sh | 45 - Online/TestBeam/job/SmiController.sh | 32 - Online/TestBeam/job/SmiTestController.sh | 42 - Online/TestBeam/job/runTask.sh | 39 - Online/TestBeam/src/gui/NodeFSMPanel.cpp | 266 ++- Online/TestBeam/src/gui/NodeGUI.cpp | 4 +- Online/TestBeam/src/gui/TaskManager.cpp | 86 + Online/TestBeam/src/gui/gui_main.cpp | 30 +- Online/sqldb/CMakeLists.txt | 53 + Online/sqldb/sqldb/mysql.h | 34 + Online/sqldb/sqldb/oracle.h | 34 + Online/sqldb/sqldb/sqldb-imp.h | 529 +++++ Online/sqldb/sqldb/sqldb.h | 697 ++++++ Online/sqldb/sqldb/sqlite.h | 37 + Online/sqldb/src/mysql.cpp | 424 ++++ Online/sqldb/src/oracle.cpp | 645 ++++++ Online/sqldb/src/sql.h | 823 +++++++ Online/sqldb/src/sqldb.cpp | 327 +++ Online/sqldb/src/sqlext.h | 1975 +++++++++++++++++ Online/sqldb/src/sqlite.cpp | 344 +++ Online/sqldb/src/sqltypes.h | 266 +++ Online/sqldb/test/sqldb_test.cpp | 66 + 63 files changed, 7122 insertions(+), 1483 deletions(-) mode change 100755 => 100644 Online/EventBuilding/options/EB_RU_algo.opts mode change 100755 => 100644 Online/EventBuilding/options/MFP_generator_params.opts delete mode 100644 Online/OnlineBase/OnlineBase/CPP/sqlite.h delete mode 100644 Online/SmiController/options/Empty.opts delete mode 100755 Online/SmiController/scripts/configure.sh delete mode 100755 Online/SmiController/scripts/createEnvironment.sh delete mode 100755 Online/SmiController/scripts/runDummyTask.sh rename Online/Storage/src/server/{fdb_SQLite.cpp => fdb_sqldb.cpp} (72%) rename Online/Storage/src/server/{fdb_SQLite.h => fdb_sqldb.h} (80%) create mode 100644 Online/TestBeam/TestBeam/gui/TaskManager.h delete mode 100755 Online/TestBeam/job/Controller.sh delete mode 100755 Online/TestBeam/job/Monitor.sh delete mode 100755 Online/TestBeam/job/PassThrough.sh delete mode 100644 Online/TestBeam/job/SmiController.sh delete mode 100755 Online/TestBeam/job/SmiTestController.sh delete mode 100755 Online/TestBeam/job/runTask.sh create mode 100644 Online/TestBeam/src/gui/TaskManager.cpp create mode 100644 Online/sqldb/CMakeLists.txt create mode 100644 Online/sqldb/sqldb/mysql.h create mode 100644 Online/sqldb/sqldb/oracle.h create mode 100644 Online/sqldb/sqldb/sqldb-imp.h create mode 100644 Online/sqldb/sqldb/sqldb.h create mode 100644 Online/sqldb/sqldb/sqlite.h create mode 100644 Online/sqldb/src/mysql.cpp create mode 100644 Online/sqldb/src/oracle.cpp create mode 100755 Online/sqldb/src/sql.h create mode 100644 Online/sqldb/src/sqldb.cpp create mode 100755 Online/sqldb/src/sqlext.h create mode 100644 Online/sqldb/src/sqlite.cpp create mode 100755 Online/sqldb/src/sqltypes.h create mode 100644 Online/sqldb/test/sqldb_test.cpp diff --git a/Online/EventBuilding/options/EB_BU_algo.opts b/Online/EventBuilding/options/EB_BU_algo.opts index 1fd2b0b01..9a57a1f49 100755 --- a/Online/EventBuilding/options/EB_BU_algo.opts +++ b/Online/EventBuilding/options/EB_BU_algo.opts @@ -2,6 +2,6 @@ BU.MBM_name = { 'Input', 'Input', 'Input', 'Input' }; BU.buffer_size = {}; BU.buffer_type = { 2, 2, 2, 2 }; BU.out_file_prefix = ""; -BU.shmem_prefix = ""; +BU.shmem_prefix = "BU_"; BU.stop_timeout = 60; BU.write_to_file = { FALSE, FALSE, FALSE, FALSE }; diff --git a/Online/EventBuilding/options/EB_RU_algo.opts b/Online/EventBuilding/options/EB_RU_algo.opts old mode 100755 new mode 100644 index 135eced95..1fd2b0b01 --- a/Online/EventBuilding/options/EB_RU_algo.opts +++ b/Online/EventBuilding/options/EB_RU_algo.opts @@ -1,10 +1,7 @@ -RU.MDF_filename = "/home/fpisani/mc_data/biger_file.mdf"; -RU.PCIe40_ids = {}; -RU.PCIe40_names = { 'tdtel041_0', 'tdtel041_1', 'tdtel042_0', 'tdtel043_0', 'tdtel043_1', 'tdtel071_0', 'tdtel071_1', 'tdtel072_0', 'tdtel072_1' }; -RU.buffer_sizes = {}; -RU.buffer_type = { 3, 3, 3, 3 }; -RU.n_MFPs = 100; -RU.n_fragment = 3000; -RU.shmem_prefix = "RU"; -RU.stop_timeout = 60; -RU.dummy_src_ids = {}; +BU.MBM_name = { 'Input', 'Input', 'Input', 'Input' }; +BU.buffer_size = {}; +BU.buffer_type = { 2, 2, 2, 2 }; +BU.out_file_prefix = ""; +BU.shmem_prefix = ""; +BU.stop_timeout = 60; +BU.write_to_file = { FALSE, FALSE, FALSE, FALSE }; diff --git a/Online/EventBuilding/options/EB_Transport_properties.opts b/Online/EventBuilding/options/EB_Transport_properties.opts index 3d06991a2..a0a3fd72c 100755 --- a/Online/EventBuilding/options/EB_Transport_properties.opts +++ b/Online/EventBuilding/options/EB_Transport_properties.opts @@ -2,7 +2,8 @@ EB_transport.BU_ranks = {}; EB_transport.MPI_errors_return = FALSE; EB_transport.RU_ranks = {}; EB_transport.RUs_per_nic = {}; -EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.n18450.json"; +EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.J22931.json"; EB_transport.n_par_mess = 0; EB_transport.n_sources_per_ru = { 3, 2, 2, 2 }; +EB_transport.shift_pattern = {}; EB_transport.src_ids = {}; diff --git a/Online/EventBuilding/options/MFP_generator_params.opts b/Online/EventBuilding/options/MFP_generator_params.opts old mode 100755 new mode 100644 diff --git a/Online/FarmConfig/job/Controller.sh b/Online/FarmConfig/job/Controller.sh index 4e89e04a7..d6d1b371d 100755 --- a/Online/FarmConfig/job/Controller.sh +++ b/Online/FarmConfig/job/Controller.sh @@ -42,20 +42,20 @@ if test -z "${DIM_DNS_NODE}"; then export DIM_DNS_NODE=${HOST_LONG}; fi; # -# -exec -a ${UTGID} `which gentest.exe` libSmiController.so smi_controller \ - -print=4 \ - -logger=fifo \ - -part=${PARTITION_NAME} \ - -dns=${DIM_DNS_NODE} \ - -tmsdns=${TMS_DNS} \ - -smidns=${SMI_DNS} \ - -smidomain=${SMI_DOMAIN} \ - -smidebug=0 \ - -smifile=${SMI_FILE} \ - -count=${NBOFSLAVES} \ - -service=none \ - -runinfo=${RUNINFO} \ - -taskconfig=${ARCH_FILE} \ - -standalone=2 \ +exec -a ${UTGID} `which genRunner.exe` libSmiController.so smi_controller \ + -print=4 \ + -logger=fifo \ + -part=${PARTITION_NAME} \ + -dns=${DIM_DNS_NODE} \ + -tmsdns=${TMS_DNS} \ + -smidns=${SMI_DNS} \ + -smidomain=${SMI_DOMAIN} \ + -smidebug=0 \ + -smifile=${SMI_FILE} \ + -count=${NBOFSLAVES} \ + -service=none \ + -runinfo=${RUNINFO} \ + -taskconfig=${ARCH_FILE} \ + "${CONTROLLER_REPLACEMENTS}" \ + -standalone=2 \ -bindcpus=0 diff --git a/Online/OnlineBase/OnlineBase/CPP/sqlite.h b/Online/OnlineBase/OnlineBase/CPP/sqlite.h deleted file mode 100644 index b602a28a6..000000000 --- a/Online/OnlineBase/OnlineBase/CPP/sqlite.h +++ /dev/null @@ -1,524 +0,0 @@ -//========================================================================== -// LHCb Online software suite -//-------------------------------------------------------------------------- -// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) -// All rights reserved. -// -// For the licensing terms see OnlineSys/LICENSE. -// -//-------------------------------------------------------------------------- -// -// Package : RPC -// -// Author : Markus Frank -//========================================================================== -#ifndef ONLINEBASE_CPP_SQLITE_H -#define ONLINEBASE_CPP_SQLITE_H - -// Framework include files - -// SQLite header -#include <sqlite3.h> - -// C/C++ files -#include <string> -#include <vector> -#include <cstring> -#include <cstdint> -#include <cstdarg> -#include <stdexcept> - -/// SQLITE namespace declaration -namespace sqlite { - - - class traits { - public: - typedef sqlite3 dbase_t; - typedef sqlite3_stmt statement_t; - typedef std::pair<const uint8_t*, size_t> blob_t; - static constexpr int OK = SQLITE_OK; - struct column { - static int get_int( statement_t* stmt, int col) { return ::sqlite3_column_int(stmt, col); } - static int64_t get_int64(statement_t* stmt, int col) { return ::sqlite3_column_int64(stmt, col); } - static int64_t get_real( statement_t* stmt, int col) { return ::sqlite3_column_double(stmt, col); } - static blob_t get_blob( statement_t* stmt, int col) - { return blob_t((const uint8_t*)::sqlite3_column_blob(stmt, col), ::sqlite3_column_bytes(stmt, col)); } - }; - }; - - class database final { - public: - typedef traits::dbase_t dbase_t; - dbase_t* db {nullptr}; - - traits::dbase_t* handle() const; - int statement_check(const char* fun, int expected, int ret) const; - - public: - /// Initializing constructor - database(dbase_t* ptr) : db(ptr) {} - /// Default constructor - database() = default; - /// Move constructor - database(database&& copy) = default; - /// Copy constructor - database(const database& copy) = default; - /// Destructor - ~database() = default; - - /// Move assignment - database& operator = (database&& copy) = default; - /// Copy assignment - database& operator = (const database& copy) = default; - - const char* name() const; - static const char* name(database db); - const char* errmsg() const; - static const char* errmsg(database db); - bool is_valid() const; - - static database open(const std::string& name, std::string& err); - int open(const std::string& name); - int close(); - int begin() const; - int commit() const; - int changes() const; - int execute(std::string& error, const char* sql) const; - int execute(std::string& error, - const char* sql, - void* param, - int (*callback)(void*, int, char**, char**)) const; - int execute_sql(std::string& error, const char* fmt, ...) const; - int execute_sql(std::string& error, - void* param, - int (*callback)(void*, int, char**, char**), - const char* fmt, ...) const; - int execute_sql(std::string& error, - void* param, - int (*callback)(void*, int, char**, char**), - const char* fmt, - va_list& args) const; - void list_tables() const; - }; - - class statement final { - public: - typedef std::vector<unsigned char> vblob_t; - typedef traits::statement_t statement_t; - typedef traits::dbase_t dbase_t; - typedef traits::blob_t blob_t; - private: - statement_t* stmt {nullptr}; - - int statement_check(const char* fun, int expected, int ret) const; - - public: - /// Initializing constructor - statement(statement_t* s) : stmt(s) {} - /// Default constructor - statement() = default; - /// Move constructor - statement(statement&& copy) = default; - /// Copy constructor - statement(const statement& copy) = default; - /// Destructor - ~statement() = default; - - /// Move assignment - statement& operator = (statement&& copy) = default; - /// Copy assignment - statement& operator = (const statement& copy) = default; - - static statement create(database db, const std::string& sql); - bool prepare(database db, const std::string& sql); - void reset() const; - int execute() const; - void finalize(); - - database db() const; - const char* errmsg() const; - static const char* errmsg(statement stm); - bool is_valid() const; - statement_t* handle() const; - - const char* sql() const; - const char* expanded_sql() const; - - int column_count() const; - - void bind_int (int column, int data) const; - void bind_int64 (int column, int64_t data) const; - void bind_real (int column, double data) const; - void bind_text (int column, const char* data, size_t len) const; - void bind_blob (int column, const void* data, size_t len) const; - - void bind (int column, char data) const; - void bind (int column, unsigned char data) const; - void bind (int column, short data) const; - void bind (int column, unsigned short data) const; - void bind (int column, int data) const; - void bind (int column, unsigned int data) const; - void bind (int column, long data) const; - void bind (int column, unsigned long data) const; - void bind (int column, float data) const; - void bind (int column, double data) const; - void bind (int column, const std::string& data) const; - void bind (int column, const char* data) const; - void bind (int column, const blob_t& data) const; - void bind (int column, const vblob_t& data) const; - void bind (int column, const uint8_t* data, size_t len) const; - - int get_int (int col) const; - int64_t get_int64 (int col) const; - double get_real (int col) const; - std::string get_text (int col) const; - vblob_t get_vblob (int col) const; - blob_t get_blob (int col) const; - template <typename T> T get (int column) const; - }; -} // End namespace sqlite - - -#include <sstream> -#include <cstdarg> - -#ifdef HAVE_NO_RTL -namespace sqlite { - inline size_t mysqlite_print_always(const char* format, va_list& args) { - std::string fmt = format; - fmt += "\n"; - return ::vfprintf(stdout,fmt.c_str(), args); - } - inline size_t mysqlite_print_error(const char* format, va_list& args) { - return mysqlite_print_always(format, args); - } -} -#else -#include <RTL/rtl.h> -/// SQLITE namespace declaration -namespace sqlite { - inline size_t mysqlite_print_error(const char* format, va_list& args) { - return ::lib_rtl_log(LIB_RTL_ERROR, format, args); - } - inline size_t mysqlite_print_always(const char* format, va_list& args) { - return ::lib_rtl_log(LIB_RTL_ALWAYS, format, args); - } -} -#endif - -/// SQLITE namespace declaration -namespace sqlite { - - inline size_t print_error(const char* format, ...) { - va_list args; - va_start( args, format ); - auto result = mysqlite_print_error(format, args); - va_end(args); - return result; - } - inline size_t print_always(const char* format, ...) { - va_list args; - va_start( args, format ); - auto result = mysqlite_print_always(format, args); - va_end(args); - return result; - } - static inline void no_free(void* /* ptr */) {} - - inline int database::statement_check(const char* fun, int expected, int ret) const { - if ( ret != expected ) { - std::stringstream str; - str << "Failed: " << fun << ". Expected result: " << expected - << " obtained result: " << ret << " [" << errmsg() << "]"; - print_error(str.str().c_str()); - throw std::runtime_error(str.str()); - } - return ret; - } - - inline traits::dbase_t* database::handle() const { - if ( db ) return db; - throw std::runtime_error("Failed to access database handle. [Invalid handle]"); - } - - inline bool database::is_valid() const { - return db != nullptr; - } - - inline const char* database::errmsg() const { - return errmsg(*this); - } - - inline const char* database::errmsg(database db) { - return ::sqlite3_errmsg(db.handle()); - } - - inline const char* database::name() const { - return ::sqlite3_db_filename(this->handle(), nullptr); - } - - inline const char* database::name(database db) { - return db.name(); - } - - inline int database::changes() const { - return ::sqlite3_changes(this->handle()); - } - - inline int database::open(const std::string& name) { - int rc = ::sqlite3_open(name.c_str(), &db); - if ( traits::OK != rc ) { - return rc; - } - return traits::OK; - } - - inline database database::open(const std::string& name, std::string& err) { - database d; - int rc = ::sqlite3_open(name.c_str(), &d.db); - if ( traits::OK != rc ) { - err = d.errmsg(); - return database(); - } - return d; - } - - inline int database::close() { - if ( this->db ) { - ::sqlite3_close(this->db); - this->db = nullptr; - } - return traits::OK; - } - - inline int database::begin() const { - std::string error; - return this->execute(error, "BEGIN TRANSACTION;"); - } - - inline int database::commit() const { - std::string error; - return this->execute(error, "COMMIT TRANSACTION;"); - } - - inline int database::execute(std::string& error, const char* sql) const { - return this->execute(error, sql, nullptr, nullptr); - } - - inline int database::execute(std::string& error, - const char* sql, - void* param, - int (*callback)(void*, int, char**, char**)) const { - char* err = nullptr; - int ret = ::sqlite3_exec(this->db, sql, callback, param, &err); - if ( ret != traits::OK ) { - error = err; - ::sqlite3_free(err); - } - return ret; - } - - inline int database::execute_sql(std::string& error, const char* fmt, ...) const { - va_list args; - va_start(args, fmt); - return this->execute_sql(error, nullptr, nullptr, fmt, args); - } - - inline int database::execute_sql(std::string& error, - void* param, - int (*callback)(void*, int, char**, char**), - const char* fmt, ...) const { - va_list args; - va_start(args, fmt); - return this->execute_sql(error, param, callback, fmt, args); - } - - inline int database::execute_sql(std::string& error, - void* param, - int (*callback)(void*, int, char**, char**), - const char* fmt, - va_list& args) const { - char str[4096]; - ::vsnprintf(str, sizeof(str), fmt, args); - va_end(args); - return this->execute(error, str, param, callback); - } - - inline void database::list_tables() const { - struct handler { - static int callback(void* /* NotUsed */, int argc, char **argv, char **name) { - std::stringstream s; - for ( int i = 0; i < argc; i++ ) - s << name[i] << " = " << (char*)(argv[i] ? argv[i] : "NULL") << " "; - print_always("TABLE: %s", s.str().c_str()); - return 0; - } - }; - std::string error; - const char* sql = "SELECT * FROM sqlite_master WHERE type='table'"; - if ( this->execute(error, sql, nullptr, handler::callback) != traits::OK ) { - print_error("%s: list_tables: SQL error: %s", name(), error.c_str()); - } - } - - inline int statement::column_count() const { - return ::sqlite3_column_count(handle()); - } - - inline bool statement::is_valid() const { - return stmt != nullptr; - } - - inline const char* statement::errmsg() const { - return this->db().errmsg(); - } - - inline const char* statement::errmsg(statement stm) { - return stm.db().errmsg(); - } - - inline traits::statement_t* statement::handle() const { - if ( stmt ) return stmt; - throw std::runtime_error("Failed to access statement handle. [Invalid handle]"); - } - - inline int statement::statement_check(const char* fun, int expected, int ret) const { - if ( ret != expected ) { - std::stringstream str; - str << "Failed: " << fun << ". Expected result: " << expected - << " obtained result: " << ret << " [" << db().errmsg() << "]"; - print_error(str.str().c_str()); - throw std::runtime_error(str.str()); - } - return ret; - } - - inline void statement::finalize() { - if ( stmt != nullptr ) { - ::sqlite3_finalize(stmt); - stmt = nullptr; - } - } - - inline bool statement::prepare(database dbase, const std::string& sql_str) { - finalize(); - int ret = ::sqlite3_prepare_v2(dbase.handle(), sql_str.c_str(), sql_str.length()+1, &stmt, nullptr); - if ( ret != traits::OK ) { - print_error("%s: FAILED prepare: SQL error: %s", dbase.name(), dbase.errmsg()); - print_error(" %s", sql_str.c_str()); - return false; - } - //print_always("%s: Successfully prepared statement:", dbase.name()); - //print_always(" %s", sql_str.c_str()); - return true; - } - - inline statement statement::create(database db, const std::string& sql) { - statement stmt; - if ( stmt.prepare(db, sql) ) - return stmt; - return statement(); - } - - inline int statement::execute() const { - return ::sqlite3_step(handle()); - } - - inline database statement::db() const { - return ::sqlite3_db_handle(handle()); - } - - inline const char* statement::sql() const { - return ::sqlite3_sql(handle()); - } - - inline const char* statement::expanded_sql() const { - return ::sqlite3_expanded_sql(stmt); - } - - inline void statement::reset() const { - ::sqlite3_reset(handle()); - } - - inline void statement::bind_int(int col, int data) const { - statement_check("statement::bind_int", traits::OK, ::sqlite3_bind_int(stmt, col, data)); - } - - inline void statement::bind_int64(int col, int64_t data) const { - statement_check("statement::bind_int64",traits::OK, ::sqlite3_bind_int64(stmt, col, data)); - } - - inline void statement::bind_real(int col, double data) const { - statement_check("statement::bind_real", traits::OK, ::sqlite3_bind_double(stmt, col, data)); - } - - inline void statement::bind_text(int col, const char* data, size_t len) const { - statement_check("statement::bind_text", traits::OK, ::sqlite3_bind_text(stmt,col,data,len,SQLITE_STATIC)); - } - - inline void statement::bind_blob(int col, const void* data, size_t len) const { - statement_check("statement::bind_blob", traits::OK, ::sqlite3_bind_blob(stmt,col,data,len,SQLITE_STATIC)); - } - - inline int statement::get_int (int col) const { return traits::column::get_int(stmt, col); } - inline int64_t statement::get_int64(int col) const { return traits::column::get_int64(stmt, col); } - inline double statement::get_real (int col) const { return traits::column::get_real(stmt, col); } - inline traits::blob_t statement::get_blob(int col) const { return traits::column::get_blob(stmt, col);} - - inline statement::vblob_t statement::get_vblob (int col) const { - traits::blob_t b = traits::column::get_blob(stmt, col); - return vblob_t(b.first, b.first + b.second); - } - - inline std::string statement::get_text (int col) const { - size_t len = ::sqlite3_column_bytes(stmt, col); - const uint8_t *ptr = ::sqlite3_column_text(stmt, col); - return std::string(ptr, ptr+len); - } - - inline void statement::bind(int col, char data) const { bind_int(col, data); } - inline void statement::bind(int col, unsigned char data) const { bind_int(col, data); } - inline void statement::bind(int col, short data) const { bind_int(col, data); } - inline void statement::bind(int col, unsigned short data) const { bind_int(col, data); } - inline void statement::bind(int col, int data) const { bind_int(col, data); } - inline void statement::bind(int col, unsigned int data) const { bind_int(col, data); } - inline void statement::bind(int col, long data) const { bind_int64(col, data); } - inline void statement::bind(int col, unsigned long data) const { bind_int64(col, data); } - inline void statement::bind(int col, float data) const { bind_real(col, data); } - inline void statement::bind(int col, double data) const { bind_real(col, data); } - inline void statement::bind(int col, const std::string& data) const - { bind_text(col, data.c_str(), data.length()); } - inline void statement::bind(int col, const char* data) const - { bind_text(col, data ? data : "", data ? ::strlen(data) : 0); } - inline void statement::bind(int col, const blob_t& data) const - { bind_blob(col, data.first, data.second); } - inline void statement::bind(int col, const vblob_t& data) const - { bind_blob(col, &data.at(0), data.size()); } - inline void statement::bind(int col, const uint8_t* data, size_t len) const - { bind_blob(col, data, len); } - - template <> inline char statement::get<char>(int col) const - { return (char)get_int(col); } - template <> inline unsigned char statement::get<unsigned char>(int col) const - { return (unsigned char)get_int(col); } - template <> inline short statement::get<short>(int col) const - { return (short)get_int(col); } - template <> inline unsigned short statement::get<unsigned short>(int col) const - { return (unsigned short)get_int(col); } - template <> inline int statement::get<int>(int col) const - { return get_int(col); } - template <> inline unsigned int statement::get<unsigned int>(int col) const - { return (unsigned int)get_int(col); } - template <> inline long statement::get<long>(int col) const - { return get_int64(col); } - template <> inline unsigned long statement::get<unsigned long>(int col) const - { return (unsigned long)get_int64(col); } - template <> inline float statement::get<float>(int col) const - { return (float)get_real(col); } - template <> inline double statement::get<double>(int col) const - { return get_real(col); } - template <> inline std::string statement::get<std::string>(int col) const - { return get_text(col); } -} // End namespace sqlite -#endif // ONLINEBASE_CPP_SQLITE_H diff --git a/Online/OnlineBase/OnlineBase/RTL/strdef.h b/Online/OnlineBase/OnlineBase/RTL/strdef.h index 9321021e0..b7d5c9ff5 100755 --- a/Online/OnlineBase/OnlineBase/RTL/strdef.h +++ b/Online/OnlineBase/OnlineBase/RTL/strdef.h @@ -13,15 +13,15 @@ #define _RTL_STRDEF_H #ifdef __cplusplus -#include <cstdio> -#include <cstdlib> +#include <string> +#include <vector> #include <cstring> #define __CXX_CONSTANT const extern "C" { + #else -#include <stdio.h> -#include <stdlib.h> -#include <string.h> + +#include <cstring.h> #define __CXX_CONSTANT #endif /* __cplusplus */ @@ -39,17 +39,34 @@ extern "C" { #endif /* __cplusplus */ #ifdef __cplusplus -#include <string> namespace RTL { + /// C++ version to convert a string to lower case std::string str_lower(const std::string& str); + /// C++ version to convert a string to upper case std::string str_upper(const std::string& str); + /// C++ version: replace all occurrences of a string std::string str_replace(const std::string& str, const std::string& pattern, const std::string& replacement); + /// C++: split string according to non-empty delimiter character + std::vector<std::string> str_split(const std::string& source, char delimiter); + /// C++: split string according to non-empty delimiter string + std::vector<std::string> str_split(const std::string& source, const std::string& delimiter); } inline const char* strstr_safe(const char* p1, const char* p2) { return ::strstr((char*)p1,p2); } inline const char* strchr_safe(const char* p1, char p2) { return ::strchr((char*)p1,p2); } #ifdef _WIN32 + +#ifdef __cplusplus +#include <cstdio> +#include <cstdlib> +#include <cstring> +#else +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#endif /* __cplusplus */ + #define vsnprintf _vsnprintf #define snprintf _snprintf inline int str_casecmp(const char *s1, const char *s2) { return ::_stricmp(s1, s2); } diff --git a/Online/OnlineBase/main/genPython.cpp b/Online/OnlineBase/main/genPython.cpp index 083904bcd..e40b52949 100644 --- a/Online/OnlineBase/main/genPython.cpp +++ b/Online/OnlineBase/main/genPython.cpp @@ -31,8 +31,8 @@ __attribute__((constructor)) void s__fifolog_initialize_logger() { /// Shutdown the loggetr unit __attribute__((destructor)) void s__fifolog_finalize_logger() { - ::usleep(3000); - fifolog_finalize_logger(); + //::usleep(3000); + // fifolog_finalize_logger(); } /// Framework include files diff --git a/Online/OnlineBase/main/genRunner.cpp b/Online/OnlineBase/main/genRunner.cpp index 99ff457b6..aa00b2830 100644 --- a/Online/OnlineBase/main/genRunner.cpp +++ b/Online/OnlineBase/main/genRunner.cpp @@ -26,8 +26,8 @@ __attribute__((constructor)) void s__fifolog_initialize_logger() { /// Shutdown the loggetr unit __attribute__((destructor)) void s__fifolog_finalize_logger() { - ::usleep(3000); - fifolog_finalize_logger(); + // ::usleep(3000); + // fifolog_finalize_logger(); } diff --git a/Online/OnlineBase/src/LOG/FifoLog.cpp b/Online/OnlineBase/src/LOG/FifoLog.cpp index 2556d3ccd..258d9dc56 100644 --- a/Online/OnlineBase/src/LOG/FifoLog.cpp +++ b/Online/OnlineBase/src/LOG/FifoLog.cpp @@ -286,6 +286,8 @@ namespace { return fun; } + //static LibCLogger* __s__log = nullptr; + inline logger_imp& log() { static LibCLogger s_log; //static PollLogger s_log; diff --git a/Online/OnlineBase/src/RTL/rtl_strdef.cpp b/Online/OnlineBase/src/RTL/rtl_strdef.cpp index 017121564..8dc81007e 100755 --- a/Online/OnlineBase/src/RTL/rtl_strdef.cpp +++ b/Online/OnlineBase/src/RTL/rtl_strdef.cpp @@ -13,6 +13,7 @@ #include <cctype> #include <cstring> #include <cstdlib> +#include <stdexcept> enum { STR_NOMATCH, STR_MATCH }; @@ -150,6 +151,43 @@ std::string RTL::str_replace(const std::string& str, const std::string& pattern, return res; } +/// C++: split string according to non-empty delimiter character +std::vector<std::string> RTL::str_split(const std::string& source, char delimiter) { + std::string tmp; + std::vector<std::string> result; + size_t start = 0, idx = source.find(delimiter, start); + result.reserve(10); + tmp = source.substr(start, idx); + if ( !tmp.empty() ) result.emplace_back(tmp); + while( idx != std::string::npos ) { + start = idx+1; + idx = source.find(delimiter, start); + tmp = source.substr(start, idx-start); + if ( !tmp.empty() ) result.emplace_back(tmp); + } + return result; +} + +/// C++: split string according to non-empty delimiter string +std::vector<std::string> RTL::str_split(const std::string& source, const std::string& delimiter) { + if ( !delimiter.empty() ) { + std::string tmp; + std::vector<std::string> result; + size_t start = 0, idx = source.find(delimiter, start); + result.reserve(10); + tmp = source.substr(start, idx); + if ( !tmp.empty() ) result.emplace_back(tmp); + while( idx != std::string::npos ) { + start = idx+1; + idx = source.find(delimiter, start); + tmp = source.substr(start, idx-start); + if ( !tmp.empty() ) result.emplace_back(tmp); + } + return result; + } + throw std::runtime_error("RTL: Cannot split string with empty delimiter"); +} + #if 0 int str_match_wild (const char *candidate_string, const char *pattern_string) { int result = STR_NOMATCH; diff --git a/Online/OnlineKernel/python/fifo_log/__init__.py b/Online/OnlineKernel/python/fifo_log/__init__.py index ffdca6cbc..1a6640283 100644 --- a/Online/OnlineKernel/python/fifo_log/__init__.py +++ b/Online/OnlineKernel/python/fifo_log/__init__.py @@ -23,13 +23,22 @@ """ from __future__ import print_function from builtins import object + +logger = None + class _init_logs(object): def __init__(self): import _fifo_log - _fifo_log.logger_start(); - def __del__(self): - import _fifo_log - _fifo_log.logger_stop(); + self.fifo = _fifo_log + self.fifo.logger_start(); + def stop(self): + self.fifo.logger_stop(); print('Stopped fifo logger output.') + def set_tag(self, tag): + self.fifo.logger_set_tag(tag) + + def set_utgid(self, utgid): + self.fifo.logger_set_utgid(utgid) + logger = _init_logs() diff --git a/Online/ROLogger/kafka/DimNodeConsumer.cpp b/Online/ROLogger/kafka/DimNodeConsumer.cpp index 58d0f21c0..e2963582e 100644 --- a/Online/ROLogger/kafka/DimNodeConsumer.cpp +++ b/Online/ROLogger/kafka/DimNodeConsumer.cpp @@ -180,7 +180,9 @@ void NodeConsumerImp::_add_server(const std::string& server) { } } } +#if 0 ::lib_rtl_output(LIB_RTL_VERBOSE,"+++ Check: %s", server.c_str()); +#endif if ( match && this->service_list.find(server) == this->service_list.end() ) { std::string svc = server + "/log"; unsigned int id = ::dic_info_service_dns(this->in_dns_ID, svc.c_str(), MONITORED, diff --git a/Online/ROLogger/kafka/DimPublish.cpp b/Online/ROLogger/kafka/DimPublish.cpp index 88fe5051b..26eebd89d 100644 --- a/Online/ROLogger/kafka/DimPublish.cpp +++ b/Online/ROLogger/kafka/DimPublish.cpp @@ -77,10 +77,12 @@ void DimPublish::set(const void* pay, size_t len) { } ::memcpy(payload, pay, len); payload[len] = 0; - if ( payload[len-1] == '\n' ) payload[len-1] = 0, --len; - else if ( payload[len-2] == '\n' ) payload[len-2] = 0, len -= 2; - else if ( payload[len-3] == '\n' ) payload[len-3] = 0, len -= 3; + if ( len >= 1 && payload[len-1] == '\n' ) payload[len-1] = 0, --len; + else if ( len >= 2 && payload[len-2] == '\n' ) payload[len-2] = 0, len -= 2; + else if ( len >= 3 && payload[len-3] == '\n' ) payload[len-3] = 0, len -= 3; payload_length = len+1; - ::dis_update_service(serviceID); + if ( payload_length > 1 ) { + ::dis_update_service(serviceID); + } } } diff --git a/Online/ROLogger/kafka/OutputListener.cpp b/Online/ROLogger/kafka/OutputListener.cpp index bea09ee24..5e462e9c1 100644 --- a/Online/ROLogger/kafka/OutputListener.cpp +++ b/Online/ROLogger/kafka/OutputListener.cpp @@ -48,7 +48,7 @@ void OutputListener::handle_monitoring(const Monitor& mon_info) { static char* mark_end(char* ptr) { static constexpr size_t e_tag_len = sizeof(END_TAG) - 2; if ( ptr ) { - char* end = ::strstr(ptr+1, END_TAG); + char* end = ::strstr(ptr, END_TAG); if ( end ) { *end = 0; return end+e_tag_len; // @@ -89,7 +89,7 @@ void OutputListener::print_msg(const char* time_stamp, const char* /* physical */, const char* counter) { - if ( message ) { + if ( message && ::strlen(message)>0 ) { const char* tag = systag ? systag : "SYSTEM"; if ( have_colors ) { int severity = message[0]; @@ -135,13 +135,13 @@ void OutputListener::print_msg(const char* time_stamp, } len += ::snprintf(text+len,sizeof(text)-len,"%-34s%s\n", utgid ? utgid : "", message ? message : ""); + ::fputs(text,stdout); + ::fflush(stdout); if ( have_colors ) { graphics::white(); graphics::plain(); graphics::bg_black(); } - ::fputs(text,stdout); - ::fflush(stdout); } } #include <unistd.h> diff --git a/Online/SmiController/SmiController/SmiController.h b/Online/SmiController/SmiController/SmiController.h index 95e4771a1..cb5b7901f 100644 --- a/Online/SmiController/SmiController/SmiController.h +++ b/Online/SmiController/SmiController/SmiController.h @@ -112,6 +112,8 @@ namespace FiniteStateMachine { std::string smi_utgid; /// stdin and stderr output files std::string stdout_file, stderr_file; + /// Replacement items + std::map<std::string, std::string> replacements; /// Number of active worker processes on the node int num_workers = -1; /// Flag to steer CPU binding diff --git a/Online/SmiController/SmiController/SmiTaskConfiguration.h b/Online/SmiController/SmiController/SmiTaskConfiguration.h index 955d8eac5..4a5198631 100644 --- a/Online/SmiController/SmiController/SmiTaskConfiguration.h +++ b/Online/SmiController/SmiController/SmiTaskConfiguration.h @@ -34,31 +34,34 @@ namespace FiniteStateMachine { * @version 0.1 */ struct SmiTaskConfiguration { + public: + typedef std::map<std::string, std::string> replacements_t; + protected: + /// Replacement item list + replacements_t m_replacements; /// String with the partition name - std::string m_partition; + std::string m_partition; /// String with the name of the configuration file - std::string m_config; + std::string m_config; /// String with the location of the run-info python file - std::string m_runinfo; + std::string m_runinfo; /// stdin and stderr output files - std::string m_stdout_file, m_stderr_file; - /// The number of instances to be forked - int m_instances; + std::string m_stdout_file, m_stderr_file; public: /// Standard constructor - SmiTaskConfiguration(const std::string& partition, + SmiTaskConfiguration(const replacements_t& replacements, + const std::string& partition, const std::string& config, - const std::string& runinfo, + const std::string& runinfo, const std::string& out, - const std::string& err, - int num_moore); + const std::string& err); /// Default destructor virtual ~SmiTaskConfiguration(); /// Analyse the configuration file and attach the corresponding tasks to the FSM machine std::map<std::string,SmiTask*> taskList(int bind_cpu, - int output_level); + int output_level); }; // End class SmiTaskConfiguration } // End namespace #endif // ONLINE_SMICONTROLLER_SMITASKCONFIGURATION_H diff --git a/Online/SmiController/SmiController/Tasklist.h b/Online/SmiController/SmiController/Tasklist.h index 89f4446d5..32e496d56 100644 --- a/Online/SmiController/SmiController/Tasklist.h +++ b/Online/SmiController/SmiController/Tasklist.h @@ -85,9 +85,9 @@ namespace FiniteStateMachine { /// Timeouts for this task type Timeouts timeouts; /// Flag if the task should not be started, only controlled - bool doStart {true}; + bool doStart { true }; /// Flag to indicate this is a VIP task - bool isVIP {false}; + bool isVIP { false }; /// Constructor @@ -112,7 +112,7 @@ namespace FiniteStateMachine { /// Standard destructor virtual ~Tasklist(); /// End-iterator over the task container - Tasks::const_iterator end() const { return tasks.end(); } + Tasks::const_iterator end() const { return tasks.end(); } /// Begin-iterator over the task container Tasks::const_iterator begin() const { return tasks.begin(); } }; // End class Tasklist diff --git a/Online/SmiController/options/Empty.opts b/Online/SmiController/options/Empty.opts deleted file mode 100644 index 8b1378917..000000000 --- a/Online/SmiController/options/Empty.opts +++ /dev/null @@ -1 +0,0 @@ - diff --git a/Online/SmiController/options/lbDataflowArch_SmiTest.xml b/Online/SmiController/options/lbDataflowArch_SmiTest.xml index f95b59beb..7b2bbe8f6 100644 --- a/Online/SmiController/options/lbDataflowArch_SmiTest.xml +++ b/Online/SmiController/options/lbDataflowArch_SmiTest.xml @@ -1,6 +1,6 @@ <tasks_inventory> <task name="MEPInit" user="online" group="onliners"> - <command>runDummyTask.sh</command> + <command>runTask.sh</command> <argument name="-type" value="${NAME}"/> <argument name="-runinfo" value="${RUNINFO}"/> <argument name="-partition" value="${PARTITION}"/> @@ -12,7 +12,7 @@ </task> <task name="FileProd0" user="online" group="onliners"> - <command>runDummyTask.sh</command> + <command>runTask.sh</command> <argument name="-type" value="${NAME}"/> <argument name="-runinfo" value="${RUNINFO}"/> <argument name="-partition" value="${PARTITION}"/> @@ -24,7 +24,7 @@ </task> <task name="Monitor" user="online" group="onliners" instances="NUMBER_OF_INSTANCES"> - <command>runDummyTask.sh</command> + <command>runTask.sh</command> <argument name="-type" value="${NAME}"/> <argument name="-runinfo" value="${RUNINFO}"/> <argument name="-partition" value="${PARTITION}"/> @@ -37,7 +37,7 @@ </task> <task name="Adder" user="online" group="onliners"> - <command>runDummyTask.sh</command> + <command>runTask.sh</command> <argument name="-type" value="${NAME}"/> <argument name="-runinfo" value="${RUNINFO}"/> <argument name="-partition" value="${PARTITION}"/> @@ -51,7 +51,7 @@ </task> <comment> <task name="Writer" user="online" group="onliners"> - <command>runDummyTask.sh</command> + <command>runTask.sh</command> <argument name="-type" value="${NAME}"/> <argument name="-runinfo" value="${RUNINFO}"/> <argument name="-partition" value="${PARTITION}"/> diff --git a/Online/SmiController/scripts/DummyTask.py b/Online/SmiController/scripts/DummyTask.py index 30f288021..c7db34b13 100755 --- a/Online/SmiController/scripts/DummyTask.py +++ b/Online/SmiController/scripts/DummyTask.py @@ -12,7 +12,7 @@ from builtins import object import os import sys import time -import fifo_log +# import _fifo_log import copy import pydim import threading diff --git a/Online/SmiController/scripts/configure.sh b/Online/SmiController/scripts/configure.sh deleted file mode 100755 index bdaebb756..000000000 --- a/Online/SmiController/scripts/configure.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -# ========================================================================= -# -# Default script to start any task on the HLT farm. -# -# Author M.Frank -# Version: 1.0 -# Date: 20/05/2013 -# -# ========================================================================= -# -export HOST=`echo ${DIM_DNS_HOST} | tr a-z A-Z`; -export PREAMBLE_OPTS=`pwd`/../options/Empty.opts; -# -# -HOST_TYPE=`hostname -s | tr a-z A-Z`; -if test "`echo ${HOST_TYPE} | cut -b 1-3`" = "MON"; then - export NON_DYNAMIC_OPTS=/group/online/dataflow/options/${PARTITION_NAME}/MONITORING; -elif test "`echo ${HOST_TYPE} | cut -b 3-4`" = "EB"; then - export NON_DYNAMIC_OPTS=/group/online/dataflow/options/${PARTITION_NAME}; -else - export NON_DYNAMIC_OPTS=/group/online/dataflow/options/${PARTITION_NAME}; -fi; -# -# -export DYNAMIC_OPTS=${NON_DYNAMIC_OPTS}; -export PYTHONPATH=${NON_DYNAMIC_OPTS}:${PYTHONPATH}; -export INFO_OPTIONS=${NON_DYNAMIC_OPTS}/OnlineEnv.opts; -# -export ONLINETASKS=/group/online/dataflow/templates; -export DATAINTERFACE=`python /group/online/dataflow/scripts/getDataInterface.py`; -export DATAFLOW_task="gentest.exe libDataflow.so dataflow_run_task -msg=fifo -mon=Dataflow_DIMMonitoring"; -# -# -# -dataflow_task_setup() -{ - echo "echo bla > /dev/null;"; -} -# -dataflow_specialized_options() -{ - if test -n "${1}"; then - echo "-opts=/group/online/dataflow/options/${PARTITION_NAME}/MONITORING/${1}.opts"; - else - echo "-opts=/group/online/dataflow/options/${PARTITION_NAME}/MONITORING/${UTGID}.opts"; - fi; -} -# -dataflow_default_options() -{ - echo "-opts=/group/online/dataflow/options/${PARTITION_NAME}/MONITORING/${UTGID}.opts"; -} -# -dataflow_task() -{ - #echo "exec -a ${UTGID} gentest.exe libDataflow.so dataflow_run_task -msg=Dataflow_FmcLogger -mon=Dataflow_DIMMonitoring -class=$1"; - echo "exec -a ${UTGID} genRunner.exe libDataflow.so dataflow_run_task -msg=fifo -mon=Dataflow_DIMMonitoring -class=$1"; -} diff --git a/Online/SmiController/scripts/createEnvironment.sh b/Online/SmiController/scripts/createEnvironment.sh deleted file mode 100755 index b960c62a2..000000000 --- a/Online/SmiController/scripts/createEnvironment.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash -# ========================================================================= -# -# Generic farm task startup script -# -# Author M.Frank -# Version: 1.0 -# Date: 20/05/2013 -# -# ========================================================================= -export UTGID=${UTGID} -if test -n "${LOGFIFO}"; then - export LOGFIFO=${LOGFIFO}; -fi; -if [ -r /etc/sysconfig/dim ]; then - . /etc/sysconfig/dim; - export DIM_DNS_NODE; -fi; -# -#echo "ERROR ARGS: $*" -# -# Evaluate the information in the runinfo file -# -------------------------------------------- -eval `python <<EOF -import os, sys, imp - -dbg = False - -def _p(x): - if dbg: print 'echo '+os.environ['UTGID']+' '+x+';' - print x+';' - -args = "$*".split() -runinfo = None -while len(args): - itms = args[0].split('=') - type = itms[0].lower() - value = '' - if len(itms)>1: value=itms[1] - del args[0] - - if type == '-type': - _p('export TASK_TYPE='+value) - elif type == '-partition': - _p('export PARTITION_NAME='+value) - elif type == '-runinfo': - _p('export RUNINFO='+value) - runinfo = value - elif type == '-taskconfig': - _p('export ARCH_FILE='+value) - elif type == '-taskinfo': - _p('export ARCH_FILE='+value) - elif type == '-count': - _p('export NBOFSLAVES='+value) - elif type == '-instances': - _p('export NBOFSLAVES='+value) - elif type == '-utgid': - _p('export UTGID='+value) - elif type == '-logfifo': - _p('export LOGFIFO='+value) - elif type == '-auto': - _p('export AUTO_STARTUP=-auto') - elif type == '-debug': - _p('export DEBUG_STARTUP=-debug') - else: - _p('echo Unknown task argument '+type) - - -if runinfo is None: - _p('echo '+os.environ['UTGID']+': [ERROR] Failed to find RUNINFO file. Cannot start task;') - _p('exit 11') - sys.exit(11); -try: - dir,fname = os.path.split(runinfo) - nam,ext = os.path.splitext(fname) - sys.path.insert(1,dir) - mod = imp.find_module(nam,None) - Online=imp.load_module(nam,mod[0],mod[1],mod[2]) - OnlineVersion = Online.OnlineVersion - idx = OnlineVersion.find('Online_') - if OnlineVersion == 'Online': - OnlineVersion = 'OnlineRelease' - elif idx >= 0: - OnlineVersion = OnlineVersion.replace('Online_','OnlineDev_') - _p('export ONLINE_ENV_DIR='+dir) - _p('export PARTITION_NAME='+str(Online.PartitionName)) - _p('export RUN_TYPE='+str(Online.Activity).replace('|','\|')) - _p('export MOOREONLINE_VERSION='+str(Online.MooreOnlineVersion)) - _p('export MOORE_VERSION='+str(Online.MooreVersion)) - _p('export ONLINE_VERSION='+str(OnlineVersion)) - _p('export OUTPUT_LEVEL='+str(Online.OutputLevel)) - _p('export MBM_SETUP_OPTIONS=/group/online/dataflow/options/'+str(Online.PartitionName)+'/HLT/MBM_setup.opts') - if hasattr(Online,'TAE') and Online.TAE != 0: - _p('export TAE_PROCESSING=TAE') - if os.environ.has_key('CREATE_CHECKPOINT'): - _p('export MOORESTARTUP_MODE=CREATE_CHECKPOINT') - elif os.environ.has_key('TEST_CHECKPOINT'): - _p('export MOORESTARTUP_MODE=RESTORE') - elif hasattr(Online,'MooreStartupMode'): - if Online.MooreStartupMode==0: - _p('export MOORESTARTUP_MODE=NORMAL') - elif Online.MooreStartupMode==1: - _p('export MOORESTARTUP_MODE=FORKING') - elif Online.MooreStartupMode==2: - _p('export MOORESTARTUP_MODE=RESTORE') - if hasattr(Online,'HLTType') and hasattr(Online,'HltArchitecture'): - fn = '/group/online/dataflow/cmtuser/checkpoints/' + str(Online.HLTType) + '/' + \ - str(Online.OnlineVersion) + '/' + str(Online.HltArchitecture) - _p('export CHECKPOINT_DIR='+fn) - -except Exception,X: - _p('echo '+str(X)) - _p('exit 11') - -EOF` diff --git a/Online/SmiController/scripts/runDummyTask.sh b/Online/SmiController/scripts/runDummyTask.sh deleted file mode 100755 index 6a6f4e748..000000000 --- a/Online/SmiController/scripts/runDummyTask.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# ========================================================================= -# -# Generic farm task startup script -# -# Author M.Frank -# Version: 1.0 -# Date: 20/05/2013 -# -# ========================================================================= -echo "[ERROR] `pwd`" -cd `dirname $0`; -echo "[ERROR] `pwd`" -. createEnvironment.sh $*; -echo "[ERROR] `pwd`" -# -# Switch to the task directory and execute the controller process -# --------------------------------------------------------------- -export CMTCONFIG=x86_64+avx2+fma-centos7-gcc8-opt; -export CMTCONFIG=x86_64-centos7-gcc9-do0; -export CMTCONFIG=x86_64-centos7-gcc9-opt; -# -. /home/frankm/upgrade_sw/Online/setup.${CMTCONFIG}.vars; -if test -z "${FARMCONFIGROOT}";then - echo "[ERROR] Partition: $PARTITION_NAME $PARTITION " - export PARTITION=$PARTITION_NAME; - echo "[ERROR] FARMCONFIGROOT: ${FARMCONFIGROOT}"; - echo "[ERROR] Task type: ${TASK_TYPE}"; - echo "[ERROR] Log fifo: ${LOGFIFO}"; - echo "[ERROR] UTGID: ${UTGID}"; -else - echo "Skip execution of setup.${CMTCONFIG}.vars"; - #echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"; - #echo "FARMCONFIGROOT: ${FARMCONFIGROOT}"; - #echo "Task type: ${TASK_TYPE}"; - #echo "Log fifo: ${LOGFIFO}"; - #echo "UTGID: ${UTGID}"; -fi; -echo "Warning: ${DIM_DNS_NODE} -- ${DIM_DNS_HOST}" -cd ${SMICONTROLLERROOT}/scripts; -exec -a ${UTGID} genPython.exe ${SMICONTROLLERROOT}/scripts/DummyTask.py -utgid ${UTGID} -partition 103 -print ${OUTPUT_LEVEL} -# -echo "Going asleep....." diff --git a/Online/SmiController/scripts/runTask.sh b/Online/SmiController/scripts/runTask.sh index 029b7f499..d1ceb9014 100755 --- a/Online/SmiController/scripts/runTask.sh +++ b/Online/SmiController/scripts/runTask.sh @@ -1,53 +1,47 @@ -g#!/bin/bash +#!/bin/bash # ========================================================================= # # Generic farm task startup script # # Author M.Frank # Version: 1.0 -# Date: 20/05/2013 +# Date: 20/05/2020 # # ========================================================================= -cd `dirname $0`; -. ./createEnvironment.sh $*; # -# Switch to the task directory and execute the controller process -# --------------------------------------------------------------- -export CMTCONFIG=x86_64+avx2+fma-centos7-gcc8-opt; export CMTCONFIG=x86_64-centos7-gcc9-opt; export CMTCONFIG=x86_64-centos7-gcc9-do0; # . /home/frankm/upgrade_sw/Online/setup.${CMTCONFIG}.vars; -# #. /group/online/dataflow/cmtuser/OnlineRelease/setup.${CMTCONFIG}.vars; # -cd ${SMICONTROLLERROOT}/scripts; +. ${FARMCONFIGROOT}/job/createEnvironment.sh $*; # -if test -z "${FARMCONFIGROOT}";then - export PARTITION=$PARTITION_NAME; - #echo "ERROR FARMCONFIGROOT: ${FARMCONFIGROOT}"; - #echo "UTGID:${UTGID} Partition: $PARTITION_NAME $PARTITION Task type:${TASK_TYPE} Log fifo:${LOGFIFO}"; -else - echo "Skip execution of setup.${CMTCONFIG}.vars" > /dev/null; - #echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"; - #echo "FARMCONFIGROOT: ${FARMCONFIGROOT}"; - #echo "UTGID:${UTGID} Partition: $PARTITION_NAME $PARTITION Task type:${TASK_TYPE} Log fifo:${LOGFIFO}"; -fi; +export DATAFLOW_task="genRunner.exe libDataflow.so dataflow_run_task -msg=fifo -mon=Dataflow_DIMMonitoring"; +export GEN_OPTIONS=/group/online/dataflow/options/${PARTITION}/MONITORING; # -#echo "[WARN] ${DIM_DNS_NODE} -- ${DIM_DNS_HOST}" +export ONLINETASKS=/group/online/dataflow/templates; +export INFO_OPTIONS=${GEN_OPTIONS}/OnlineEnv.opts; +export PREAMBLE_OPTS=${FARMCONFIGROOT}/options/Empty.opts; +export PYTHONPATH=${GEN_OPTIONS}:${PYTHONPATH} # -#echo "[ERROR] ${UTGID} Current directory: `pwd`"; +dataflow_default_options() +{ + echo "-opts=/group/online/dataflow/options/${PARTITION_NAME}/MONITORING/${UTGID}.opts"; +} # -if test -f ./${TASK_TYPE}.sh; then - # echo "[ERROR] ${UTGID} Running specialized task"; - . ./configure.sh; +dataflow_task() +{ + echo "exec -a ${UTGID} ${DATAFLOW_task} -class=$1"; +} +# +if test "${TASK_TYPE}" = "Controller"; then + . ${FARMCONFIGROOT}/job/Controller.sh; +elif test -f ${SMICONTROLLERROOT}/scripts/${TASK_TYPE}.sh; then + cd ${SMICONTROLLERROOT}/scripts; . ./${TASK_TYPE}.sh $*; elif test -n "`echo ${UTGID} | grep TestEvent`"; then - # echo "[ERROR] ${UTGID} Running event processing task"; - . ./configure.sh; - . ./EventProcessor.sh $*; + exec -a ${UTGID} genPython.exe `which gaudirun.py` ${SMICONTROLLERROOT}/scripts/EventProcessor.py --application=OnlineEvents; else exec -a ${UTGID} python ${SMICONTROLLERROOT}/scripts/DummyTask.py -utgid ${UTGID} -partition 103 -print ${OUTPUT_LEVEL}; fi; -# -echo "Going asleep....." diff --git a/Online/SmiController/src/SmiController.cpp b/Online/SmiController/src/SmiController.cpp index c04dd6391..5c92586a5 100644 --- a/Online/SmiController/src/SmiController.cpp +++ b/Online/SmiController/src/SmiController.cpp @@ -33,6 +33,9 @@ #include <sstream> #include <sys/stat.h> #include <sys/types.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <netdb.h> using namespace std; using namespace FiniteStateMachine; @@ -47,30 +50,6 @@ namespace { static constexpr char UNKNOWN_ACTION = 'U'; }; - /// Feed data to DIS when updating data - void feed_std_string(void* tag, void** buff, int* size, int* /* first */) { - static const char* data = ""; - string* s = *(string**)tag; - if ( !s->empty() ) { - *buff = (void*)s->data(); - *size = s->length()+1; - return; - } - *buff = (void*)data; - *size = 1; - } - void num_thread_handler(void* tag, void* address, int* size) { - if ( tag && address ) { - int len = size ? *size : 0; - if ( len > 0 ) { - SmiController* c = *(SmiController**)tag; - int m = *(int*)address; - if ( len == sizeof(int) && m > 0 ) { - c->publish_instances(m); - } - } - } - } } /// Helper base class to represent a SMI object @@ -81,6 +60,7 @@ namespace { */ class SmiControllerObject : public SmiObject { public: + /// Reference to the controller object SmiController* controller = 0; public: /// Initializing constructor @@ -93,7 +73,9 @@ public: { } /// Default destructor virtual ~SmiControllerObject() = default; + /// Callback to handle state changes virtual void smiStateChangeHandler() override final {} + /// Callback to handle state changes virtual void smiExecutingHandler() override final {} }; @@ -422,8 +404,14 @@ SmiController::TaskProxy::TaskProxy(SmiController* ctrl, const std::string& nam, { string proc = RTL::processName(); const SmiController::config_t& config = ctrl->config; +#if 0 + m_tms_dic_dns_ID = ctrl->m_tms_dic_dns_ID; + m_local_dic_dns_ID = ctrl->m_local_dic_dns_ID; + m_local_dis_dns_ID = ctrl->m_local_dis_dns_ID; +#else m_tms_dic_dns_ID = ::dic_add_dns(config.tms_dns.c_str(), ::dim_get_dns_port()); m_local_dic_dns_ID = ::dic_add_dns(config.dns.c_str(), ::dim_get_dns_port()); +#endif m_dim_state.first = ::dic_info_service_dns(m_local_dic_dns_ID, (nam+"/status").c_str(), MONITORED,0,0,0,state_handler,(long)this,0,0); @@ -736,29 +724,7 @@ void SmiController::TaskProxy::notify_controller() { //============================================================================== /// Fully qualified domain string (config domain + "_" + host name) std::string SmiController::config_t::smiDomain() const { - return this->smi_domain; // + "_" + RTL::str_upper(RTL::nodeNameShort()); -} - -namespace { - void feed_transition(void* tag, void* address, int* size) { - if ( tag && address && size && *size > 0 ) { - SmiController* controller = *(SmiController**)tag; - char cmd[512]; - size_t len = std::min((size_t)*size, sizeof(cmd)-1); - ::strncpy(cmd, (const char*)address, len); - cmd[len] = 0; - if ( controller->config.standalone ) - controller->m_nodeProxy->invoke_transition(cmd); - else - controller->invoke_transition(cmd); - } - } - void node_state_handler(void* tag, void* address, int* size) { - if ( tag && address && size && *size > 0 ) { - SmiController* ctrl = *(SmiController**)tag; - IocSensor::instance().send(ctrl, SmiController::CONTROLLER_SETNODESTATE, new string((char*)address)); - } - } + return this->smi_domain; } //============================================================================== @@ -766,6 +732,34 @@ namespace { SmiController::SmiController(const config_t& cfg) : config(cfg), m_num_worker_threads(cfg.num_workers) { +#if 0 + struct in_addr raw_ip, dns_ip, tms_ip, smi_ip; + string dns = ::getenv("DIM_DNS_NODE") ? ::getenv("DIM_DNS_NODE") : ""; + struct hostent* raw_host = ::gethostbyname(dns.c_str()); + struct hostent* dns_host = ::gethostbyname(config.dns.c_str()); + struct hostent* tms_host = ::gethostbyname(config.tms_dns.c_str()); + struct hostent* smi_host = ::gethostbyname(config.smi_dns.c_str()); + + ::memset(&raw_ip, 0, sizeof(raw_ip)); + ::memset(&dns_ip, 0, sizeof(dns_ip)); + ::memset(&tms_ip, 0, sizeof(tms_ip)); + ::memset(&smi_ip, 0, sizeof(smi_ip)); + if ( raw_host ) raw_ip = *(struct in_addr*)raw_host->h_addr; + if ( dns_host ) dns_ip = *(struct in_addr*)dns_host->h_addr; + if ( tms_host ) tms_ip = *(struct in_addr*)tms_host->h_addr; + if ( smi_host ) smi_ip = *(struct in_addr*)smi_host->h_addr; + + m_local_dic_dns_ID = (0 == memcmp(&raw_ip,&dns_ip,sizeof(struct in_addr))) + ? 0 : ::dic_add_dns(config.dns.c_str(), ::dim_get_dns_port()); + m_tms_dic_dns_ID = (0 == memcmp(&tms_ip,&dns_ip,sizeof(struct in_addr))) + ? m_local_dic_dns_ID : ::dic_add_dns(config.tms_dns.c_str(), ::dim_get_dns_port()); + m_smi_dic_dns_ID = (0 == memcmp(&smi_ip,&dns_ip,sizeof(struct in_addr))) + ? m_local_dic_dns_ID : ::dic_add_dns(config.smi_dns.c_str(), ::dim_get_dns_port()); + + m_local_dis_dns_ID = 0;//::dis_add_dns(config.dns.c_str(), ::dim_get_dns_port()); + m_smi_dis_dns_ID = (0 == memcmp(&smi_ip,&dns_ip,sizeof(struct in_addr))) + ? m_local_dis_dns_ID : ::dis_add_dns(config.smi_dns.c_str(), ::dim_get_dns_port()); +#endif m_log.reset(new RTL::Logger(RTL::Logger::LogDevice::getGlobalDevice(), "Controller", cfg.output_level)); m_num_worker_threads = config.num_workers; m_node_state = State::UNKNOWN; @@ -803,6 +797,56 @@ void SmiController::configure() { //============================================================================== void SmiController::initialize() { + class dim_handlers { + public: + /// Feed data to DIS when updating data + static void feed_std_string(void* tag, void** buff, int* size, int* /* first */) { + static const char* data = ""; + string* s = *(string**)tag; + if ( !s->empty() ) { + *buff = (void*)s->data(); + *size = s->length()+1; + return; + } + *buff = (void*)data; + *size = 1; + } + /// DIM Service update handler to publish the FSM state + static void handle_node_state(void* tag, void* address, int* size) { + if ( tag && address && size && *size > 0 ) { + SmiController* ctrl = *(SmiController**)tag; + IocSensor::instance().send(ctrl, SmiController::CONTROLLER_SETNODESTATE, new string((char*)address)); + } + } + /// DIM Service update handler to change the required number of threads in dependent processes + static void handle_num_threads(void* tag, void* address, int* size) { + if ( tag && address ) { + int len = size ? *size : 0; + if ( len > 0 ) { + SmiController* c = *(SmiController**)tag; + int m = *(int*)address; + if ( len == sizeof(int) && m > 0 ) { + c->publish_instances(m); + } + } + } + } + /// DIM client update handler to communicate state changes to the controller object + static void change_transition(void* tag, void* address, int* size) { + if ( tag && address && size && *size > 0 ) { + SmiController* controller = *(SmiController**)tag; + char cmd[512]; + size_t len = std::min((size_t)*size, sizeof(cmd)-1); + ::strncpy(cmd, (const char*)address, len); + cmd[len] = 0; + if ( controller->config.standalone ) + controller->m_nodeProxy->invoke_transition(cmd); + else + controller->invoke_transition(cmd); + } + } + + }; const auto& nam = config.name; m_monitor.targetState = m_monitor.state = MonState::UNKNOWN; m_monitor.lastCmd = m_monitor.doneCmd = (int)::time(0); @@ -813,25 +857,25 @@ void SmiController::initialize() { m_command_ID = ::dis_add_cmnd_dns( m_local_dis_dns_ID, config.name.c_str(), - "C", feed_transition, (long)this); + "C", dim_handlers::change_transition,(long)this); m_userCmd_ID = ::dis_add_cmnd_dns( m_local_dis_dns_ID, (config.name+"/command").c_str(), - "C", feed_transition, (long)this); + "C", dim_handlers::change_transition,(long)this); m_status_ID = ::dis_add_service_dns(m_local_dis_dns_ID, (nam+"/status").c_str(), - "C",0,0,feed_std_string,(long)&m_state); + "C",0,0,dim_handlers::feed_std_string,(long)&m_state); m_state_ID = ::dis_add_service_dns(m_local_dis_dns_ID, (nam+"/state").c_str(), - "C",0,0,feed_std_string,(long)&m_node_state); + "C",0,0,dim_handlers::feed_std_string,(long)&m_node_state); m_sub_status_ID = ::dis_add_service_dns(m_local_dis_dns_ID, (nam+"/fsm_status").c_str(), "L:2;I:1;C",&m_monitor,sizeof(m_monitor),0,0); m_fsm_tasks_ID = ::dis_add_service_dns(m_local_dis_dns_ID, (nam+"/tasks").c_str(), - "C",0,0,feed_std_string,(long)&m_task_info); + "C",0,0,dim_handlers::feed_std_string,(long)&m_task_info); m_fsm_tags_ID = ::dis_add_service_dns(m_local_dis_dns_ID, (nam+"/instances").c_str(), - "C",0,0,feed_std_string,(long)&m_instance_info); + "C",0,0,dim_handlers::feed_std_string,(long)&m_instance_info); m_log->info("DNS dic id: %ld TMS dic id: %ld SMI dic ID: %ld SMI dis id: %ld", m_local_dic_dns_ID,m_tms_dic_dns_ID,m_smi_dic_dns_ID,m_smi_dis_dns_ID); if ( config.num_workers < 1 ) { @@ -842,7 +886,7 @@ void SmiController::initialize() { m_log->info("Using dim service %s to adjust number of instances",config.num_thread_svc.c_str()); m_num_worker_ID = ::dic_info_service_dns(m_local_dic_dns_ID, config.num_thread_svc.c_str(), - MONITORED,0,0,0,num_thread_handler, + MONITORED,0,0,0,dim_handlers::handle_num_threads, (long)this,0,0); } string slice = config.standalone ? string("Manager") : config.smiDomain() + "_Manager"; @@ -858,7 +902,7 @@ void SmiController::initialize() { m_node_state_ID = ::dic_info_service_dns(m_local_dic_dns_ID, node_state.c_str(), MONITORED, 0, 0, 0, - node_state_handler, + dim_handlers::handle_node_state, (long)this, 0, 0); print ? m_self->setPrintOn() : m_self->setPrintOff(); print ? m_defTask->setPrintOn() : m_defTask->setPrintOff(); @@ -898,9 +942,9 @@ void SmiController::set_target_state(const std::string& new_state) { //============================================================================== void SmiController::load_tasks() { - SmiTaskConfiguration cfg(config.partition, config.architecture, config.runinfo, - config.stdout_file, config.stderr_file, - config.num_workers); + SmiTaskConfiguration cfg(config.replacements, config.partition, + config.architecture, config.runinfo, + config.stdout_file, config.stderr_file); auto tasks = cfg.taskList(config.bind_cpus, config.output_level); m_log->info("Found %ld tasks in architecture %s %s [0x%X]", tasks.size(), config.architecture.c_str(), diff --git a/Online/SmiController/src/SmiTaskConfiguration.cpp b/Online/SmiController/src/SmiTaskConfiguration.cpp index d695d023c..8732d3d40 100644 --- a/Online/SmiController/src/SmiTaskConfiguration.cpp +++ b/Online/SmiController/src/SmiTaskConfiguration.cpp @@ -37,15 +37,14 @@ using namespace FiniteStateMachine; //============================================================================== /// Standard constructor -SmiTaskConfiguration::SmiTaskConfiguration(const string& part, +SmiTaskConfiguration::SmiTaskConfiguration(const replacements_t& repl, + const string& part, const string& cfg, const string& info, const string& out, - const string& err, - int num_moore) - : m_partition(part), m_config(cfg), m_runinfo(info), - m_stdout_file(out), m_stderr_file(err), - m_instances(num_moore) + const string& err) + : m_replacements(repl), m_partition(part), m_config(cfg), m_runinfo(info), + m_stdout_file(out), m_stderr_file(err) { } @@ -57,13 +56,13 @@ SmiTaskConfiguration::~SmiTaskConfiguration() { //============================================================================== /// Analyse the configuration file and attach the corresponding slaves to the FSM machine std::map<std::string,SmiTask*> SmiTaskConfiguration::taskList(int bind_cpu, int print_level) { - char text[32], instances_text[64]; Tasklist tasks; + size_t num_sockets = 1; + map<int,string> affinity_args; + TasklistAnalyzer analyzer(tasks); + char text[32], instances_text[64]; std::map<std::string,SmiTask*> slave_tasks; string node = RTL::str_upper(RTL::nodeNameShort()); - TasklistAnalyzer analyzer(tasks); - map<int,string> affinity_args; - size_t num_sockets = 1; RTL::Logger log(RTL::Logger::getGlobalDevice(), "XmlConfig", print_level); affinity_args[0] = ""; @@ -80,32 +79,32 @@ std::map<std::string,SmiTask*> SmiTaskConfiguration::taskList(int bind_cpu, int affinity_args[i] = str.str(); } } - xml::enableEnvironResolution(false); - ::snprintf(instances_text,sizeof(instances_text),"%d",m_instances); - log.info("Partition:%s Node:%s sockets:%d instances:%s", - m_partition.c_str(), node.c_str(), num_sockets, instances_text); - xml::_toDictionary(Unicode("NUMBER_OF_INSTANCES"),Unicode(instances_text)); - ::snprintf(instances_text,sizeof(instances_text),"%d",m_instances); + log.info("Partition: %s Node: %s sockets: %d", + m_partition.c_str(), node.c_str(), num_sockets); + + xml::enableEnvironResolution(false); + for( const auto& r : m_replacements ) + xml::_toDictionary(Unicode(r.first),Unicode(r.second)); xml_h inventory = xml::DocumentHandler().load(m_config).root(); xml_coll_t(inventory,_Unicode(task)).for_each(analyzer); log.info("------------------------------------ Task list -------------------------------------"); for(Tasklist::Tasks::const_iterator i=tasks.begin(); i!=tasks.end(); ++i) { Tasklist::Task* t = *i; - long instances = t->instances; - string arguments = t->arguments(), fmc_start = t->fmcStartParams(), utgid=t->utgid; - fmc_start = RTL::str_replace(fmc_start,"${NODE}",node); - fmc_start = RTL::str_replace(fmc_start,"${PARTITION}",m_partition); - fmc_start = RTL::str_replace(fmc_start,"${RUNINFO}",m_runinfo); - fmc_start = RTL::str_replace(fmc_start,"${NAME}",t->name); - arguments = RTL::str_replace(arguments,"${NODE}",node); - arguments = RTL::str_replace(arguments,"${PARTITION}",m_partition); - arguments = RTL::str_replace(arguments,"${RUNINFO}",m_runinfo); - arguments = RTL::str_replace(arguments,"${NAME}",t->name); - utgid = RTL::str_replace(utgid,"${NODE}",node); - utgid = RTL::str_replace(utgid,"${PARTITION}",m_partition); - utgid = RTL::str_replace(utgid,"${RUNINFO}",m_runinfo); - utgid = RTL::str_replace(utgid,"${NAME}",t->name); + long instances = t->instances; + string arguments = t->arguments(), fmc_start = t->fmcStartParams(), utgid=t->utgid; + fmc_start = RTL::str_replace(fmc_start, "${NODE}", node); + fmc_start = RTL::str_replace(fmc_start, "${PARTITION}", m_partition); + fmc_start = RTL::str_replace(fmc_start, "${RUNINFO}", m_runinfo); + fmc_start = RTL::str_replace(fmc_start, "${NAME}", t->name); + arguments = RTL::str_replace(arguments, "${NODE}", node); + arguments = RTL::str_replace(arguments, "${PARTITION}", m_partition); + arguments = RTL::str_replace(arguments, "${RUNINFO}", m_runinfo); + arguments = RTL::str_replace(arguments, "${NAME}", t->name); + utgid = RTL::str_replace(utgid, "${NODE}", node); + utgid = RTL::str_replace(utgid, "${PARTITION}", m_partition); + utgid = RTL::str_replace(utgid, "${RUNINFO}", m_runinfo); + utgid = RTL::str_replace(utgid, "${NAME}", t->name); map<int,int> num_instance, init_instance, task_instance; // Init counters @@ -116,9 +115,9 @@ std::map<std::string,SmiTask*> SmiTaskConfiguration::taskList(int bind_cpu, int } // Determine the number of task instances per CPU slot - for(long j=0; j < instances; ++j) + for( long j=0; j < instances; ++j ) ++init_instance[j%num_sockets]; - for(long j=0; j < instances; ++j) + for( long j=0; j < instances; ++j ) ++num_instance[j%num_sockets]; // Create 'real' and 'internal' slaves diff --git a/Online/SmiController/src/TasklistHandlers.cpp b/Online/SmiController/src/TasklistHandlers.cpp index 23ad6c801..5187b42db 100644 --- a/Online/SmiController/src/TasklistHandlers.cpp +++ b/Online/SmiController/src/TasklistHandlers.cpp @@ -126,9 +126,8 @@ void TasklistPrinter::Params::operator()(const xml_h& h) { /// Action operator when analyzing data void TasklistAnalyzer::operator()(const xml_h& h) { - TaskParams t = h; - Task* task = new Task(); - tasks.tasks.push_back(task); + TaskParams t = h; + Task* task = new Task(); task->name = t.name(); task->user = t.user(); task->group = t.group(); @@ -136,6 +135,7 @@ void TasklistAnalyzer::operator()(const xml_h& h) { task->instances = t.instances(); task->doStart = t.start(); task->isVIP = t.is_vip(); + tasks.tasks.push_back(task); xml_coll_t(t,_U(argument)).for_each(Args(task)); xml_coll_t(t,_U(fmcparam)).for_each(Params(task)); xml_coll_t(t,_U(ioparam)).for_each(Params(task)); @@ -144,9 +144,9 @@ void TasklistAnalyzer::operator()(const xml_h& h) { /// Action operator when analyzing data void TasklistAnalyzer::Args::operator()(const xml_h& h) { - TaskParams a = h; - string val = a.hasValue() ? a.value() : string(); - task->args.push_back(make_pair(a.name(),val)); + TaskParams par = h; + string val = par.hasValue() ? par.value() : string(); + task->args.push_back(make_pair(par.name(),val)); } /// Action operator when analyzing data diff --git a/Online/SmiController/src/smi_controller.cpp b/Online/SmiController/src/smi_controller.cpp index 6bf1b7af8..b92f49d8c 100644 --- a/Online/SmiController/src/smi_controller.cpp +++ b/Online/SmiController/src/smi_controller.cpp @@ -47,6 +47,10 @@ static void help_ctrl() { " -taskconfig=[string] Path to xml file with slave information. \n" " -service=[name] DIM service name to steer number of threads. \n" " -count=[number] Number of Moore processes to be forked. \n" + " -replacements=[string] String with replacement items to interprete \n" + " the architecture. Items are name-value pairs \n" + " separated by a smi-colon: \n" + " Name1=<value1>:Name2=<value2>:.... \n" " -bindcpus=[number] Flag to bind main processors to CPU slots. \n" " \n" " -smidomain=<name> SMI domain name to join. \n" @@ -90,6 +94,7 @@ extern "C" int smi_controller(int argc, char** argv) { auto logger = std::make_shared<RTL::Logger::LogDevice>(); RTL::CLI cli(argc, argv, help_ctrl); string logger_type = "term"; + string replacements; char log_path[PATH_MAX]; int secs_sleep=0; bool dbg = false; @@ -127,25 +132,26 @@ extern "C" int smi_controller(int argc, char** argv) { config.name = RTL::processName(); config.dns = ::getenv("DIM_DNS_NODE") ? ::getenv("DIM_DNS_NODE") : ""; - cli.getopt("partition", 2, config.partition); - cli.getopt("runinfo", 2, config.runinfo); - cli.getopt("taskconfig", 2, config.architecture); - cli.getopt("service", 2, config.num_thread_svc); - cli.getopt("dns", 2, config.dns); + cli.getopt("partition", 2, config.partition); + cli.getopt("runinfo", 2, config.runinfo); + cli.getopt("taskconfig", 2, config.architecture); + cli.getopt("service", 2, config.num_thread_svc); + cli.getopt("dns", 2, config.dns); dbg = cli.getopt("debug", 3) != 0; - cli.getopt("tmsdns", 2, config.tms_dns); + cli.getopt("tmsdns", 2, config.tms_dns); - cli.getopt("smidomain", 5, config.smi_domain); - cli.getopt("smifile", 5, config.smi_file); - cli.getopt("smidns", 5, config.smi_dns); - cli.getopt("smidebug", 5, config.smi_debug); - cli.getopt("count", 2, config.num_workers); - cli.getopt("bindcpus", 2, config.bind_cpus); - cli.getopt("standalone", 2, config.standalone); + cli.getopt("smidomain", 5, config.smi_domain); + cli.getopt("smifile", 5, config.smi_file); + cli.getopt("smidns", 5, config.smi_dns); + cli.getopt("smidebug", 5, config.smi_debug); + cli.getopt("replacements", 3, replacements); + cli.getopt("count", 2, config.num_workers); + cli.getopt("bindcpus", 2, config.bind_cpus); + cli.getopt("standalone", 2, config.standalone); - cli.getopt("loggertype", 2, logger_type); - cli.getopt("print", 2, config.output_level); - cli.getopt("sleep", 2, secs_sleep); + cli.getopt("loggertype", 2, logger_type); + cli.getopt("print", 2, config.output_level); + cli.getopt("sleep", 2, secs_sleep); if ( secs_sleep > 0 ) { for(secs_sleep *= 1000; secs_sleep >= 0; secs_sleep -= 100) @@ -171,11 +177,34 @@ extern "C" int smi_controller(int argc, char** argv) { else { logger->compileFormat("%TIME %LEVEL%-8NODE: %-32PROCESS %-20SOURCE"); } + RTL::Logger::LogDevice::setGlobalDevice(logger, config.output_level); RTL::Logger log(RTL::Logger::LogDevice::getGlobalDevice(),"Controller", config.output_level); + if ( dbg ) { + log.error("+---------------------------------------+"); + log.error("| Attach debugger to %s: gdb --pid %d", config.name.c_str(), ::lib_rtl_pid()); + log.error("+---------------------------------------+"); + while( dbg ) { + ::lib_rtl_sleep(100); + } + } + config.dns = RTL::str_upper(config.dns); config.tms_dns = RTL::str_upper(config.tms_dns); config.smi_dns = RTL::str_upper(config.smi_dns); + if ( !replacements.empty() ) { + auto items = RTL::str_split(replacements,','); + for( const auto& it : items ) { + const auto& nv = RTL::str_split(it,':'); + config.replacements.emplace(nv[0], nv[1]); + } + } + /// For backwards compatibility: add the instance argument: + if ( config.replacements.find("NUMBER_OF_INSTANCES") == config.replacements.end() ) { + char text[64]; + ::snprintf(text,sizeof(text),"%d",config.num_workers); + config.replacements.emplace("NUMBER_OF_INSTANCES", text); + } if ( config.name.substr(0,3)=="P00") invalid_arg("Invalid UTGID environment='%s'\n", config.name.c_str()); else if( config.partition.empty() ) invalid_arg("Invalid argument -partition='%s'\n", config.partition.c_str()); else if( config.architecture.empty() ) invalid_arg("Invalid argument -taskcondig='%s'\n", config.architecture.c_str()); @@ -199,14 +228,6 @@ extern "C" int smi_controller(int argc, char** argv) { log.info("SMI: domain %s %s %s", config.smi_domain.c_str(), config.smi_file.empty() ? "" : "file:", config.smi_file.c_str()); - if ( dbg ) { - log.error("+---------------------------------------+"); - log.error("| Attach debugger to %s: gdb --pid %d", config.name.c_str(), ::lib_rtl_pid()); - log.error("+---------------------------------------+"); - while( dbg ) { - ::lib_rtl_sleep(100); - } - } ::dis_set_dns_node(config.dns.c_str()); ::dic_set_dns_node(config.dns.c_str()); ::dim_init(); diff --git a/Online/Storage/CMakeLists.txt b/Online/Storage/CMakeLists.txt index 9d47debc3..7d092becf 100644 --- a/Online/Storage/CMakeLists.txt +++ b/Online/Storage/CMakeLists.txt @@ -16,6 +16,7 @@ gaudi_subdir(Storage v0r1) ################################################################################ # gaudi_depends_on_subdirs(Online/OnlineBase + Online/sqldb Online/HTTP Online/dim) # @@ -35,9 +36,9 @@ gaudi_add_library(StorageClient src/client/*.cpp target_compile_definitions(StorageClient PRIVATE -DSTORAGECLIENT=1) # gaudi_add_library(StorageServer src/server/*.cpp - INCLUDE_DIRS Boost HTTP SQLite3 MySQL dim + INCLUDE_DIRS Boost HTTP sqldb dim SQLite3 MySQL NO_PUBLIC_HEADERS - LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt z dim SQLite3 Boost HTTP StorageClient) + LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt z dim SQLite3 sqldb_sqlite Boost HTTP StorageClient) # target_compile_definitions(StorageServer PRIVATE -DSTORAGECLIENT=1) if (MYSQL_FOUND) @@ -48,7 +49,7 @@ endif() # Testing # --------------------------------------------------------------------------------------- gaudi_add_library(StorageCli tests/src/*.cpp - INCLUDE_DIRS Boost HTTP SQLite3 + INCLUDE_DIRS Boost HTTP sqldb SQLite3 NO_PUBLIC_HEADERS LINK_LIBRARIES ${CMAKE_DL_LIBS} -lrt z Boost HTTP StorageClient) # diff --git a/Online/Storage/Storage/fdb_dbase.h b/Online/Storage/Storage/fdb_dbase.h index c21bad757..9ca7bd1b0 100644 --- a/Online/Storage/Storage/fdb_dbase.h +++ b/Online/Storage/Storage/fdb_dbase.h @@ -56,7 +56,8 @@ namespace Online { * \date 02.02.2021 */ struct file_t { - std::string name, state, size, date, host; + std::string name, /* state, size, */ date, host; + int64_t state, size; }; /// Interface class to database handler diff --git a/Online/Storage/src/server/fdb_dbase.cpp b/Online/Storage/src/server/fdb_dbase.cpp index 81b563917..192feaa82 100644 --- a/Online/Storage/src/server/fdb_dbase.cpp +++ b/Online/Storage/src/server/fdb_dbase.cpp @@ -47,7 +47,7 @@ fdb_dbase_t::handler_t::query(std::string& object, if ( ec == system::errc::success ) { object = file.name; host = file.host; - length = ::atol(file.size.c_str()); + length = file.size; // ::atol(file.size.c_str()); date = file.date; ::lib_rtl_output(_prt(this), "+++ Query '%s'", object.c_str()); return ec; @@ -67,7 +67,7 @@ fdb_dbase_t::handler_t::next( std::string& object, if ( ec == system::errc::success ) { object = file.name; host = file.host; - length = ::atol(file.size.c_str()); + length = file.size; // ::atol(file.size.c_str()); date = file.date; ec = this->set(object, handler_t::STATE_READ); if ( ec == system::errc::success ) { diff --git a/Online/Storage/src/server/fdb_SQLite.cpp b/Online/Storage/src/server/fdb_sqldb.cpp similarity index 72% rename from Online/Storage/src/server/fdb_SQLite.cpp rename to Online/Storage/src/server/fdb_sqldb.cpp index 41da23c57..da1a0799c 100644 --- a/Online/Storage/src/server/fdb_SQLite.cpp +++ b/Online/Storage/src/server/fdb_sqldb.cpp @@ -14,7 +14,8 @@ //========================================================================== // Framework include files -#include "fdb_SQLite.h" +#include "fdb_sqldb.h" +#include <sqldb/sqlite.h> #include <RTL/strdef.h> #include <RTL/rtl.h> @@ -24,16 +25,16 @@ using namespace boost; using namespace Online::storage; /// Initializing constructor -SQLite_handler_t::SQLite_handler_t(const std::string& p, int dbg) +sqldb_handler_t::sqldb_handler_t(const std::string& p, int dbg) : fdb_dbase_t::handler_t(dbg), path(p) { this->init(path); } /// Default destructor -SQLite_handler_t::~SQLite_handler_t() { +sqldb_handler_t::~sqldb_handler_t() { this->fini(); - if ( this->database.handle() ) { + if ( this->database.is_valid() ) { this->database.commit(); this->database.close(); } @@ -41,19 +42,19 @@ SQLite_handler_t::~SQLite_handler_t() { /// Initialize object system::error_code -SQLite_handler_t::init(const std::string& db_name) { +sqldb_handler_t::init(const std::string& db_name) { if ( !this->_inited ) { std::string err; const char* nam = db_name.c_str(); - this->database = sqlite::database::open(db_name, err); - if ( this->database.db ) { + this->database = sqldb::database::open<sqlite>(db_name, err); + if ( this->database.is_valid() ) { const char* sql = "CREATE TABLE IF NOT EXISTS Files (" "Name TEXT PRIMARY KEY, " "State INT NOT NULL, " "Size INT NOT NULL, " "Date TEXT NOT NULL, " "Host TEXT NOT NULL)"; - if ( this->database.execute(err, sql) != sqlite::traits::OK ) { + if ( this->database.execute(err, sql) != sqldb::OK ) { ::lib_rtl_output(LIB_RTL_ERROR,"%s: Create table: SQL error: %s", nam, err.c_str()); return system::error_code(system::errc::permission_denied, system::system_category()); } @@ -80,7 +81,7 @@ SQLite_handler_t::init(const std::string& db_name) { } /// Finalize object -void SQLite_handler_t::fini() { +void sqldb_handler_t::fini() { if ( this->_inited ) { this->insert_record.finalize(); this->delete_record.finalize(); @@ -93,17 +94,18 @@ void SQLite_handler_t::fini() { /// Check the existence of a given object in the database system::error_code -SQLite_handler_t::query_file(const std::string& object_name, file_t& file, int state) { +sqldb_handler_t::query_file(const std::string& object_name, file_t& file, int state) { this->query_record.reset(); - this->query_record.bind(1, object_name); - this->query_record.bind(2, state); + this->query_record.bind(0, object_name); + this->query_record.bind(1, state); int ret = this->query_record.execute(); - while ( ret == SQLITE_ROW ) { + while ( (ret = query_record.fetch_one()) == sqldb::OK ) { file.name = query_record.get<std::string>(0); - file.state = query_record.get<std::string>(1); - file.size = query_record.get<std::string>(2); + file.state = query_record.get<int>(1); + file.size = query_record.get<int64_t>(2); file.date = query_record.get<std::string>(3); file.host = query_record.get<std::string>(4); + this->query_record.reset(); return system::error_code(system::errc::success, system::system_category()); } return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); @@ -111,30 +113,24 @@ SQLite_handler_t::query_file(const std::string& object_name, file_t& file, int s /// Add a new object to the database system::error_code -SQLite_handler_t::add (const std::string& object_name, - const std::string& date, - std::size_t length, - const std::string& host) { - this->insert_record.reset(); - this->insert_record.bind(1, object_name); - this->insert_record.bind(2, STATE_OPEN); - this->insert_record.bind(3, length); - this->insert_record.bind(4, date); - this->insert_record.bind(5, host); +sqldb_handler_t::add (const std::string& object_name, + const std::string& date, + std::size_t length, + const std::string& host) { + this->query_record.reset(); + this->insert_record.bind(0, object_name); + this->insert_record.bind(1, STATE_OPEN); + this->insert_record.bind(2, length); + this->insert_record.bind(3, date); + this->insert_record.bind(4, host); this->database.begin(); int ret = this->insert_record.execute(); - this->insert_record.reset(); this->database.commit(); - if ( ret == SQLITE_DONE ) { + this->insert_record.reset(); + if ( ret == sqldb::DONE ) { return system::error_code(system::errc::success, system::system_category()); } - else if ( ret == SQLITE_CONSTRAINT_PRIMARYKEY ) { - return system::error_code(system::errc::file_exists, system::system_category()); - } - else if ( ret == SQLITE_CONSTRAINT_FOREIGNKEY ) { - return system::error_code(system::errc::file_exists, system::system_category()); - } - else if ( ret != SQLITE_OK ) { + else if ( ret != sqldb::OK ) { return system::error_code(system::errc::file_exists, system::system_category()); } return system::error_code(system::errc::success, system::system_category()); @@ -142,33 +138,31 @@ SQLite_handler_t::add (const std::string& object_name, /// Remove an object from the database system::error_code -SQLite_handler_t::del (const std::string& object_name) { - this->delete_record.reset(); - this->delete_record.bind(1, object_name); +sqldb_handler_t::del (const std::string& object_name) { + this->delete_record.bind(0, object_name); this->database.begin(); int ret = this->delete_record.execute(); this->delete_record.reset(); this->database.commit(); - if ( ret == SQLITE_OK ) + if ( ret == sqldb::OK ) return system::error_code(system::errc::success, system::system_category()); - else if ( ret == SQLITE_DONE ) + else if ( ret == sqldb::DONE ) return system::error_code(system::errc::success, system::system_category()); return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); } /// lock database record system::error_code -SQLite_handler_t::set (const std::string& object_name, int value) { - this->lock_record.reset(); - this->lock_record.bind(1, value); - this->lock_record.bind(2, object_name); +sqldb_handler_t::set (const std::string& object_name, int value) { + this->lock_record.bind(0, value); + this->lock_record.bind(1, object_name); this->database.begin(); int ret = this->lock_record.execute(); this->lock_record.reset(); this->database.commit(); - if ( ret == SQLITE_DONE ) + if ( ret == sqldb::DONE ) return system::error_code(system::errc::success, system::system_category()); - else if ( ret == SQLITE_OK ) + else if ( ret == sqldb::OK ) return system::error_code(system::errc::success, system::system_category()); return system::error_code(system::errc::no_such_file_or_directory, system::system_category()); } @@ -188,7 +182,7 @@ extern "C" int fdb_sqlite_server(int argc, char** argv) { s.create(argc, argv); auto* sql = new Online::storage::fdb_dbase_t(s.server); sql->_debug = s.application_debug; - sql->_engine.reset(new SQLite_handler_t(s.dbase, s.application_debug)); + sql->_engine.reset(new sqldb_handler_t(s.dbase, s.application_debug)); s.ptr->implementation->dbase.reset(sql); s.ptr->implementation->monitor.reset(new fdb_dbase_monitor_t()); ::dis_start_serving(srv.c_str()); diff --git a/Online/Storage/src/server/fdb_SQLite.h b/Online/Storage/src/server/fdb_sqldb.h similarity index 80% rename from Online/Storage/src/server/fdb_SQLite.h rename to Online/Storage/src/server/fdb_sqldb.h index d20063d52..46ca15dad 100644 --- a/Online/Storage/src/server/fdb_SQLite.h +++ b/Online/Storage/src/server/fdb_sqldb.h @@ -12,12 +12,12 @@ // // Author : Markus Frank //========================================================================== -#ifndef ONLINE_STORAGE_FDB_SQLITE_H -#define ONLINE_STORAGE_FDB_SQLITE_H +#ifndef ONLINE_STORAGE_FDB_SQLDB_H +#define ONLINE_STORAGE_FDB_SQLDB_H // Framework include files #include <Storage/fdb_dbase.h> -#include <CPP/sqlite.h> +#include <sqldb/sqldb.h> // C/C++ include files @@ -36,14 +36,14 @@ namespace Online { * \version 1.0 * \date 02.02.2021 */ - class SQLite_handler_t : public fdb_dbase_t::handler_t { + class sqldb_handler_t : public fdb_dbase_t::handler_t { public: std::string path; - sqlite::database database; - sqlite::statement insert_record; - sqlite::statement query_record; - sqlite::statement delete_record; - sqlite::statement lock_record; + sqldb::database database; + sqldb::statement insert_record; + sqldb::statement query_record; + sqldb::statement delete_record; + sqldb::statement lock_record; /// Check the existence of a given object in the database virtual boost::system::error_code @@ -52,10 +52,10 @@ namespace Online { public: /// Initializing constructor - SQLite_handler_t(const std::string& p, int dbg); + sqldb_handler_t(const std::string& p, int dbg); /// Default destructor - virtual ~SQLite_handler_t(); + virtual ~sqldb_handler_t(); /// Initialize object boost::system::error_code init(const std::string& db_name); @@ -77,4 +77,4 @@ namespace Online { }; } // End namespace storage } // End namespace Online -#endif // ONLINE_STORAGE_FDB_SQLITE_H +#endif // ONLINE_STORAGE_FDB_SQLDB_H diff --git a/Online/Storage/src/server/sqlite_test.cpp b/Online/Storage/src/server/sqlite_test.cpp index af0577abe..876e04f57 100644 --- a/Online/Storage/src/server/sqlite_test.cpp +++ b/Online/Storage/src/server/sqlite_test.cpp @@ -23,18 +23,20 @@ gentest.exe libStorageClientTest.so sqlite_update_database_direct -database=sq gentest.exe libStorageServer.so fdb_cli_dumpdb -database=/home/frankm/storage_files.dbase */ + // Framework inclde files -#include <CPP/sqlite.h> +#include <sqldb/sqlite.h> #include <HTTP/HttpHeader.h> #include <RTL/Logger.h> -#include "fdb_SQLite.h" +#include "fdb_sqldb.h" // C/C++ include files +#include <sstream> #include <fstream> #include <iostream> using namespace std; -using namespace sqlite; +using namespace sqldb; // // @@ -66,10 +68,10 @@ namespace { ~Setup() { ::lib_rtl_output(LIB_RTL_ALWAYS,"%s: Test finished!", name.c_str()); } - sqlite::database db() const { - sqlite::database d; - d.open(this->database); - if ( d.db == nullptr ) { + sqldb::database db() const { + sqldb::database d; + d.open<sqlite>(this->database); + if ( d.intern->db == nullptr ) { ::lib_rtl_output(LIB_RTL_ERROR,"%s: Can't open database [%s]", name.c_str(), d.errmsg()); } return d; @@ -89,7 +91,7 @@ extern "C" int sqlite_create_database(int argc, char** argv) { "Size INT NOT NULL, " "Date TEXT NOT NULL, " "Host TEXT NOT NULL)"; - if ( db.execute(error, sql) != sqlite::traits::OK ) { + if ( db.execute(error, sql) != sqldb::OK ) { ::lib_rtl_output(LIB_RTL_ERROR,"%s: failed to execute %s : %s", db.name(), sql, error.c_str()); } @@ -104,13 +106,13 @@ extern "C" int sqlite_populate_database(int argc, char** argv) { Setup setup("sqlite_populate_database", argc, argv); auto db = setup.db(); try { - sqlite::statement insert - = statement::create(db, - "INSERT INTO Files ('Name', 'State', 'Size', 'Date', 'Host') " - "VALUES ( ? , ? , ? , ? , ? )"); + sqldb::statement insert; + insert.prepare(db, + "INSERT INTO Files ('Name', 'State', 'Size', 'Date', 'Host') " + "VALUES ( ? , ? , ? , ? , ? )"); for(size_t i=0; i<setup.count; ++i) { - std::string date = http::HttpHeader::now(); - std::string host = "http://127.0.0.1:8100"; + std::string date = http::HttpHeader::now(); + std::string host = "http://127.0.0.1:8100"; size_t length = 4*1024; char obj[1024]; @@ -123,7 +125,7 @@ extern "C" int sqlite_populate_database(int argc, char** argv) { insert.bind(4, date); insert.bind(5, host); int ret = insert.execute(); - if ( ret != sqlite::traits::OK ) { + if ( ret != sqldb::OK ) { ::lib_rtl_output(LIB_RTL_ERROR,"%s: failed INSERT: %s",db.name(),insert.errmsg()); return EINVAL; } @@ -157,7 +159,7 @@ extern "C" int sqlite_populate_database_direct(int argc, char** argv) { "INSERT INTO Files ('Name', 'State', 'Size', 'Date', 'Host') " "VALUES ( '%s' , %d , %ld , '%s' , '%s' )", obj, 0, length, date.c_str(), host.c_str()); - if ( ret != sqlite::traits::OK ) { + if ( ret != sqldb::OK ) { ::lib_rtl_output(LIB_RTL_ERROR,"%s: failed INSERT: %s",db.name(),db.errmsg()); return EINVAL; } @@ -175,40 +177,32 @@ extern "C" int sqlite_populate_database_direct(int argc, char** argv) { //========================================================================== extern "C" int sqlite_read_database_direct(int argc, char** argv) { typedef Online::storage::fdb_dbase_t::file_t file_t; - struct Reader { - std::vector<file_t> files; - static int callback(void* ptr, int /* argc */, char **argv, char **/*name*/) { - Reader* rdr = (Reader*)ptr; - file_t file {argv[0], argv[1], argv[2], argv[3], argv[4]}; - rdr->files.emplace_back(file); - return 0; - } - } reader; Setup setup("sqlite_populate_database", argc, argv); string error; auto db = setup.db(); - int ret = db.execute_sql(error, - &reader, - Reader::callback, - "SELECT Name, State, Size, Date, Host" - " FROM Files" - " WHERE Size>%d",0); - if ( ret != sqlite::traits::OK ) { - } - try { - std::stringstream s; - for(size_t i=0; i<reader.files.size(); ++i) { - const file_t& f = reader.files[i]; + if ( db.is_valid() ) { + sqldb::statement query = + sqldb::statement::create(db, "SELECT Name, State, Size, Date, Host" + " FROM Files" + " WHERE Size>%d",0); + if ( query.is_valid() ) { + int ret = query.execute(); + std::stringstream s; + file_t f; + while ( (ret = query.fetch_one()) == sqldb::OK ) { + f.name = query.get<std::string>(0); + f.state = query.get<int>(1); + f.size = query.get<int64_t>(2); + f.date = query.get<std::string>(3); + f.host = query.get<std::string>(4); s << f.name << " '" << f.state << "' '" << f.size << "' '" << f.date << "' " << f.host; - ::lib_rtl_output(LIB_RTL_INFO, "TABLE: %s", s.str().c_str()); - s.str(""); + ::lib_rtl_output(LIB_RTL_INFO, "TABLE: %s", s.str().c_str()); + s.str(""); + } } db.commit(); db.close(); } - catch(const exception& e) { - ::lib_rtl_output(LIB_RTL_ERROR,"%s: exception: %s",db.name(),e.what()); - } return 0; } // @@ -216,46 +210,48 @@ extern "C" int sqlite_read_database_direct(int argc, char** argv) { //========================================================================== extern "C" int sqlite_update_database_direct(int argc, char** argv) { typedef Online::storage::fdb_dbase_t::file_t file_t; - struct Reader { - std::vector<file_t> files; - static int callback(void* ptr, int /* argc */, char** argv, char** /* name */) { - Reader* rdr = (Reader*)ptr; - file_t file {argv[0], argv[1], argv[2], argv[3], argv[4]}; - rdr->files.emplace_back(file); - return 0; - } - } reader; Setup setup("sqlite_populate_database", argc, argv); auto db = setup.db(); try { + int ret; string error; std::stringstream s; - int ret = db.execute_sql(error, - &reader, - Reader::callback, - "SELECT Name, State, Size, Date, Host" - " FROM Files" - " WHERE Size>%d",0); - if ( ret != sqlite::traits::OK ) { - ::lib_rtl_output(LIB_RTL_INFO, "SQL Failure: %s", error.c_str()); + std::vector<file_t> files; + sqldb::statement query = + sqldb::statement::create(db, "SELECT Name, State, Size, Date, Host" + " FROM Files" + " WHERE Size>%d",0); + if ( query.is_valid() ) { + std::stringstream s; + file_t f; + ret = query.execute(); + while ( (ret = query.fetch_one()) == sqldb::OK ) { + f.name = query.get<std::string>(0); + f.state = query.get<int>(1); + f.size = query.get<int64_t>(2); + f.date = query.get<std::string>(3); + f.host = query.get<std::string>(4); + files.push_back(f); + } } - for(size_t i=0; i<reader.files.size(); ++i) { - const file_t& f = reader.files[i]; - if ( f.state == "0" ) { + for(size_t i=0; i<files.size(); ++i) { + const file_t& f = files[i]; + ret = sqldb::ERROR; + if ( f.state == 0 ) { ret = db.execute_sql(error, - "UPDATE Files" - " SET State=1" - " WHERE Name='%s'", - f.name.c_str()); + "UPDATE Files" + " SET State=1" + " WHERE Name='%s'", + f.name.c_str()); } - else if ( f.state == "1" ) { + else if ( f.state == 1 ) { ret = db.execute_sql(error, - "UPDATE Files" - " SET State=0" - " WHERE Name='%s'", - f.name.c_str()); + "UPDATE Files" + " SET State=0" + " WHERE Name='%s'", + f.name.c_str()); } - if ( ret != sqlite::traits::OK ) { + if ( ret != sqldb::OK ) { ::lib_rtl_output(LIB_RTL_INFO, "SQL Failure: %s", error.c_str()); } } @@ -272,14 +268,6 @@ extern "C" int sqlite_update_database_direct(int argc, char** argv) { //========================================================================== extern "C" int fdb_cli_dumpdb(int argc, char** argv) { typedef Online::storage::fdb_dbase_t::file_t file_t; - struct Reader { - static int callback(void* ptr, int /* argc */, char **argv, char **/*name*/) { - std::vector<file_t>* f = (std::vector<file_t>*)ptr; - file_t file {argv[0], argv[1], argv[2], argv[3], argv[4]}; - f->emplace_back(file); - return 0; - } - }; Setup setup("fdb_cli_dumpdb", argc, argv, [](int,char**) { ::fprintf(stderr, "fdb_cli_dumpdb -opt [-opt] \n" @@ -288,23 +276,32 @@ extern "C" int fdb_cli_dumpdb(int argc, char** argv) { }); string error; std::vector<file_t> files; - sqlite::database database = setup.db(); - if ( database.handle() == nullptr ) { + sqldb::database database = setup.db(); + if ( !database.is_valid() ) { ::lib_rtl_output(LIB_RTL_ERROR,"%s: Can't open database [%s]", setup.database.c_str(), database.errmsg()); return EINVAL; } - int ret = database.execute_sql(error, &files, Reader::callback, - "SELECT Name, State, Size, Date, Host" - " FROM Files" - " WHERE Size>0"); - if ( ret == sqlite::traits::OK ) { - for(size_t i=0; i<files.size(); ++i) { - const file_t& f = files[i]; + + sqldb::statement query = + sqldb::statement::create(database, "SELECT Name, State, Size, Date, Host" + " FROM Files" + " WHERE Size>%d",0); + if ( query.is_valid() ) { + int ret = query.execute(); + int count = 0; + file_t f; + while ( (ret = query.fetch_one()) == sqldb::OK ) { + f.name = query.get<std::string>(0); + f.state = query.get<int>(1); + f.size = query.get<int64_t>(2); + f.date = query.get<std::string>(3); + f.host = query.get<std::string>(4); ::lib_rtl_output(LIB_RTL_INFO, "%4ld File: %s Host: %s %s", - i, f.name.c_str(), f.host.c_str(), f.date.c_str()); - ::lib_rtl_output(LIB_RTL_INFO, " State:%s %9s bytes", - f.state.c_str(), f.size.c_str()); + count, f.name.c_str(), f.host.c_str(), f.date.c_str()); + ::lib_rtl_output(LIB_RTL_INFO, " State:%d %9ld bytes", + f.state, f.size); + ++count; } } database.close(); diff --git a/Online/Storage/tests/src/fdb_cli.cpp b/Online/Storage/tests/src/fdb_cli.cpp index bc3fac4f1..b0f491645 100644 --- a/Online/Storage/tests/src/fdb_cli.cpp +++ b/Online/Storage/tests/src/fdb_cli.cpp @@ -40,7 +40,6 @@ gentest.exe libStorageCli.so fdb_cli_db_delete -server=${HOST}:8000 -url=/o // C/C++ include files #include <iostream> #include <boost/filesystem.hpp> -#include <CPP/sqlite.h> using namespace std; using namespace Online::storage; diff --git a/Online/TestBeam/TestBeam/gui/NodeFSMPanel.h b/Online/TestBeam/TestBeam/gui/NodeFSMPanel.h index c6bd31f32..9d4e290f9 100644 --- a/Online/TestBeam/TestBeam/gui/NodeFSMPanel.h +++ b/Online/TestBeam/TestBeam/gui/NodeFSMPanel.h @@ -73,7 +73,7 @@ namespace testbeam { CPP::Interactor* gui = 0; TGTextButton* apply = 0; - LineEntry maindns, dns, node, part, arch, numslave; + LineEntry maindns, dns, node, part, script, arch, numslave; TaskEntry tmSrv, logSrv, logViewer, mbmmon, tanSrv, ctrl, did; std::vector<Child> children; TGComboBox* ctrl_command = 0; @@ -101,7 +101,7 @@ namespace testbeam { std::string m_user; std::string m_DNS; std::string m_mainDNS; - std::string m_ctrlType; + std::string m_ctrlType, m_ctrlScript; std::string m_userName; std::string m_logFifo; @@ -113,7 +113,8 @@ namespace testbeam { std::string m_didName; std::string m_ctrlName; - + int m_mainDNS_id = -1; + public: enum IDS { PART_ID_OFFSET = 100, @@ -129,6 +130,8 @@ namespace testbeam { NUMSLAVE_INPUT = PART_ID_OFFSET+10, ARCH_LABEL = PART_ID_OFFSET+11, ARCH_INPUT = PART_ID_OFFSET+12, + SCRIPT_LABEL = PART_ID_OFFSET+13, + SCRIPT_INPUT = PART_ID_OFFSET+14, TMSRV_ID_OFFSET = 200, TMSRV_LABEL = TMSRV_ID_OFFSET+1, @@ -191,6 +194,9 @@ namespace testbeam { void setArchitecture(const std::string& value); void architectureChanged(const char* value); + void setScript(const std::string& value); + void scriptChanged(const char* value); + void setPartition(const std::string& value); void partitionChanged(const char* value); diff --git a/Online/TestBeam/TestBeam/gui/TaskManager.h b/Online/TestBeam/TestBeam/gui/TaskManager.h new file mode 100644 index 000000000..eea63f259 --- /dev/null +++ b/Online/TestBeam/TestBeam/gui/TaskManager.h @@ -0,0 +1,70 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Author : Markus Frank +//========================================================================== +// +// Finite state machine implementation to control +// and manipulate process groups +// +//========================================================================== +#ifndef ONLINE_FINITESTATEMACHINE_TASKMANAGER_H +#define ONLINE_FINITESTATEMACHINE_TASKMANAGER_H + +// Framework include files + +// C/C++ include files +#include <string> + +/* + * FiniteStateMachine namespace declaration + */ +namespace FiniteStateMachine { + + /**@class TaskManager TaskManager.h Ctrl/TaskManager.h + * + * @author M.Frank + * @date 01/03/2013 + * @version 0.1 + */ + class TaskManager { + protected: + /// Node name + std::string m_node; + /// Name of the DIM command to start a process + std::string m_start; + /// Name of the DIM command to stop a process + std::string m_stop; + /// Name of the DIM command to kill a process + std::string m_kill; + /// DIM domain ID + long m_domain = -1; + + /// Copy constructor + TaskManager(const TaskManager& c) = default; + /// Assignment operator + TaskManager& operator=(const TaskManager& c) = default; + public: + /// Standard constructor + TaskManager(const std::string& node, long domain); + /// Standard destructor + virtual ~TaskManager() = default; + + /// Instance accessor + static TaskManager& instance(const std::string& node, long domain=0); + /// Start a process + int start(const std::string& utgid, const std::string& fmc_args, const std::string& cmd, const std::string& args) const; + /// Kill a process + int stop(const std::string& utgid, int sig_num, int wait_before_kill) const; + /// Kill a process + int kill(const std::string& utgid, int sig_num) const; + }; // End class State +} // End namespace +#endif // ONLINE_FINITESTATEMACHINE_TASKMANAGER diff --git a/Online/TestBeam/TestBeam/gui/config.h b/Online/TestBeam/TestBeam/gui/config.h index cf8ec0e8e..1f32013af 100644 --- a/Online/TestBeam/TestBeam/gui/config.h +++ b/Online/TestBeam/TestBeam/gui/config.h @@ -1,11 +1,24 @@ #ifndef TESTBEAM_CONFIG_H #define TESTBEAM_CONFIG_H -#define TMSTART_COMMAND "LD_LIBRARY_PATH=/home/frankm/bin/FMC-3.9.15/lib tmStart" -#define TMKILL_COMMAND "LD_LIBRARY_PATH=/home/frankm/bin/FMC-3.9.15/lib tmKill" -#define LOGVIEWER_COMMAND "LD_LIBRARY_PATH=/home/frankm/bin/FMC-3.9.15/lib /home/frankm/bin/FMC-3.9.15/bin/logViewer" -#define LOGSRV_COMMAND "-DLD_LIBRARY_PATH=/home/frankm/bin/FMC-3.9.15/lib /home/frankm/bin/FMC-3.9.15/sbin/logSrv" -#define TMSRV_COMMAND "-DLD_LIBRARY_PATH=/home/frankm/bin/FMC-3.9.15/lib /home/frankm/bin/FMC-3.9.15/sbin/tmSrv" +#if 0 + +#define TMSTART_COMMAND "LD_LIBRARY_PATH=/home/frankm/bin/FMC-3.9.15.old/lib tmStart" +#define TMKILL_COMMAND "LD_LIBRARY_PATH=/home/frankm/bin/FMC-3.9.15.old/lib tmKill" +#define LOGVIEWER_COMMAND "LD_LIBRARY_PATH=/home/frankm/bin/FMC-3.9.15.old/lib /home/frankm/bin/FMC-3.9.15.old/bin/logViewer" +#define LOGSRV_COMMAND "-DLD_LIBRARY_PATH=/home/frankm/bin/FMC-3.9.15.old/lib /home/frankm/bin/FMC-3.9.15.old/sbin/logSrv" +#define TMSRV_COMMAND "-DLD_LIBRARY_PATH=/home/frankm/bin/FMC-3.9.15.old/lib /home/frankm/bin/FMC-3.9.15.old/sbin/tmSrv" + +#else + +#define TMSTART_COMMAND "LD_LIBRARY_PATH=/opt/FMC/lib /opt/FMC/bin/tmStart" +#define TMKILL_COMMAND "LD_LIBRARY_PATH=/opt/FMC/lib /opt/FMC/bin/tmKill" +#define LOGVIEWER_COMMAND "LD_LIBRARY_PATH=/opt/FMC/lib /opt/FMC/bin/logViewer" +#define LOGSRV_COMMAND "-DLD_LIBRARY_PATH=/opt/FMC/lib /opt/FMC/sbin/logSrv" +#define TMSRV_COMMAND "-DLD_LIBRARY_PATH=/opt/FMC/lib /opt/FMC/sbin/tmSrv" + +#endif + #define CONTROLLER_TYPE "Controller" //#define CONTROLLER_COMMAND_OPT "" diff --git a/Online/TestBeam/job/Controller.sh b/Online/TestBeam/job/Controller.sh deleted file mode 100755 index 568878b59..000000000 --- a/Online/TestBeam/job/Controller.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# ========================================================================= -# -# Farm worker node controller startup script -# -# Author M.Frank -# Version: 1.0 -# Date: 20/05/2013 -# -# ========================================================================= -HOST=`hostname -s | tr a-z A-Z`; -HOST_LONG=`hostname -f | tr a-z A-Z`; -# echo "ERROR DIM_DNS_NODE: ${DIM_DNS_NODE}" -# -# Check the existence of various arguments. -# Otherwise use default values. -# -if test -z "${TMS_DNS}"; then - export TMS_DNS=${HOST_LONG}; -fi; -if test -z "${SMI_DNS}"; then - export SMI_DNS=${HOST_LONG}; -fi; -if test -z "${SMI_FILE}"; then - export SMI_FILE=${SMICONTROLLERROOT}/options/MonNode -fi; -if test -z "${SMI_DOMAIN}"; then - export SMI_DOMAIN=${PARTITION_NAME}_${HOST}_SMI; -fi; -if test -z "${DIM_DNS_NODE}"; then - export DIM_DNS_NODE=${HOST_LONG}; -fi; -# -exec -a ${UTGID} `which genRunner.exe` libSmiController.so smi_controller \ - -print=4 \ - -logger=fifo \ - -part=${PARTITION_NAME} \ - -dns=${DIM_DNS_NODE} \ - -tmsdns=${TMS_DNS} \ - -smidns=${SMI_DNS} \ - -smidomain=${SMI_DOMAIN} \ - -smidebug=0 \ - -smifile=${SMI_FILE} \ - -count=${NBOFSLAVES} \ - -service=none \ - -runinfo=${RUNINFO} \ - -taskconfig=${ARCH_FILE} \ - -standalone=2 \ - -bindcpus=0 diff --git a/Online/TestBeam/job/Monitor.sh b/Online/TestBeam/job/Monitor.sh deleted file mode 100755 index c27d3f4bc..000000000 --- a/Online/TestBeam/job/Monitor.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# ========================================================================= -# -# Default script to start the passthrough process on a farm node. -# -# Author M.Frank -# Version: 1.0 -# Date: 20/05/2013 -# -# ========================================================================= -# -export CHECKPOINT_SETUP_OPTIONS=${FARMCONFIGROOT}/options/Checkpoint.opts; - -if test "${TAE_PROCESSING}" = "TAE"; -then - OPTIONS=-opts=../options/${TASK_TYPE}_TAE.opts; -else - OPTIONS=-opts=../options/${TASK_TYPE}.opts; -fi; -# -export PYTHONPATH=${FARMCONFIGROOT}/job:${PYTHONPATH}; -#echo `python ${FARMCONFIGROOT}/job/ConfigureCheckpoint.py -r ${RUNINFO} -s` | tr ";" ";\n"; -eval `python ${FARMCONFIGROOT}/job/ConfigureCheckpoint.py -r ${RUNINFO} -s`; -if test "${APP_STARTUP_OPTS}" = "-restore"; ## RunInfo flag=2 - then - echo "+++ [INFO] ${ONLINE_PROJECT_ROOT} ==> ${RESTORE_CMD}"; - mkdir -p /tmp/Commands; - python -c "import ConfigureFromCheckpoint" > /tmp/Commands/$$.cmds; - ${RESTORE_CMD} < /tmp/Commands/$$.cmds; - #eval "python -c \"import ConfigureFromCheckpoint\" | ${RESTORE_CMD}"; -else - echo "+++ [INFO] ${ONLINE_PROJECT_ROOT} ==> exec -a ${UTGID} ${Checkpoint_task} ${OPTIONS}"; - echo "+++ [INFO] exec -a ${UTGID} ${Checkpoint_task} ${OPTIONS} ${APP_STARTUP_OPTS}"; - exec -a ${UTGID} ${Checkpoint_task} ${OPTIONS} ${APP_STARTUP_OPTS}; - #gdb --args ${Checkpoint_task} ${OPTIONS} ${APP_STARTUP_OPTS}; -fi; diff --git a/Online/TestBeam/job/PassThrough.sh b/Online/TestBeam/job/PassThrough.sh deleted file mode 100755 index 8b6f42131..000000000 --- a/Online/TestBeam/job/PassThrough.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -# ========================================================================= -# -# Default script to start the passthrough process on a farm node. -# -# Author M.Frank -# Version: 1.0 -# Date: 20/05/2013 -# -# ========================================================================= -# -export ONLINETASKS=/group/online/dataflow/templates; -export PARTITIONOPTS=/group/online/dataflow/options/${PARTNAME}/${PARTNAME}_Info.opts; -export PARTITIONOPTS=${INFO_OPTIONS}; -export CHECKPOINT_SETUP_OPTIONS=${FARMCONFIGROOT}/options/Checkpoint.opts; -export CHECKPOINT_RESTART_OPTIONS=${FARMCONFIGROOT}/options/CheckpointRestart.opts; - -OPTIONS=-opts=../options/${TASK_TYPE}.opts; - -if test "${TAE_PROCESSING}" = "TAE"; -then - OPTIONS=-opts=../options/${TASK_TYPE}_TAE.opts; -fi; -# -export PYTHONPATH=${FARMCONFIGROOT}/job:${PYTHONPATH}; -echo `python ${FARMCONFIGROOT}/job/ConfigureCheckpoint.py -r ${RUNINFO} -s` | tr ";" ";\n"; -eval `python ${FARMCONFIGROOT}/job/ConfigureCheckpoint.py -r ${RUNINFO} -s`; -if test "${APP_STARTUP_OPTS}" = "-restore"; ## RunInfo flag=2 - then - echo "+++ [INFO] ${ONLINE_PROJECT_ROOT} ==> ${RESTORE_CMD}"; - mkdir -p /tmp/Commands; - python -c "import ConfigureFromCheckpoint" > /tmp/Commands/$$.cmds; - ${RESTORE_CMD} < /tmp/Commands/$$.cmds; - #eval "python -c \"import ConfigureFromCheckpoint\" | ${RESTORE_CMD}"; -else - echo "+++ [INFO] ${ONLINE_PROJECT_ROOT} ==> exec -a ${UTGID} ${Checkpoint_task} ${OPTIONS}"; -# HH=`hostname -s | tr a-z A-Z`; -# if test "${HH}" = "HLTA0328"; then -# echo "+++ [INFO] Executing modified online version from frankm."; -# . /home/frankm/cmtuser/OnlineDev_v5r24/setup.x86_64-slc6-gcc48-dbg.vars; -# fi; -# export LD_LIBRARY_PATH="/home/beat/cmtuser/OnlineDev_v5r31/InstallArea/x86_64-slc6-gcc49-do0/lib:/group/online/dataflow/cmtuser/OnlineDev_v5r31/InstallArea/x86_64-slc6-gcc49-do0/lib:/cvmfs/lhcb.cern.ch/lib/lcg/releases/Boost/1.59.0_python2.7-682d6/x86_64-slc6-gcc49-dbg/lib:$LD_LIBRARY_PATH" -# - exec -a ${UTGID} ${Checkpoint_task} ${OPTIONS} ${APP_STARTUP_OPTS}; -fi; diff --git a/Online/TestBeam/job/SmiController.sh b/Online/TestBeam/job/SmiController.sh deleted file mode 100644 index d246f1857..000000000 --- a/Online/TestBeam/job/SmiController.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# ========================================================================= -# -# Farm worker node controller startup script -# -# Author M.Frank -# Version: 1.0 -# Date: 20/05/2013 -# -# ========================================================================= -HOST=`hostname -s | tr a-z A-Z`; -echo "ERROR DIM_DNS_NODE: ${DIM_DNS_NODE}" -exec -a ${UTGID} `which gentest.exe` libSmiController.so smi_controller \ - -print=1 \ - -logger=fifo \ - -part=${PARTITION_NAME} \ - -dns=${DIM_DNS_NODE} \ - -tmsdns=${TMS_DNS} \ - -smidns=${SMI_DNS} \ - -smidomain=${SMI_DOMAIN} \ - -smisteer=2 \ - -smifile=/home/frankm/upgrade_sw/Online/NODE \ - -count=1 \ - -service=none \ - -runinfo=${RUNINFO} \ - -taskconfig=${ARCH_FILE} - -## -smidomain=${PARTITION_NAME}_${HOST}_CTRL \ -## -logger=fmc \ -## -tms_dns=fmc01 \ -## -smidns=pluscc06 \ -## -standalone=0 diff --git a/Online/TestBeam/job/SmiTestController.sh b/Online/TestBeam/job/SmiTestController.sh deleted file mode 100755 index db6055910..000000000 --- a/Online/TestBeam/job/SmiTestController.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# ========================================================================= -# -# Farm worker node controller startup script -# -# Author M.Frank -# Version: 1.0 -# Date: 20/05/2013 -# -# ========================================================================= -HOST=`hostname -s | tr a-z A-Z`; -HOST_LONG=`hostname -f | tr a-z A-Z`; -echo "ERROR DIM_DNS_NODE: ${DIM_DNS_NODE}" -if test -z "${SMI_DNS}"; then - export SMI_DNS=${HOST_LONG}; -fi; -if test -z "${TMS_DNS}"; then - export TMS_DNS=${HOST_LONG}; -fi; -if test -z "${SMI_DOMAIN}"; then - export SMI_DOMAIN=${PARTITION_NAME}_${HOST}_SMI; -fi; -if test -z "${DIM_DNS_NODE}"; then - export DIM_DNS_NODE=${HOST_LONG}; -fi; -# -exec -a ${UTGID} `which gentest.exe` libSmiController.so smi_controller \ - -print=4 \ - -logger=fifo \ - -part=${PARTITION_NAME} \ - -dns=${DIM_DNS_NODE} \ - -tmsdns=${TMS_DNS} \ - -smidns=${SMI_DNS} \ - -smidomain=${SMI_DOMAIN} \ - -smidebug=0 \ - -smifile=${SMICONTROLLERROOT}/options/MonNode \ - -count=${NBOFSLAVES} \ - -service=none \ - -runinfo=${RUNINFO} \ - -taskconfig=${ARCH_FILE} \ - -standalone=2 \ - -bindcpus=1 diff --git a/Online/TestBeam/job/runTask.sh b/Online/TestBeam/job/runTask.sh deleted file mode 100755 index c5fef55ed..000000000 --- a/Online/TestBeam/job/runTask.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# ========================================================================= -# -# Default script to start any task on the HLT farm. -# -# Author M.Frank -# Version: 1.0 -# Date: 20/05/2013 -# -# ========================================================================= -# -export SUBFARM=`echo ${DIM_DNS_NODE} | tr a-z A-Z`; -export STATIC_OPTS=${FARMCONFIGROOT}/options; -if test -z "${DATAINTERFACE}"; then - DATAINTERFACE=`python /group/online/dataflow/scripts/getDataInterface.py`; -fi; -export DATAINTERFACE; -export STATIC_OPTS=${FARMCONFIGROOT}/options -export OPTIONS_DIR=${TESTBEAMROOT}/options; -export INFO_OPTIONS=${OPTIONS_DIR}/OnlineEnv.opts; -export MBM_SETUP_OPTIONS=${OPTIONS_DIR}/MBMSetup.opts; -export DATAFLOW_task="gentest.exe libDataflow.so dataflow_run_task -msg=fifo -mon=Dataflow_DIMMonitoring -print=${OUTPUT_LEVEL}"; -# -if test -f ./${TASK_TYPE}.sh; -then - #echo "RunTask: TASK_TYPE: ${TASK_TYPE} ./${TASK_TYPE}.sh" - . ./${TASK_TYPE}.sh $*; -elif test -f ./${PARTITION_NAME}${TASK_TYPE}.sh; -then - # echo "RunTask: TASK_TYPE: ${TASK_TYPE} ./${PARTITION_NAME}${TASK_TYPE}.sh" - . ./${PARTITION_NAME}${TASK_TYPE}.sh; -elif test -f ./${PARTITION_NAME}DefaultTask.sh; -then - # echo "RunTask: TASK_TYPE: ${TASK_TYPE} ./${PARTITION_NAME}DefaultTask.sh" - . ./${PARTITION_NAME}DefaultTask.sh; -else - # echo "RunTask: TASK_TYPE: ${TASK_TYPE} ./DefaultTask.sh" - . ./DefaultTask.sh; -fi; diff --git a/Online/TestBeam/src/gui/NodeFSMPanel.cpp b/Online/TestBeam/src/gui/NodeFSMPanel.cpp index b0c99c930..f3d349384 100644 --- a/Online/TestBeam/src/gui/NodeFSMPanel.cpp +++ b/Online/TestBeam/src/gui/NodeFSMPanel.cpp @@ -18,6 +18,7 @@ #include "TestBeam/gui/GuiException.h" #include "TestBeam/gui/GuiCommand.h" #include "TestBeam/gui/GuiMsg.h" +#include "TestBeam/gui/TaskManager.h" #include "CPP/TimeSensor.h" #include "CPP/IocSensor.h" @@ -82,6 +83,7 @@ NodeFSMPanel::NodeFSMPanel(TGFrame* pParent, CPP::Interactor* g, RTL::CLI& opts) opts.getopt("architecture",2, m_architecture); opts.getopt("startup", 2, m_startup); opts.getopt("controller", 2, m_ctrlType); + opts.getopt("ctrl_script", 9, m_ctrlScript); gClient->GetColorByName("#c0c0c0", disabled); gClient->GetColorByName("white", enabled); gClient->GetColorByName("#30B030", green); @@ -122,6 +124,9 @@ void NodeFSMPanel::init() { numslave.label = new TGLabel( para_group, "Num.slaves", NUMSLAVE_LABEL); numslave.input = new TGTextEntry(para_group, str.str().c_str(), NUMSLAVE_INPUT); + script.label = nullptr; + script.input = new TGTextEntry(para_group, m_ctrlScript.c_str(), SCRIPT_INPUT); + arch.label = new TGLabel( para_group, "Architecture", ARCH_LABEL); arch.input = new TGTextEntry(para_group, m_architecture.c_str(), ARCH_INPUT); @@ -129,6 +134,7 @@ void NodeFSMPanel::init() { part.input = new TGTextEntry(para_group, m_partition.c_str(), PART_INPUT); arch.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "architectureChanged(const char*)"); + script.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "scriptChanged(const char*)"); maindns.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "mainDnsChanged(const char*)"); dns.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "dnsChanged(const char*)"); node.input->Connect( "TextChanged(const char*)", "testbeam::NodeFSMPanel", this, "hostChanged(const char*)"); @@ -211,59 +217,62 @@ void NodeFSMPanel::init() { mbmmon.start->SetEnabled(kFALSE); mbmmon.kill->SetEnabled(kFALSE); #define _PAD 0 - para_group->AddFrame(maindns.label, new TGTableLayoutHints(0, 1, 0, 1, kLHintsLeft |kLHintsCenterY, _PAD,40,_PAD,_PAD)); - para_group->AddFrame(maindns.input, new TGTableLayoutHints(1, 2, 0, 1, kLHintsLeft |kLHintsCenterY|kLHintsExpandY, _PAD,_PAD,10,_PAD)); - para_group->AddFrame(dns.label, new TGTableLayoutHints(0, 1, 1, 2, kLHintsLeft |kLHintsCenterY, _PAD,40,_PAD,_PAD)); - para_group->AddFrame(dns.input, new TGTableLayoutHints(1, 2, 1, 2, kLHintsLeft |kLHintsCenterY|kLHintsExpandY, _PAD,_PAD,10,_PAD)); - para_group->AddFrame(node.label, new TGTableLayoutHints(0, 1, 2, 3, kLHintsLeft |kLHintsCenterY, _PAD,40,_PAD,_PAD)); - para_group->AddFrame(node.input, new TGTableLayoutHints(1, 2, 2, 3, kLHintsLeft |kLHintsCenterY|kLHintsExpandY, _PAD,_PAD,10,_PAD)); - para_group->AddFrame(part.label, new TGTableLayoutHints(0, 1, 3, 4, kLHintsLeft |kLHintsCenterY, _PAD,40,_PAD,_PAD)); - para_group->AddFrame(part.input, new TGTableLayoutHints(1, 2, 3, 4, kLHintsLeft |kLHintsCenterY|kLHintsExpandY, _PAD,_PAD,10,_PAD)); - para_group->AddFrame(numslave.label, new TGTableLayoutHints(0, 1, 4, 5, kLHintsLeft |kLHintsCenterY, _PAD,40,_PAD,_PAD)); - para_group->AddFrame(numslave.input, new TGTableLayoutHints(1, 2, 4, 5, kLHintsLeft |kLHintsCenterY|kLHintsExpandY, _PAD,_PAD,10,_PAD)); - - para_group->AddFrame(arch.label, new TGTableLayoutHints(0, 1, 8, 9, kLHintsLeft |kLHintsCenterY, _PAD,40,_PAD,_PAD)); - para_group->AddFrame(arch.input, new TGTableLayoutHints(1, 5, 8, 9, kLHintsLeft |kLHintsCenterY|kLHintsExpandY|kLHintsExpandX, _PAD,_PAD,10,_PAD)); - para_group->AddFrame(did.label, new TGTableLayoutHints(5, 6, 8, 9, kLHintsCenterX |kLHintsCenterY|kLHintsExpandY|kLHintsExpandX, _PAD,_PAD,10,_PAD)); - para_group->AddFrame(did.start, new TGTableLayoutHints(6, 7, 8, 9, kLHintsCenterX |kLHintsCenterY|kLHintsExpandY|kLHintsExpandX, _PAD,_PAD,10,_PAD)); - para_group->AddFrame(did.kill, new TGTableLayoutHints(7, 8, 8, 9, kLHintsCenterX |kLHintsCenterY|kLHintsExpandY|kLHintsExpandX, _PAD,_PAD,10,_PAD)); - para_group->AddFrame(apply, new TGTableLayoutHints(8, 9, 8, 9, kLHintsCenterX |kLHintsCenterY|kLHintsExpandY|kLHintsExpandX, _PAD,_PAD,10,_PAD)); - - para_group->AddFrame(logSrv.label, new TGTableLayoutHints(3, 4, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,40,10,_PAD)); - para_group->AddFrame(logSrv.utgid, new TGTableLayoutHints(4, 5, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(logSrv.start, new TGTableLayoutHints(5, 6, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(logSrv.kill, new TGTableLayoutHints(6, 7, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(logSrv.status, new TGTableLayoutHints(7, 9, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - - para_group->AddFrame(tmSrv.label, new TGTableLayoutHints(3, 4, 1, 2, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,40,10,_PAD)); - para_group->AddFrame(tmSrv.utgid, new TGTableLayoutHints(4, 5, 1, 2, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(tmSrv.start, new TGTableLayoutHints(5, 6, 1, 2, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(tmSrv.kill, new TGTableLayoutHints(6, 7, 1, 2, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(tmSrv.status, new TGTableLayoutHints(7, 9, 1, 2, kLHintsLeft|kLHintsCenterY|kLHintsExpandY|kLHintsExpandX, 30,_PAD,10,_PAD)); - - para_group->AddFrame(tanSrv.label, new TGTableLayoutHints(3, 4, 2, 3, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,40,10,_PAD)); - para_group->AddFrame(tanSrv.utgid, new TGTableLayoutHints(4, 5, 2, 3, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(tanSrv.start, new TGTableLayoutHints(5, 6, 2, 3, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(tanSrv.kill, new TGTableLayoutHints(6, 7, 2, 3, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - //para_group->AddFrame(tanSrv.status, new TGTableLayoutHints(7, 9, 2, 3, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - - para_group->AddFrame(logViewer.label, new TGTableLayoutHints(3, 4, 3, 4, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,40,10,_PAD)); - para_group->AddFrame(logViewer.utgid, new TGTableLayoutHints(4, 5, 3, 4, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(logViewer.start, new TGTableLayoutHints(5, 6, 3, 4, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(logViewer.kill, new TGTableLayoutHints(6, 7, 3, 4, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - //para_group->AddFrame(logViewer.status, new TGTableLayoutHints(7, 9, 3, 4, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - - para_group->AddFrame(mbmmon.label, new TGTableLayoutHints(3, 4, 4, 5, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,40,10,_PAD)); - para_group->AddFrame(mbmmon.utgid, new TGTableLayoutHints(4, 5, 4, 5, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(mbmmon.start, new TGTableLayoutHints(5, 6, 4, 5, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(mbmmon.kill, new TGTableLayoutHints(6, 7, 4, 5, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - //para_group->AddFrame(mbmmon.status, new TGTableLayoutHints(7, 9, 4, 5, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - - para_group->AddFrame(ctrl.label, new TGTableLayoutHints(3, 4, 5, 6, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,40,10,_PAD)); - para_group->AddFrame(ctrl.utgid, new TGTableLayoutHints(4, 5, 5, 6, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(ctrl.start, new TGTableLayoutHints(5, 6, 5, 6, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(ctrl.kill, new TGTableLayoutHints(6, 7, 5, 6, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); - para_group->AddFrame(ctrl.status, new TGTableLayoutHints(7, 9, 5, 6, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, 30,_PAD,10,_PAD)); +#define _PAD2 15 +#define _PADY 35 + para_group->AddFrame(maindns.label, new TGTableLayoutHints(0, 1, 0, 1, kLHintsLeft |kLHintsCenterY, _PAD,_PADY,_PAD,_PAD)); + para_group->AddFrame(maindns.input, new TGTableLayoutHints(1, 2, 0, 1, kLHintsLeft |kLHintsCenterY|kLHintsExpandY, _PAD,_PAD,_PAD2,_PAD)); + para_group->AddFrame(dns.label, new TGTableLayoutHints(0, 1, 1, 2, kLHintsLeft |kLHintsCenterY, _PAD,_PADY,_PAD,_PAD)); + para_group->AddFrame(dns.input, new TGTableLayoutHints(1, 2, 1, 2, kLHintsLeft |kLHintsCenterY|kLHintsExpandY, _PAD,_PAD,_PAD2,_PAD)); + para_group->AddFrame(node.label, new TGTableLayoutHints(0, 1, 2, 3, kLHintsLeft |kLHintsCenterY, _PAD,_PADY,_PAD,_PAD)); + para_group->AddFrame(node.input, new TGTableLayoutHints(1, 2, 2, 3, kLHintsLeft |kLHintsCenterY|kLHintsExpandY, _PAD,_PAD,_PAD2,_PAD)); + para_group->AddFrame(part.label, new TGTableLayoutHints(0, 1, 3, 4, kLHintsLeft |kLHintsCenterY, _PAD,_PADY,_PAD,_PAD)); + para_group->AddFrame(part.input, new TGTableLayoutHints(1, 2, 3, 4, kLHintsLeft |kLHintsCenterY|kLHintsExpandY, _PAD,_PAD,_PAD2,_PAD)); + para_group->AddFrame(numslave.label, new TGTableLayoutHints(0, 1, 4, 5, kLHintsLeft |kLHintsCenterY, _PAD,_PADY,_PAD,_PAD)); + para_group->AddFrame(numslave.input, new TGTableLayoutHints(1, 2, 4, 5, kLHintsLeft |kLHintsCenterY|kLHintsExpandY, _PAD,_PAD,_PAD2,_PAD)); + para_group->AddFrame(script.input, new TGTableLayoutHints(0, 2, 5, 6, kLHintsLeft |kLHintsCenterY|kLHintsExpandY|kLHintsExpandX, _PAD,_PAD,_PAD2,_PAD)); + para_group->AddFrame(arch.label, new TGTableLayoutHints(0, 1, 8, 9, kLHintsLeft |kLHintsCenterY, _PAD,_PADY,_PAD,_PAD)); + para_group->AddFrame(arch.input, new TGTableLayoutHints(1, 5, 8, 9, kLHintsLeft |kLHintsCenterY|kLHintsExpandY|kLHintsExpandX, _PAD,_PAD,_PAD2,_PAD)); + + para_group->AddFrame(did.label, new TGTableLayoutHints(5, 6, 8, 9, kLHintsCenterX |kLHintsCenterY|kLHintsExpandY|kLHintsExpandX, _PAD,_PAD,_PAD2,_PAD)); + para_group->AddFrame(did.start, new TGTableLayoutHints(6, 7, 8, 9, kLHintsCenterX |kLHintsCenterY|kLHintsExpandY|kLHintsExpandX, _PAD,_PAD,_PAD2,_PAD)); + para_group->AddFrame(did.kill, new TGTableLayoutHints(7, 8, 8, 9, kLHintsCenterX |kLHintsCenterY|kLHintsExpandY|kLHintsExpandX, _PAD,_PAD,_PAD2,_PAD)); + para_group->AddFrame(apply, new TGTableLayoutHints(8, 9, 8, 9, kLHintsCenterX |kLHintsCenterY|kLHintsExpandY|kLHintsExpandX, _PAD,_PAD,_PAD2,_PAD)); + + para_group->AddFrame(logSrv.label, new TGTableLayoutHints(3, 4, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PADY, _PAD,_PAD)); + para_group->AddFrame(logSrv.utgid, new TGTableLayoutHints(4, 5, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(logSrv.start, new TGTableLayoutHints(5, 6, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(logSrv.kill, new TGTableLayoutHints(6, 7, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(logSrv.status, new TGTableLayoutHints(7, 9, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + + para_group->AddFrame(tmSrv.label, new TGTableLayoutHints(3, 4, 1, 2, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PADY, _PAD,_PAD)); + para_group->AddFrame(tmSrv.utgid, new TGTableLayoutHints(4, 5, 1, 2, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(tmSrv.start, new TGTableLayoutHints(5, 6, 1, 2, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(tmSrv.kill, new TGTableLayoutHints(6, 7, 1, 2, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(tmSrv.status, new TGTableLayoutHints(7, 9, 1, 2, kLHintsLeft|kLHintsCenterY|kLHintsExpandY|kLHintsExpandX, _PAD2,_PAD,_PAD,_PAD)); + + para_group->AddFrame(tanSrv.label, new TGTableLayoutHints(3, 4, 2, 3, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PADY, _PAD,_PAD)); + para_group->AddFrame(tanSrv.utgid, new TGTableLayoutHints(4, 5, 2, 3, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(tanSrv.start, new TGTableLayoutHints(5, 6, 2, 3, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(tanSrv.kill, new TGTableLayoutHints(6, 7, 2, 3, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + //para_group->AddFrame(tanSrv.status, new TGTableLayoutHints(7, 9, 2, 3, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + + para_group->AddFrame(logViewer.label, new TGTableLayoutHints(3, 4, 3, 4, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PADY, _PAD,_PAD)); + para_group->AddFrame(logViewer.utgid, new TGTableLayoutHints(4, 5, 3, 4, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(logViewer.start, new TGTableLayoutHints(5, 6, 3, 4, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(logViewer.kill, new TGTableLayoutHints(6, 7, 3, 4, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + //para_group->AddFrame(logViewer.status, new TGTableLayoutHints(7, 9, 3, 4, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + + para_group->AddFrame(mbmmon.label, new TGTableLayoutHints(3, 4, 4, 5, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PADY, _PAD,_PAD)); + para_group->AddFrame(mbmmon.utgid, new TGTableLayoutHints(4, 5, 4, 5, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(mbmmon.start, new TGTableLayoutHints(5, 6, 4, 5, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(mbmmon.kill, new TGTableLayoutHints(6, 7, 4, 5, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + //para_group->AddFrame(mbmmon.status, new TGTableLayoutHints(7, 9, 4, 5, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + + para_group->AddFrame(ctrl.label, new TGTableLayoutHints(3, 4, 5, 6, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PADY, _PAD,_PAD)); + para_group->AddFrame(ctrl.utgid, new TGTableLayoutHints(4, 5, 5, 6, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(ctrl.start, new TGTableLayoutHints(5, 6, 5, 6, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(ctrl.kill, new TGTableLayoutHints(6, 7, 5, 6, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); + para_group->AddFrame(ctrl.status, new TGTableLayoutHints(7, 9, 5, 6, kLHintsLeft|kLHintsCenterY|kLHintsExpandY, _PAD2,_PAD,_PAD,_PAD)); TGGroupFrame* cmds_group = new TGGroupFrame(this, "Commands"); cmds_group->SetLayoutManager(new TGTableLayout(cmds_group, 2, 9)); @@ -284,12 +293,12 @@ void NodeFSMPanel::init() { ctrl_autostop->Connect( "Clicked()", "testbeam::NodeFSMPanel", this, "autoStop()"); ctrl_status = new TGTextEntry(cmds_group); - cmds_group->AddFrame(ctrl_label, new TGTableLayoutHints(0, 1, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandX|kLHintsExpandY,10,1,3,1)); - cmds_group->AddFrame(ctrl_command, new TGTableLayoutHints(1, 2, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandX|kLHintsExpandY,10,1,3,1)); - cmds_group->AddFrame(ctrl_status, new TGTableLayoutHints(2, 3, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandX|kLHintsExpandY,10,1,3,1)); - cmds_group->AddFrame(ctrl_autostart,new TGTableLayoutHints(5, 6, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandX|kLHintsExpandY,10,1,3,1)); - cmds_group->AddFrame(ctrl_autostop, new TGTableLayoutHints(6, 7, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandX|kLHintsExpandY,10,1,3,1)); - cmds_group->AddFrame(ctrl_killall, new TGTableLayoutHints(7, 8, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandX|kLHintsExpandY,10,1,3,1)); + cmds_group->AddFrame(ctrl_label, new TGTableLayoutHints(0, 1, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandX|kLHintsExpandY,_PAD2,1,3,1)); + cmds_group->AddFrame(ctrl_command, new TGTableLayoutHints(1, 2, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandX|kLHintsExpandY,_PAD2,1,3,1)); + cmds_group->AddFrame(ctrl_status, new TGTableLayoutHints(2, 3, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandX|kLHintsExpandY,_PAD2,1,3,1)); + cmds_group->AddFrame(ctrl_autostart,new TGTableLayoutHints(5, 6, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandX|kLHintsExpandY,_PAD2,1,3,1)); + cmds_group->AddFrame(ctrl_autostop, new TGTableLayoutHints(6, 7, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandX|kLHintsExpandY,_PAD2,1,3,1)); + cmds_group->AddFrame(ctrl_killall, new TGTableLayoutHints(7, 8, 0, 1, kLHintsLeft|kLHintsCenterY|kLHintsExpandX|kLHintsExpandY,_PAD2,1,3,1)); m_cmds_group = cmds_group; TGGroupFrame* child_group = new TGGroupFrame(this, new TGString("Controller's Child States")); int num_rows = NUM_ENTRIES_CHILDREN; @@ -330,9 +339,17 @@ void NodeFSMPanel::init() { AddFrame(cmds_group, new TGLayoutHints(kLHintsRight|kLHintsTop|kLHintsExpandX, 1,1,1,1)); AddFrame(child_group, new TGLayoutHints(kLHintsRight|kLHintsTop|kLHintsExpandX|kLHintsExpandY, 1,1,1,1)); + maindns.input->Resize(250, maindns.input->GetDefaultHeight()); + dns.input->Resize(250, dns.input->GetDefaultHeight()); + node.input->Resize(250, node.input->GetDefaultHeight()); + part.input->Resize(250, part.input->GetDefaultHeight()); + script.input->Resize(350, script.input->GetDefaultHeight()); + arch.input->Resize(650, arch.input->GetDefaultHeight()); + numslave.input->Resize(150, numslave.input->GetDefaultHeight()); + tmSrv.kill ->Resize(120,tmSrv.kill->GetDefaultHeight()); tmSrv.start ->Resize(220,tmSrv.start->GetDefaultHeight()); - tmSrv.utgid ->Resize(320,tmSrv.utgid->GetDefaultHeight()); + tmSrv.utgid ->Resize(270,tmSrv.utgid->GetDefaultHeight()); tmSrv.utgid ->SetEnabled(kTRUE); tmSrv.status->Resize(120,tmSrv.status->GetDefaultHeight()); tmSrv.status->SetEnabled(kFALSE); @@ -341,7 +358,7 @@ void NodeFSMPanel::init() { logSrv.kill ->Resize(120,logSrv.kill->GetDefaultHeight()); logSrv.start ->Resize(220,logSrv.start->GetDefaultHeight()); - logSrv.utgid ->Resize(320,logSrv.utgid->GetDefaultHeight()); + logSrv.utgid ->Resize(270,logSrv.utgid->GetDefaultHeight()); logSrv.utgid ->SetEnabled(kTRUE); logSrv.status->Resize(120,logSrv.status->GetDefaultHeight()); logSrv.status->SetEnabled(kFALSE); @@ -350,12 +367,12 @@ void NodeFSMPanel::init() { logViewer.kill ->Resize(120,logViewer.kill->GetDefaultHeight()); logViewer.start ->Resize(220,logViewer.start->GetDefaultHeight()); - logViewer.utgid ->Resize(320,logViewer.utgid->GetDefaultHeight()); + logViewer.utgid ->Resize(270,logViewer.utgid->GetDefaultHeight()); logViewer.utgid ->SetEnabled(kTRUE); tanSrv.kill ->Resize(120,tanSrv.kill->GetDefaultHeight()); tanSrv.start ->Resize(220,tanSrv.start->GetDefaultHeight()); - tanSrv.utgid ->Resize(320,tanSrv.utgid->GetDefaultHeight()); + tanSrv.utgid ->Resize(270,tanSrv.utgid->GetDefaultHeight()); tanSrv.utgid ->SetEnabled(kTRUE); //tanSrv.status->Resize(120,tanSrv.status->GetDefaultHeight()); //tanSrv.status->SetEnabled(kFALSE); @@ -364,29 +381,22 @@ void NodeFSMPanel::init() { mbmmon.kill ->Resize(120,mbmmon.kill->GetDefaultHeight()); mbmmon.start ->Resize(220,mbmmon.start->GetDefaultHeight()); - mbmmon.utgid ->Resize(320,mbmmon.utgid->GetDefaultHeight()); + mbmmon.utgid ->Resize(270,mbmmon.utgid->GetDefaultHeight()); mbmmon.utgid ->SetEnabled(kTRUE); ctrl.kill ->Resize(120,ctrl.kill->GetDefaultHeight()); ctrl.start ->Resize(220,ctrl.start->GetDefaultHeight()); - ctrl.utgid ->Resize(320,ctrl.utgid->GetDefaultHeight()); + ctrl.utgid ->Resize(270,ctrl.utgid->GetDefaultHeight()); ctrl.utgid ->SetEnabled(kTRUE); ctrl.status->Resize(120,ctrl.status->GetDefaultHeight()); ctrl.status->SetEnabled(kFALSE); ctrl.status->SetBackgroundColor(red); ctrl.status->SetForegroundColor(white); - maindns.input->Resize(150, maindns.input->GetDefaultHeight()); - dns.input->Resize(150, dns.input->GetDefaultHeight()); - node.input->Resize(150, node.input->GetDefaultHeight()); - part.input->Resize(150, part.input->GetDefaultHeight()); - arch.input->Resize(650, arch.input->GetDefaultHeight()); - numslave.input->Resize(150, numslave.input->GetDefaultHeight()); - TGFont *font = gClient->GetFont("-*-arial-bold-r-*-*-24-*-*-*-*-*-iso8859-1"); ctrl_command->Resize(220,20); gClient->NeedRedraw(ctrl_command); - ctrl_status->Resize(250,45); + ctrl_status->Resize(300,45); ctrl_status->SetFont(font); ctrl_status->SetAlignment(kTextCenterX); gClient->NeedRedraw(ctrl_status); @@ -423,6 +433,7 @@ void NodeFSMPanel::enableParams() { dns.input->SetEnabled(kTRUE); node.input->SetEnabled(kTRUE); part.input->SetEnabled(kTRUE); + script.input->SetEnabled(kTRUE); arch.input->SetEnabled(kTRUE); ctrl.utgid->SetEnabled(kTRUE); ctrl.start->SetEnabled(kFALSE); @@ -482,6 +493,7 @@ void NodeFSMPanel::applyParams() { node.input->SetEnabled(kFALSE); part.input->SetEnabled(kFALSE); arch.input->SetEnabled(kFALSE); + script.input->SetEnabled(kFALSE); ctrl.utgid->SetEnabled(kFALSE); ctrl.start->SetEnabled(kTRUE); ctrl.kill->SetEnabled(kTRUE); @@ -521,6 +533,7 @@ void NodeFSMPanel::applyParams() { svc = m_ctrlName+CONTROLLER_STATUS_OPT; m_ctrlCom->svcID = ::dic_info_service(svc.c_str(), MONITORED,0,0,0,dim_feed,(long)m_ctrlCom, 0, 0); m_ctrlTasks->svcID = ::dic_info_service((m_ctrlName+"/tasks").c_str(), MONITORED,0,0,0,dim_feed,(long)m_ctrlTasks, 0, 0); + m_mainDNS_id = m_mainDNS.empty() ? -1 : ::dic_add_dns(m_mainDNS.c_str(), ::dim_get_dns_port()); } void NodeFSMPanel::updateDependentValues() { @@ -754,14 +767,24 @@ void NodeFSMPanel::autoStop() { void NodeFSMPanel::tmSrv_start() { if ( m_tmSrvCom->data == "DEAD" ) { stringstream cmd; - if ( m_mainDNS.empty() ) - cmd << "export DIM_DNS_NODE=" << m_node << ";export UTGID=" << m_tmSrvName << ";"; - else + if ( m_mainDNS.empty() ) { + cmd << "export DIM_DNS_NODE=" << m_node << ";export UTGID=" << m_tmSrvName << ";" + << TMSRV_COMMAND << " -l 2 -N " << m_node << " -p 2 -u online -U root --no-auth --no-ptrace-workaround&"; + system(cmd.str().c_str()); + } + else { cmd << TMSTART_COMMAND << " -N " << m_mainDNS << " -m " << m_node << " " << startDefaults() - << " -n root -g root -DDIM_DNS_NODE=" << m_node << " -u " << m_tmSrvName << " "; - cmd << TMSRV_COMMAND << " -l 2 -N " << m_node << " -p 2 -u online -U root --no-auth --no-ptrace-workaround"; - if ( m_mainDNS.empty() ) cmd << "&"; - system(cmd.str().c_str()); + << " -n root -g root -DDIM_DNS_NODE=" << m_node << " -u " << m_tmSrvName << " " + << TMSRV_COMMAND << " -l 2 -N " << m_node << " -p 2 -u online -U root --no-auth --no-ptrace-workaround"; + system(cmd.str().c_str()); +#if 0 + FiniteStateMachine::TaskManager(m_node, m_mainDNS_id) + .start(m_tmSrvName, + " -N " + m_mainDNS + " -m " + m_node + " " + startDefaults(), + "/opt/FMC/sbin/tmSrv", + "-l 2 -N " + m_node + " -p 2 -u online -U root --no-auth --no-ptrace-workaround"); +#endif + } return; } GuiException(kMBOk,kMBIconAsterisk,"Invalid Request", @@ -772,11 +795,15 @@ void NodeFSMPanel::tmSrv_start() { /// Kill tmSrv process void NodeFSMPanel::tmSrv_kill() { string cmd; - if ( m_mainDNS.empty() ) + if ( m_mainDNS.empty() ) { cmd = "pkill -9 tmSrv"; - else + system(cmd.c_str()); + } + else { cmd = string(TMKILL_COMMAND) + " -N "+m_mainDNS+" -m "+m_node+" " + killDefaults() + m_tmSrvName; - system(cmd.c_str()); + system(cmd.c_str()); + //FiniteStateMachine::TaskManager(m_node, m_mainDNS_id).kill(m_tmSrvName, 9); + } } /// Start logSrv process @@ -787,17 +814,30 @@ void NodeFSMPanel::logSrv_start() { cmd << "export DIM_DNS_NODE=" << m_node << ";export UTGID=" << m_logSrvName << ";"; cmd << "export LOGFIFO=" << m_logFifo << "; " << "export DIM_HOST_NODE=" << m_node << "; "; + cmd << "/group/online/dataflow/scripts/NodeLogger.sh&"; + system(cmd.str().c_str()); } else { cmd << TMSTART_COMMAND << " -N " << m_mainDNS << " -m " << m_node - << " -e -o -n online -DDIM_DNS_NODE=" << m_node << " -u " << m_logSrvName << " " - << "-DLOGFIFO=" << m_logFifo << " " - << "-DDIM_HOST_NODE=" << m_node << " " - << "-DPUBLISH_SERVICE=" << m_partition << "/LOGS "; + << " -e -o -n online -DDIM_DNS_NODE=" << m_node << " -u " << m_logSrvName << " " + << "-DLOGFIFO=" << m_logFifo << " " + << "-DDIM_HOST_NODE=" << m_node << " " + << "-DPUBLISH_SERVICE=" << m_partition << "/LOGS " + << "/group/online/dataflow/scripts/NodeLogger.sh"; + system(cmd.str().c_str()); + + /* + FiniteStateMachine::TaskManager(m_node, m_mainDNS_id) + .start(m_logSrvName, + " -N " + m_mainDNS + " -m " + m_node + " -e -o -n online " + + "-DDIM_DNS_NODE=" + m_node + " " + + "-DLOGFIFO=" + m_logFifo + " " + + "-DDIM_HOST_NODE=" + m_node + " " + + "-DPUBLISH_SERVICE=" + "/LOGS ", + "/group/online/dataflow/scripts/NodeLogger.sh", + ""); + */ } - cmd << "/group/online/dataflow/scripts/NodeLogger.sh"; - if ( m_mainDNS.empty() ) cmd << "&"; - system(cmd.str().c_str()); return; } GuiException(kMBOk,kMBIconAsterisk,"Invalid Request", @@ -821,16 +861,19 @@ void NodeFSMPanel::controller_start() { if ( state == "DEAD" ) { stringstream cmd; if ( m_startup.empty() ) { - cmd << startDefaults() << " -u " << m_ctrlName << " " - << " /group/online/dataflow/scripts/runTestBeam.sh -type=" << m_ctrlType + cmd << startDefaults() + << " -u " << m_ctrlName + << " " << m_ctrlScript + << " -type=" << m_ctrlType << " -partition=" << m_partition - << " -taskinfo=" << m_architecture - << " -count=" << m_numSlaves - << " -runinfo=" << m_runinfo - << " -logfifo=" << m_logFifo; - if ( m_DNS != m_mainDNS ) { - cmd << " -dimdns=" << m_DNS; - } + << " -taskinfo=" << m_architecture + << " -count=" << m_numSlaves + << " -runinfo=" << m_runinfo + << " -logfifo=" << m_logFifo + << " -smidns=" << m_DNS + << " -tmsdns=" << m_DNS + << " -dimdns=" << m_DNS + << " -replacements=NUMBER_OF_SLAVES:" << m_numSlaves; } else { cmd << startDefaults() << " -u " << m_ctrlName << " " << m_startup; @@ -911,10 +954,18 @@ void NodeFSMPanel::did_kill() { void NodeFSMPanel::logViewer_start() { stringstream cmd; cmd << "xterm -sl 20000 -ls -132 -geometry 230x50 -title \"" << m_logViewName << "\" " - "-e \"export UTGID=" << m_logViewName << "; /group/online/dataflow/scripts/log_viewer.sh -N -broker " - << m_node << " -S " << m_logSrvName << "\" &"; + "-e \"export UTGID=" << m_logViewName << "; " + << "/group/online/dataflow/scripts/log_viewer.sh -C -N" + << " -d " << m_node + << " -broker " << m_node + << " -m " << m_node + << "\"&"; + ::printf("%s\n",cmd.str().c_str()); system(cmd.str().c_str()); } +//dis_update_service gentest.exe libROLogSrv.so run_output_logger -C -N -d $HOST -b $HOST -m $HOST +//exec -a LogViewer_22344 gentest.exe libROLogSrv.so run_output_logger -C -N -b pluscc04 -O 1 -S (.*)/LOGS -S pluscc04 + /// Kill logViewer process void NodeFSMPanel::logViewer_kill() { @@ -936,6 +987,19 @@ void NodeFSMPanel::architectureChanged(const char* value) { m_architecture = value; } +void NodeFSMPanel::setScript(const std::string& value) { + m_ctrlScript = value; + if ( script.input ) { + script.input->SetText(m_ctrlScript.c_str()); + gClient->NeedRedraw(script.input); + } + updateDependentValues(); +} + +void NodeFSMPanel::scriptChanged(const char* value) { + m_ctrlScript = value; +} + void NodeFSMPanel::setPartition(const std::string& value) { m_partition = value; if ( part.input ) { diff --git a/Online/TestBeam/src/gui/NodeGUI.cpp b/Online/TestBeam/src/gui/NodeGUI.cpp index 5e0156d71..155f6dec4 100644 --- a/Online/TestBeam/src/gui/NodeGUI.cpp +++ b/Online/TestBeam/src/gui/NodeGUI.cpp @@ -42,7 +42,7 @@ NodeGUI::NodeGUI(TGFrame* parent, RTL::CLI& opts) : TGCompositeFrame(parent, 100, 100, kVerticalFrame), CPP::Interactor() { - string maindns, dns, node, partition, architecture, runinfo, user, startup; + string maindns, dns, node, partition, architecture, runinfo, user, startup, script; int instances=20, max_instances=20; tabs = new TGTab(this, 600, 900); @@ -57,6 +57,7 @@ NodeGUI::NodeGUI(TGFrame* parent, RTL::CLI& opts) opts.getopt("partition", 2, partition); opts.getopt("architecture",2, architecture); opts.getopt("startup", 4, startup); + opts.getopt("ctrl_script", 9, script); dns = RTL::str_upper(dns); node = RTL::str_upper(node); @@ -67,6 +68,7 @@ NodeGUI::NodeGUI(TGFrame* parent, RTL::CLI& opts) nodeControl->setDns(dns); nodeControl->setHost(node); nodeControl->setUser(user); + nodeControl->setScript(script); nodeControl->setStartup(startup); nodeControl->setRunInfo(runinfo); nodeControl->setPartition(partition); diff --git a/Online/TestBeam/src/gui/TaskManager.cpp b/Online/TestBeam/src/gui/TaskManager.cpp new file mode 100644 index 000000000..e5205e482 --- /dev/null +++ b/Online/TestBeam/src/gui/TaskManager.cpp @@ -0,0 +1,86 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Author : Markus Frank +//========================================================================== +// +// Finite state machine implementation to control +// and manipulate process groups +// +//========================================================================== + +// Framework include files +#include <TestBeam/gui/TaskManager.h> +#include <RTL/strdef.h> +#include <RTL/rtl.h> +#include <dim/dic.h> +#include <dim/dim.h> + +// C/C++ include files +#include <cstdarg> +#include <map> + +using namespace FiniteStateMachine; +using namespace std; + +namespace { + bool s_debug = true; + int _execDim(long domain, const string& datapoint, const char* fmt, ...) { + va_list args; + va_start( args, fmt); + char str[4096]; + size_t len = ::vsnprintf(str,sizeof(str),fmt,args); + va_end (args); + str[len]=0; + if ( s_debug ) { + ::printf("Domain:%ld Service:%s Cmd:%s\n",domain, datapoint.c_str(),str); + ::fflush(stdout); + } + int ret = ::dic_cmnd_service_dns(domain, (char*)datapoint.c_str(), str, len+1); + if ( ret == 1 ) { + return 1; + } + return 0; + } +} + +/// Standard constructor +TaskManager::TaskManager(const string& node, long domain) { + string n = RTL::str_upper(node); + m_node = n; + m_domain = domain; + m_start = "/FMC/"+n+"/task_manager/start"; + m_stop = "/FMC/"+n+"/task_manager/stop"; + m_kill = "/FMC/"+n+"/task_manager/kill"; +} + +/// Instance accessor +TaskManager& TaskManager::instance(const string& node, long domain) { + static map<string,TaskManager*> m; + map<string,TaskManager*>::iterator i=m.find(node); + if ( i == m.end() ) { + i = m.insert(make_pair(node,new TaskManager(node, domain))).first; + } + return *((*i).second); +} + +/// Start a process +int TaskManager::start(const std::string& utgid, const string& fmc_args, const string& cmd, const string& args) const { + return _execDim(m_domain,m_start,"-u %s %s %s %s",utgid.c_str(),fmc_args.c_str(),cmd.c_str(),args.c_str()); +} +/// Kill a process +int TaskManager::stop(const std::string& utgid, int sig_num, int wait_before_kill) const { + return _execDim(m_domain,m_stop,"-s %d -d %d %s",sig_num,wait_before_kill,utgid.c_str()); +} + +/// Kill a process +int TaskManager::kill(const std::string& utgid, int sig_num) const { + return _execDim(m_domain,m_stop,"-s %d %s",sig_num,utgid.c_str()); +} diff --git a/Online/TestBeam/src/gui/gui_main.cpp b/Online/TestBeam/src/gui/gui_main.cpp index 17147410c..1886debf9 100644 --- a/Online/TestBeam/src/gui/gui_main.cpp +++ b/Online/TestBeam/src/gui/gui_main.cpp @@ -94,7 +94,9 @@ namespace { " -startup=<args> Startup command (if not default controller)\n" " -user<user-name> User name used to start applications \n" " -instances=<number> Number of parallel worker processes \n" - " -controller=<string> Controller type \n\n" + " -controller=<string> Controller type \n" + " -ctrl_script=<script> Controller startup script. \n" + " \n" " Arguments given:" << endl; for(int i=0; i<argc; ++i) cout << " " << argv[i] << endl; @@ -110,6 +112,8 @@ extern "C" void testbeam_node_gui(int argc, char** argv) { string user = "online"; string controller = CONTROLLER_TYPE; string startup; + string script = "/group/online/dataflow/scripts/runTestBeam.sh"; + string node = RTL::str_upper(RTL::nodeNameShort()); string dns = RTL::str_upper(RTL::nodeNameShort()); string maindns = dns_node ? string(dns_node) : dns; @@ -117,18 +121,20 @@ extern "C" void testbeam_node_gui(int argc, char** argv) { runinfo = gSystem->ExpandPathName(runinfo.c_str()); architecture = gSystem->ExpandPathName(architecture.c_str()); + RTL::CLI cli(argc, argv, help_gui); - cli.getopt("dns",3,dns); - cli.getopt("maindns",4,maindns); - cli.getopt("startup",4,startup); - cli.getopt("node",2,node); - cli.getopt("user",2,user); - cli.getopt("instances",2,instances); - cli.getopt("maxinstances",4,max_instances); - cli.getopt("partition",2,partition); - cli.getopt("runinfo",2,runinfo); - cli.getopt("architecture",2,architecture); - cli.getopt("controller", 2, controller); + cli.getopt("dns", 3, dns); + cli.getopt("maindns", 4, maindns); + cli.getopt("startup", 4, startup); + cli.getopt("node", 2, node); + cli.getopt("user", 2, user); + cli.getopt("instances", 2, instances); + cli.getopt("maxinstances", 4, max_instances); + cli.getopt("partition", 2, partition); + cli.getopt("runinfo", 2, runinfo); + cli.getopt("architecture", 2, architecture); + cli.getopt("controller", 2, controller); + cli.getopt("ctrl_script", 9, script); cout << "Calling XInitThreads to support concurrency...." << endl; XInitThreads(); diff --git a/Online/sqldb/CMakeLists.txt b/Online/sqldb/CMakeLists.txt new file mode 100644 index 000000000..b71ca29c3 --- /dev/null +++ b/Online/sqldb/CMakeLists.txt @@ -0,0 +1,53 @@ +#=============================================================================== +# LHCb Online software suite +#------------------------------------------------------------------------------- +# Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +# All rights reserved. +# +# For the licensing terms see OnlineSys/LICENSE. +# +# Author : M.Frank +# +#=============================================================================== +################################################################################ +# Package: sqldb +# +gaudi_subdir(sqldb v0r1) +################################################################################ +# +find_package(Boost REQUIRED) +find_package(SQLite3 REQUIRED) +find_package(MySQL) +find_package(Oracle REQUIRED) +# +include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) +# +gaudi_add_library(sqldb_sqlite src/sqlite.cpp + PUBLIC_HEADERS sqldb + INCLUDE_DIRS Boost SQLite3 + LINK_LIBRARIES SQLite3) +# +if ( MySQL_FOUND ) + gaudi_add_library(sqldb_mysql src/mysql.cpp + NO_PUBLIC_HEADERS + INCLUDE_DIRS Boost MySQL + LINK_LIBRARIES MySQL) +endif() +if ( ORACLE_FOUND ) + gaudi_add_library(sqldb_oracle src/oracle.cpp + NO_PUBLIC_HEADERS + INCLUDE_DIRS Boost Oracle + LINK_LIBRARIES Oracle) +endif() +# --------------------------------------------------------------------------------------- +# Testing +# --------------------------------------------------------------------------------------- +gaudi_add_library(sqldb_tests test/*.cpp + NO_PUBLIC_HEADERS + INCLUDE_DIRS Boost SQLite3 + LINK_LIBRARIES sqldb_sqlite SQLite3) +# +if ( MySQL_FOUND ) + target_compile_definitions(sqldb_tests PRIVATE -DHAVE_MYSQL=1) + target_link_libraries(sqldb_tests sqldb_mysql) +endif() diff --git a/Online/sqldb/sqldb/mysql.h b/Online/sqldb/sqldb/mysql.h new file mode 100644 index 000000000..0abc46b4d --- /dev/null +++ b/Online/sqldb/sqldb/mysql.h @@ -0,0 +1,34 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Package : RPC +// +// Author : Markus Frank +//========================================================================== +#ifndef ONLINEBASE_CPP_MYSQL_INTERFACE_H +#define ONLINEBASE_CPP_MYSQL_INTERFACE_H + +/// Framework include files +#include <sqldb/sqldb.h> + +/// Technology abstraction layer +/** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ +class mysql : public sqldb::traits { + public: + struct dbase_t; + struct statement_t; +}; + +#endif // ONLINEBASE_CPP_MYSQL_INTERFACE_H diff --git a/Online/sqldb/sqldb/oracle.h b/Online/sqldb/sqldb/oracle.h new file mode 100644 index 000000000..55c833ef9 --- /dev/null +++ b/Online/sqldb/sqldb/oracle.h @@ -0,0 +1,34 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Package : RPC +// +// Author : Markus Frank +//========================================================================== +#ifndef SQLDB_ORACLE_INTERFACE_H +#define SQLDB_ORACLE_INTERFACE_H + +/// Framework include files +#include <sqldb/sqldb.h> + +/// Technology abstraction layer +/** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ +class oracle : public sqldb::traits { + public: + typedef struct oracle_db dbase_t; + typedef struct oracle_stmt statement_t; +}; + +#endif // SQLDB_ORACLE_INTERFACE_H diff --git a/Online/sqldb/sqldb/sqldb-imp.h b/Online/sqldb/sqldb/sqldb-imp.h new file mode 100644 index 000000000..f643c3e1d --- /dev/null +++ b/Online/sqldb/sqldb/sqldb-imp.h @@ -0,0 +1,529 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Package : RPC +// +// Author : Markus Frank +//========================================================================== +#ifndef SQLDB_SQLDB_IMP_INTERFACE_H +#define SQLDB_SQLDB_IMP_INTERFACE_H + +/// C/C++ include files +#include <sstream> + +/// Framework include files + +/// SQLDB namespace declaration +namespace sqldb { + + /// Call on failure: Throws exception + SQLDB_INLINE int invalid_statement(const char* msg) { + throw std::runtime_error(std::string("Statement Error: ") + (msg ? msg : " [Unknwon Error]")); + } + + /// Call on failure: Throws exception + SQLDB_INLINE int invalid_statement(const std::string& msg) { + throw std::runtime_error("Statement Error: " + msg); + } + + /// Convert ascii string to time stamp + SQLDB_INLINE time time::make_time(const struct tm& tm) { + return time(::mktime((struct tm*)&tm)); + } + + /// Convert ascii string to time stamp + SQLDB_INLINE time time::make_time(const std::string& asc_time) { + return make_time(asc_time, time::default_format()); + } + + /// Convert ascii string to time stamp + SQLDB_INLINE time time::make_time(const std::string& asc_time, const std::string& format) { + struct tm tm; + if ( nullptr != ::strptime(asc_time.c_str(), format.c_str(), &tm) ) { + return make_time(tm); + } + throw std::runtime_error("Invalid data conversion to time with format: "+format); + } + + /// Conversion to local time in form of struct tm + SQLDB_INLINE struct tm time::local_time() const { + struct tm tm; + ::localtime_r(&this->stamp, &tm); + return tm; + } + + /// Conversion to greenwich mean time in form of struct tm + SQLDB_INLINE struct tm time::gm_time() const { + struct tm tm; + ::gmtime_r(&this->stamp, &tm); + return tm; + } + + /// Conversion to string + SQLDB_INLINE std::string time::to_string() const { + char buff[64]; + return ::ctime_r(&this->stamp, buff); + } + + /// Conversion to string + SQLDB_INLINE std::string time::to_string(const std::string& format) const { + struct tm tm; + char buff[256]; + ::localtime_r(&this->stamp, &tm); + size_t len = ::strftime(buff, sizeof(buff), format.c_str(), &tm); + if ( len > 0 ) return buff; + throw std::runtime_error("Invalid data conversion to time with format: "+format); + } + + /// Generic conversion to integer number + template <typename T> SQLDB_INLINE T to_integer(const column_data& column) { + switch( column.type ) { + case TYPE_BLOB: + case TYPE_STRING: + return (T)::strtol(std::string(column.item.string, column.length).c_str(),nullptr,10); + case TYPE_TINY: + case TYPE_SHORT: + case TYPE_LONG: + case TYPE_LONGLONG: + return (T)(column.item.uint64); + case TYPE_FLOAT: + return (T)(column.item.real32); + case TYPE_DOUBLE: + return (T)(column.item.real64); + case TYPE_TIME: + return column.item.time.stamp; + default: + throw std::runtime_error(std::string("Invalid data type conversion to ")+typeid(T).name()); + break; + } + } + + /// Generic conversion to floating point number + template <typename T> SQLDB_INLINE T to_real(const column_data& column) { + switch( column.type ) { + case TYPE_BLOB: + case TYPE_STRING: + return (T)::strtod((std::string(column.item.string, column.length)).c_str(), nullptr); + case TYPE_TINY: + case TYPE_SHORT: + case TYPE_LONG: + case TYPE_LONGLONG: + return (T)column.item.int64; + case TYPE_FLOAT: + return column.item.real32; + case TYPE_DOUBLE: + return column.item.real64; + case TYPE_TIME: + return (T)column.item.time.stamp; + default: + throw std::runtime_error(std::string("Invalid data type conversion to ")+typeid(T).name()); + break; + } + } + + /// Generic conversion to integer number + SQLDB_INLINE struct time to_time(const column_data& column) { + switch( column.type ) { + case TYPE_BLOB: + case TYPE_STRING: + return time::make_time(std::string(column.item.string, column.length)); + case TYPE_LONG: + case TYPE_LONGLONG: + return time::make_time(column.item.uint64); + case TYPE_TINY: + case TYPE_SHORT: + case TYPE_FLOAT: + case TYPE_DOUBLE: + default: + throw std::runtime_error("Invalid data type conversion to sqldb-time"); + break; + } + } + + /// Bind pointers for writing + SQLDB_INLINE void column_data::bind_pointer(const unsigned char* ptr, size_t len) { + if ( len < sizeof(column_data::buf) ) { + ::memcpy(this->buf, ptr, len); + this->item.pointer = this->buf; + } + else { + this->vbuf.resize(len); + this->vbuf.assign(ptr, ptr+len); + this->item.pointer = &this->vbuf.front(); + } + this->length = len; + } + + /// Access the name of a database + SQLDB_INLINE std::string_view database::name() const { + return intern.get() ? intern->name.c_str() : ""; + } + + /// Access the name of a database + SQLDB_INLINE std::string_view database::name(const database& dbase) { + return dbase.name(); + } + + /// Access error number of last failed statement + SQLDB_INLINE int database::error() const { + return intern->action(backend::ERRNO); + } + + /// Access error message on failure + SQLDB_INLINE std::string database::errmsg() const { + return intern.get() ? intern->errmsg() : " --- no database handle --- "; + } + + /// Access error message on failure + SQLDB_INLINE std::string database::errmsg(const database& dbase) { + return dbase.errmsg(); + } + + /// Close the connection to the database + SQLDB_INLINE int database::close() { + if ( intern.get() ) { + intern->action(backend::CLOSE); + intern->db = nullptr; + intern.reset(); + } + return sqldb::OK; + } + + /// Start transaction + SQLDB_INLINE int database::begin() const { + return intern->action(backend::BEGIN); + } + + /// Commit transaction + SQLDB_INLINE int database::commit() const { + return intern->action(backend::COMMIT); + } + + /// Roll-back transaction + SQLDB_INLINE int database::rollback() const { + return intern->action(backend::ROLLBACK); + } + + /// Access the number of rows affected by the last changes + SQLDB_INLINE long database::rows_affected() const { + return intern->rows_affected(); + } + + /// Execute SQL statement + SQLDB_INLINE int database::execute(std::string& error, const char* sql) const { + return intern->execute(error, sql); + } + + /// Execute SQL statement + SQLDB_INLINE int database::execute_sql(std::string& error, const char* fmt, ...) const { + char str[4096]; + va_list args; + va_start(args, fmt); + ::vsnprintf(str, sizeof(str), fmt, args); + va_end(args); + return this->execute_sql(error, str); + } + + /// Prepare a new statement and allocate resources. + SQLDB_INLINE int database::prepare(statement& stmt, const std::string& sql) const { + return intern->prepare(stmt, sql); + } + + /// Fetch next result row. Stop if result is <> traits::OK + SQLDB_INLINE int record_set::fetch_one() const { + if ( this->stmt ) return this->stmt->fetch_one(); + return invalid_statement("fetch_one: Invalid recordset statement handle"); + } + + template <> SQLDB_INLINE blob_t record::get<blob_t>(size_t col) const { + auto& column = this->field(col); + switch( column.type ) { + case TYPE_BLOB: + case TYPE_STRING: + return blob_t(column.item.blob, column.length); + default: + throw std::runtime_error("get<blob_t>: Invalid data type conversion to BLOB"); + break; + } + } + + template <> SQLDB_INLINE vblob_t record::get<vblob_t>(size_t col) const { + const auto& column = this->field(col); + switch( column.type ) { + case TYPE_BLOB: + case TYPE_STRING: + return vblob_t(column.item.blob, column.item.blob+column.length); + default: + throw std::runtime_error("get<vblob_t>: Invalid data type conversion to BLOB"); + break; + } + } + + template <> SQLDB_INLINE std::string record::get<std::string>(size_t col) const { + const auto& column = this->field(col); + switch( column.type ) { + case TYPE_BLOB: + case TYPE_STRING: + return column.item.string ? column.item.string : ""; + case TYPE_TINY: + case TYPE_SHORT: + case TYPE_LONG: + case TYPE_LONGLONG: { + std::stringstream buf; + buf << column.item.uint64 << std::ends; + return buf.str(); + } + case TYPE_FLOAT: { + std::stringstream buf; + buf << column.item.real32 << std::ends; + return buf.str(); + } + case TYPE_DOUBLE: { + std::stringstream buf; + buf << column.item.real64 << std::ends; + return buf.str(); + } + case TYPE_TIME: + return column.item.time.to_string(); + default: + throw std::runtime_error("Invalid data type conversion to STRING."); + break; + } + } + + template <> SQLDB_INLINE signed char record::get<signed char>(size_t col) const { + return to_integer<signed char>(this->field(col)); + } + + template <> SQLDB_INLINE char record::get<char>(size_t col) const { + return to_integer<char>(this->field(col)); + } + + template <> SQLDB_INLINE unsigned char record::get<unsigned char>(size_t col) const { + return to_integer<unsigned char>(this->field(col)); + } + + template <> SQLDB_INLINE int16_t record::get<int16_t>(size_t col) const { + return to_integer<int16_t>(this->field(col)); + } + + template <> SQLDB_INLINE uint16_t record::get<uint16_t>(size_t col) const { + return to_integer<uint16_t>(this->field(col)); + } + + template <> SQLDB_INLINE int32_t record::get<int32_t>(size_t col) const { + return to_integer<int32_t>(this->field(col)); + } + + template <> SQLDB_INLINE uint32_t record::get<uint32_t>(size_t col) const { + return to_integer<uint32_t>(this->field(col)); + } + + template <> SQLDB_INLINE int64_t record::get<int64_t>(size_t col) const { + return to_integer<int64_t>(this->field(col)); + } + + template <> SQLDB_INLINE uint64_t record::get<uint64_t>(size_t col) const { + return to_integer<uint64_t>(this->field(col)); + } + + template <> SQLDB_INLINE float record::get<float>(size_t col) const { + return to_real<float>(this->field(col)); + } + + template <> SQLDB_INLINE double record::get<double>(size_t col) const { + return to_real<double>(this->field(col)); + } + + template <> SQLDB_INLINE time record::get<struct time>(size_t col) const { + return to_time(this->field(col)); + } + + /// Generic bind of a single parameter column + SQLDB_INLINE int record_bind::bind (size_t column, field_type typ, const void* data, size_t len) const { + return stmt->_bind(column, typ, data, len); + } + + /// Type specific bind operator for blobs + SQLDB_INLINE int record_bind::bind(size_t col, const blob_t& data) const { + return stmt->_bind(col, TYPE_BLOB, data.first, data.second); + } + + /// Type specific bind operator for blobs + SQLDB_INLINE int record_bind::bind(size_t col, const vblob_t& data) const { + return stmt->_bind(col, TYPE_BLOB, &data.at(0), data.size()); + } + + /// Type specific bind operator for blobs + SQLDB_INLINE int record_bind::bind(size_t col, const uint8_t* data, size_t len) const { + return stmt->_bind(col, TYPE_BLOB, data, len); + } + + /// Type specific bind operator for strings + SQLDB_INLINE int record_bind::bind(size_t col, std::string& data) const { + return stmt->_bind(col, TYPE_STRING, data.c_str(), data.length()); + } + + /// Type specific bind operator for strings + SQLDB_INLINE int record_bind::bind(size_t col, const std::string& data) const { + return stmt->_bind(col, TYPE_STRING, data.c_str(), data.length()); + } + + /// Type specific bind operator for strings + SQLDB_INLINE int record_bind::bind(size_t col, char* const data) const { + return stmt->_bind(col, TYPE_STRING, data ? data : "", data ? ::strlen(data) : 0); + } + + /// Type specific bind operator for strings + SQLDB_INLINE int record_bind::bind(size_t col, const char* const data) const { + return stmt->_bind(col, TYPE_STRING, data ? data : "", data ? ::strlen(data) : 0); + } + + /// Type specific bind operator for integer type + template <> SQLDB_INLINE int record_bind::bind<char>(size_t col, char data) const { + return stmt->_bind(col, TYPE_TINY, &data, sizeof(data)); + } + + /// Type specific bind operator for integer type + template <> SQLDB_INLINE int record_bind::bind<signed char>(size_t col, signed char data) const { + return stmt->_bind(col, TYPE_TINY, &data, sizeof(data)); + } + + /// Type specific bind operator for integer type + template <> SQLDB_INLINE int record_bind::bind<unsigned char>(size_t col, unsigned char data) const { + return stmt->_bind(col, TYPE_TINY, &data, sizeof(data)); + } + + /// Type specific bind operator for integer type + template <> SQLDB_INLINE int record_bind::bind<int16_t>(size_t col, int16_t data) const { + return stmt->_bind(col, TYPE_SHORT, &data, sizeof(data)); + } + + /// Type specific bind operator for integer type + template <> SQLDB_INLINE int record_bind::bind<uint16_t>(size_t col, uint16_t data) const { + return stmt->_bind(col, TYPE_SHORT, &data, sizeof(data)); + } + + /// Type specific bind operator for integer type + template <> SQLDB_INLINE int record_bind::bind<int32_t>(size_t col, int32_t data) const { + return stmt->_bind(col, TYPE_LONG, &data, sizeof(data)); + } + + /// Type specific bind operator for integer type + template <> SQLDB_INLINE int record_bind::bind<uint32_t>(size_t col, uint32_t data) const { + return stmt->_bind(col, TYPE_LONG, &data, sizeof(data)); + } + + /// Type specific bind operator for integer type + template <> SQLDB_INLINE int record_bind::bind<int64_t>(size_t col, int64_t data) const { + return stmt->_bind(col, TYPE_LONGLONG, &data, sizeof(data)); + } + + /// Type specific bind operator for integer type + template <> SQLDB_INLINE int record_bind::bind<uint64_t>(size_t col, uint64_t data) const { + return stmt->_bind(col, TYPE_LONGLONG, &data, sizeof(data)); + } + + /// Type specific bind operator for + template <> SQLDB_INLINE int record_bind::bind<float>(size_t col, float data) const { + return stmt->_bind(col, TYPE_FLOAT, &data, sizeof(data)); + } + + template <> SQLDB_INLINE int record_bind::bind<double>(size_t col, double data) const { + return stmt->_bind(col, TYPE_DOUBLE, &data, sizeof(data)); + } + + template <> SQLDB_INLINE int record_bind::bind<struct time>(size_t col, struct time data) const { + return stmt->_bind(col, TYPE_TIME, &data.stamp, sizeof(data.stamp)); + } + + /// Copy assignment + SQLDB_INLINE statement& statement::operator = (statement&& copy) { + if ( © != this ) { + this->stmt = this; + this->intern = move(copy.intern); + } + return *this; + } + + /// Copy assignment + SQLDB_INLINE statement& statement::operator = (const statement& copy) { + if ( © != this ) { + this->stmt = this; + this->intern = copy.intern; + } + return *this; + } + + /// Access error number of last failed statement + SQLDB_INLINE int statement::error() const { + return intern->action(backend::ERRNO); + } + + /// Create fully prepared statement in a database context + SQLDB_INLINE int statement::prepare(const database& db, const std::string& sql) { + return db.prepare(*this, sql); + } + + /// Execute SQL statement with variable number of arguments + SQLDB_INLINE int statement::prepare(const database& db, const char* fmt, ...) { + char sql[4096]; + va_list args; + va_start(args, fmt); + ::vsnprintf(sql, sizeof(sql), fmt, args); + va_end(args); + return db.prepare(*this, sql); + } + + /// Execute SQL statement with variable number of arguments + SQLDB_INLINE statement statement::create(const database& db, const char* fmt, ...) { + char sql[4096]; + va_list args; + va_start(args, fmt); + ::vsnprintf(sql, sizeof(sql), fmt, args); + va_end(args); + statement stmt; + int ret = db.prepare(stmt, sql); + if ( ret == sqldb::OK ) + return stmt; + return statement(); + } + + /// Execute prepared statement + SQLDB_INLINE int statement::execute() const { + return intern->action(backend::EXEC); + } + + /// Fetch next result row. Stop if result is <> traits::OK + SQLDB_INLINE int statement::fetch_one() const { + return intern->action(backend::FETCH); + } + + /// Reset SQL statement: Reset all data to re-execute the statement with new parameters + SQLDB_INLINE int statement::reset() const { + return intern->action(backend::RESET); + } + + /// Finalize and run-down the statement. Release all resources. + SQLDB_INLINE int statement::finalize() { + if ( intern.get() ) { + intern->action(backend::FINALIZE); + intern.reset(); + } + return sqldb::OK; + } + + /// Generic bind of a single parameter column + SQLDB_INLINE int statement::_bind(size_t column, field_type typ, const void* data, size_t len) const { + return intern->bind(column, typ, data, len); + } + +} /// End namespace sqldb +#endif // SQLDB_SQLDB_IMP_INTERFACE_H diff --git a/Online/sqldb/sqldb/sqldb.h b/Online/sqldb/sqldb/sqldb.h new file mode 100644 index 000000000..f291f0c52 --- /dev/null +++ b/Online/sqldb/sqldb/sqldb.h @@ -0,0 +1,697 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Package : RPC +// +// Author : Markus Frank +//========================================================================== +#ifndef SQLDB_SQLDB_INTERFACE_H +#define SQLDB_SQLDB_INTERFACE_H + +// C/C++ files +#include <map> +#include <ctime> +#include <string> +#include <vector> +#include <memory> +#include <cstring> +#include <cstdint> +#include <cstdarg> + +/// SQLDB namespace declarations +namespace sqldb { + + /// Forward declarations + + + /// Technology abstraction layer + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + class traits { + public: + }; + + static constexpr int OK = 0; + static constexpr int ERROR = EINVAL; + static constexpr int DONE = 10000; + static constexpr int NO_DATA = 10002; + static constexpr int INVALID_HANDLE = 10004; + + enum field_type { + TYPE_NULL = 14, + TYPE_TINY = 15, + TYPE_SHORT = 16, + TYPE_LONG = 17, + TYPE_LONGLONG = 18, + TYPE_FLOAT = 19, + TYPE_DOUBLE = 20, + TYPE_STRING = 21, + TYPE_BLOB = 22, + TYPE_TIME = 23 + }; + + typedef std::pair<const uint8_t*, size_t> blob_t; + typedef std::vector<unsigned char> vblob_t; + + class statement; + class database; + class time; + + /// Call on failure: Throws exception + int invalid_statement(const char* msg = "[Unknown Error]"); + int invalid_statement(const std::string& msg); + + /// Connection argument parser + /** + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + class connection_args { + public: + /// Argument type definition + typedef std::map<std::string, std::string> arguments_t; + + public: + /// Call to parse the connection string + arguments_t parse(const std::string& /* connect_string */) const { + arguments_t result; + return result; + } + }; + + /// Pointer cast + /** + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + union pointers_t { + void* pointer; + const void* pvoid; + const char* pstring; + const std::int8_t* pint8; + const std::uint8_t* puint8; + const std::int16_t* pint16; + const std::uint16_t* puint16; + const std::int32_t* pint32; + const std::uint32_t* puint32; + const std::int64_t* pint64; + const std::uint64_t* puint64; + const float* preal32; + const double* preal64; + const class time* ptime; + pointers_t(const void* p) { pvoid = p; } + }; + + /// Safe-access handle + /** + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + template <typename T> class handle { + public: + T* pointer { nullptr }; + + T* invalid_handle() const { + throw std::runtime_error("Attempt to access an invalid handle [NULL pointer]"); + } + T* safe_pointer() { + if ( pointer ) return pointer; + return invalid_handle(); + } + const T* safe_pointer() const { + if ( pointer ) return pointer; + return invalid_handle(); + } + public: + handle(T* p) : pointer(p) {} + T* operator-> () const { return pointer; } + T& operator* () const { return *pointer; } + operator T* () const { return pointer; } + }; + + /// Time stamp structure + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + class time { + public: + /// Actual time stamp + time_t stamp; + + public: + /// Default constructor + time() = default; + /// Initializing constructor + time(time_t t) : stamp(t) {} + /// Move constructor + time(time&& copy) = default; + /// Copy constructor + time(const time& copy) = default; + /// Move assignment + time& operator=(time&& copy) = default; + /// Copy assigment + time& operator=(const time& copy) = default; + /// Equality check + bool operator==(const time& copy) const { return stamp == copy.stamp; } + /// Operator to convert to integer time representation + operator time_t () const { return this->stamp; } + /// Conversion to local time in form of struct tm + struct tm local_time() const; + /// Conversion to greenwich mean time in form of struct tm + struct tm gm_time() const; + /// Conversion to string + std::string to_string() const; + /// Conversion to string + std::string to_string(const std::string& format) const; + + /// Default time format + static std::string default_format() { return "%Y.%m.%d %H:%M:%S"; } + /// Current time in seconds + static time_t now() { return ::time(nullptr); } + /// Convert integer time representation to time stamp + static time make_time(time_t int_time) { return time(int_time); } + /// Convert ascii string to time stamp + static time make_time(const std::string& asc_time); + /// Convert ascii string to time stamp + static time make_time(const std::string& asc_time, const std::string& format); + /// Convert struct tm time to time stamp + static time make_time(const struct tm& tm_time); + }; + + /// Data binder shadow + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + struct column_data { + char buf[128] {0}; + std::vector<unsigned char> vbuf; + std::size_t length { 0 }; + union _item { + std::int8_t int8; + std::uint8_t uint8; + std::int16_t int16; + std::uint16_t uint16; + std::int32_t int32; + std::uint32_t uint32; + std::int64_t int64; + std::uint64_t uint64; + float real32; + double real64; + unsigned char* blob; + char* string; + const void* pointer; + class time time; + } item; + int type { TYPE_NULL }; + bool null { false }; + bool error { false }; + + /// Constructor + column_data() { + this->length = 0; + this->item.int64 = 0; + this->type = TYPE_NULL; + } + + /// Copy Constructor + column_data(const column_data& copy) { + this->type = copy.type; + this->length = copy.length; + this->item.int64 = copy.item.int64; + } + + /// Assignment operatpr + column_data& operator=(const column_data& copy) { + if ( this != © ) { + this->type = copy.type; + this->length = copy.length; + this->item.int64 = copy.item.int64; + } + return *this; + } + + /// Check for null data + bool is_null() const { + return length == 0 || type == TYPE_NULL; + } + + /// Bind pointers for writing + void bind_pointer(const unsigned char* ptr, size_t len); + }; + + /// Database handle class + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + class database final { + public: + typedef std::vector<column_data> columns_t; + + public: + + /// Database implementation class + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + class backend { + public: + backend* db {nullptr}; + columns_t params; + columns_t fields; + std::string name; + + enum action_type { BEGIN, COMMIT, ROLLBACK, CLOSE, ERRNO }; + + public: + /// Initializing constructor + backend() = default; + /// Default destructor + virtual ~backend() = default; + /// Access error message on failure + virtual std::string errmsg() = 0; + /// Access the number of rows affected by the last statement + virtual long rows_affected() = 0; + /// Perform multiple actions + virtual int action(action_type type) = 0; + /// Execute prepared statement + virtual int execute(std::string& err, const std::string& sql) = 0; + /// Prepare a new statement and allocate resources. + virtual int prepare(statement& stmt, const std::string& sql) = 0; + /// Open the database connection using all information in passed string + template <typename BACKEND> + static std::pair<std::shared_ptr<backend>, int> open(const std::string& name); + }; + std::shared_ptr<backend> intern; + + public: + /// Initializing constructor from statement handle + database(const statement& stmt); + /// Default constructor + database() = default; + /// Move constructor + database(database&& copy) = default; + /// Copy constructor + database(const database& copy) = default; + /// Destructor + ~database() = default; + + /// Move assignment + database& operator = (database&& copy) = default; + /// Copy assignment + database& operator = (const database& copy) = default; + + /// Access the name of a database + std::string_view name() const; + /// Access the name of a database + static std::string_view name(const database& db); + /// Access error message on failure + std::string errmsg() const; + /// Access error message on failure + static std::string errmsg(const database& db); + /// Check handle validity + bool is_valid() const; + /// Access error number of last failed statement + int error() const; + + /// Open the database connection using all information in passed string + template <typename BACKEND> int open(const std::string& con) { + auto result = backend::open<BACKEND>(con); + this->intern = result.first; + if ( this->intern.get() ) { + this->intern->db = this->intern.get(); + } + return result.second; + } + /// Open the database connection using all information in passed string + template <typename BACKEND> + static database open(const std::string& con, std::string& err) { + database db; + err.clear(); + if ( sqldb::OK != db.open<BACKEND>(con) ) { + err = db.errmsg(); + } + return db; + } + /// Close the connection to the database + int close(); + /// Start transaction + int begin() const; + /// Commit transaction + int commit() const; + /// Roll-back transaction + int rollback() const; + /// Access the number of rows affected by the last changes + long rows_affected() const; + /// Execute SQL statement + int execute(std::string& error, const char* sql) const; + /// Execute SQL statement with variable number of arguments + int execute_sql(std::string& error, const char* fmt, ...) const; + /// Prepare a new statement and allocate resources. + int prepare(statement& stmt, const std::string& sql) const; + }; + + /// Check handle validity + inline bool database::is_valid() const { + return intern.get() && intern->db != nullptr; + } + + /// Record class representing a selected row + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + class record { + public: + /// Reference to statement handle + handle<const statement> stmt { nullptr }; + + protected: + /// Default constructor + record() = default; + + public: + /// Initializing constructor + record(const statement* s) : stmt(s) {} + /// Move constructor + record(record&& copy) = delete; + /// Copy constructor + record(const record& copy) = default; + /// Destructor + ~record() = default; + + /// Move assignment + record& operator = (record&& copy) = delete; + /// Copy assignment + record& operator = (const record& copy) = default; + + /// Access the number of result fields of the prepared statement + size_t field_count() const; + /// Access raw data from result handle + column_data& field(size_t column) const; + /// Check if the field is NULL + bool is_null(size_t column) const; + /// Access field data + template <typename T> T get (size_t column) const; + }; + + /// Record set class + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + class record_set : public record { + protected: + /// Default constructor + record_set() = default; + + public: + /// Initializing constructor + record_set(const statement* s) : record(s) {} + /// Move constructor + record_set(record_set&& copy) = default; + /// Copy constructor + record_set(const record_set& copy) = default; + /// Destructor + ~record_set() = default; + + /// Move assignment + record_set& operator = (record_set&& copy) = default; + /// Copy assignment + record_set& operator = (const record_set& copy) = default; + + /// Fetch next result row. Stop if result is <> traits::OK + int fetch_one() const; + }; + + /// Record binder class + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + class record_bind : public record { + protected: + /// Default constructor + record_bind() = default; + + public: + /// Initializing constructor + record_bind(const statement* s) : record(s) {} + /// Move constructor + record_bind(record_bind&& copy) = delete; + /// Copy constructor + record_bind(const record_bind& copy) = default; + /// Destructor + ~record_bind() = default; + + /// Move assignment + record_bind& operator = (record_bind&& copy) = delete; + /// Copy assignment + record_bind& operator = (const record_bind& copy) = default; + + /// Generic bind of a single parameter column + int bind(size_t column, field_type typ, const void* data, size_t len) const; + /// Type specific bind operator for blobs + int bind(size_t column, const blob_t& data) const; + /// Type specific bind operator for blobs + int bind(size_t column, const vblob_t& data) const; + /// Type specific bind operator for blobs + int bind(size_t column, const uint8_t* data, size_t len) const; + /// Type specific bind operator for strings + int bind(size_t column, char* const data) const; + /// Type specific bind operator for strings + int bind(size_t column, const char* const data) const; + /// Type specific bind operator for strings + int bind(size_t column, std::string& data) const; + /// Type specific bind operator for strings + int bind(size_t column, const std::string& data) const; + /// Type specific bind operator for blobs + template <typename T> int bind(size_t column, T data) const; + }; + + /// Statement handle class + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + class statement : public record_bind { + public: + friend class record; + friend class record_bind; + + typedef std::vector<column_data> columns_t; + public: + + /// Statement handle class + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + class backend { + public: + backend* stmt {nullptr}; + columns_t params; + columns_t fields; + + enum action_type { EXEC, FETCH, RESET, FINALIZE, ERRNO }; + + public: + /// Initializing constructor + backend() = default; + /// Default destructor + virtual ~backend() = default; + /// Access error message on failure + virtual std::string errmsg() = 0; + /// Character string of the prepared statement + virtual const char* sql() = 0; + /// Access the number of rows affected by the last statement + virtual long rows_affected() = 0; + /// Perform multiple backend operations + virtual int action(action_type type) = 0; + /// Bind a single column to the data passed + virtual int bind(size_t col, field_type typ, const void* data, size_t len) = 0; + }; + std::shared_ptr<backend> intern; + + protected: + + /// Access raw data from result handle + column_data& _field(size_t column) const; + /// Generic bind of a single parameter column + int _bind(size_t column, field_type typ, const void* data, size_t len) const; + + public: + + /// Default constructor + statement(); + /// Move constructor + statement(statement&& copy) = delete; + /// Copy constructor + statement(const statement& copy); + /// Destructor + ~statement() = default; + + /// Move assignment + statement& operator = (statement&& copy); + /// Copy assignment + statement& operator = (const statement& copy); + + /// Create statement and execute SQL statement with variable number of arguments + static statement create(const database& db, const char* fmt, ...); + + /// Prepare a new statement and allocate resources. + int prepare(const database& db, const std::string& sql); + /// Execute SQL statement with variable number of arguments + int prepare(const database& db, const char* fmt, ...); + + /// Access the number of result fields of the prepared statement + size_t param_count() const; + + /// Check handle validity + bool is_valid() const; + /// Access error number of last failed statement + int error() const; + /// Access error message on failure + std::string errmsg() const; + /// Character string of the prepared statement + const char* sql() const; + + /// Access the number of rows affected by the last statement + long rows_affected() const; + + /// Execute prepared statement + int execute() const; + + /// Fetch next result row. Stop if result is <> traits::OK + int fetch_one() const; + + /// Reset SQL statement: Reset all data to re-execute the statement with new parameters + int reset() const; + + /// Finalize and run-down the statement. Release all resources. + int finalize(); + + /// Access to the current selected row + record current_row() const; + + /// Access raw parameter data from statement handle + column_data& param(size_t column) const; + }; + + /// Default constructor + inline statement::statement() : record_bind(this) + { + } + + /// Copy constructor + inline statement::statement(const statement& copy) + : record_bind(this), intern(copy.intern) + { + } + + /// Access the number of result fields of the prepared statement + inline size_t statement::param_count() const { + return intern->params.size(); + } + + /// Check handle validity + inline bool statement::is_valid() const { + return intern->stmt != nullptr; + } + + /// Access error message on failure + inline std::string statement::errmsg() const { + return intern->errmsg(); + } + + /// Character string of the prepared statement + inline const char* statement::sql() const { + return intern->sql(); + } + + /// Access to the current selected row + inline record statement::current_row() const { + return record(this); + } + + /// Access raw parameter data from statement handle + inline column_data& statement::param(size_t column) const { + if ( column >= intern->params.size() ) + invalid_statement("Invalid column identifier"); + return intern->params.at(column); + } + + /// Access raw data from result handle + inline column_data& statement::_field(size_t column) const { + if ( column >= intern->fields.size() ) + invalid_statement("Invalid column identifier"); + return intern->fields.at(column); + } + + /// Access the number of rows affected by the last statement + inline long statement::rows_affected() const { + return intern->rows_affected(); + } + + /// Access the number of result fields of the prepared statement + inline size_t record::field_count() const { + return this->stmt->intern->fields.size(); + } + + /// Access raw data from result handle + inline column_data& record::field(size_t column) const { + return this->stmt->_field(column); + } + + inline bool record::is_null(size_t col) const { + return this->stmt->_field(col).null; + } +} // End namespace sqldb + +/// The default is an inline implementation +/// To use a library compile with -DSQLDB_INLINE +/// +#ifndef SQLDB_INLINE +#define SQLDB_INLINE inline + +#include <sqldb/sqldb-imp.h> + +#undef SQLDB_INLINE +#endif + +#endif // SQLDB_SQLDB_INTERFACE_H diff --git a/Online/sqldb/sqldb/sqlite.h b/Online/sqldb/sqldb/sqlite.h new file mode 100644 index 000000000..f21444e4a --- /dev/null +++ b/Online/sqldb/sqldb/sqlite.h @@ -0,0 +1,37 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Package : RPC +// +// Author : Markus Frank +//========================================================================== +#ifndef SQLDB_SQLITE_INTERFACE_H +#define SQLDB_SQLITE_INTERFACE_H + +/// Framework include files +#include <sqldb/sqldb.h> + +typedef struct sqlite3 sqlite3; +typedef struct sqlite3_stmt sqlite3_stmt; + +/// Technology abstraction layer +/** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ +class sqlite : public sqldb::traits { + public: + typedef sqlite3 dbase_t; + typedef sqlite3_stmt statement_t; +}; + +#endif // SQLDB_SQLITE_INTERFACE_H diff --git a/Online/sqldb/src/mysql.cpp b/Online/sqldb/src/mysql.cpp new file mode 100644 index 000000000..697745b27 --- /dev/null +++ b/Online/sqldb/src/mysql.cpp @@ -0,0 +1,424 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Package : RPC +// +// Author : Markus Frank +//========================================================================== + +/// Framework include files +#include <sqldb/mysql.h> + +/// MySQL header +#include <mysql/mysql.h> + +struct mysql::dbase_t : public MYSQL { }; +struct mysql::statement_t : public MYSQL_STMT { }; + +/// SQLDB namespace declaration +namespace sqldb { + + namespace mysql_imp { + /// + inline bool mysql_ok(int status) { + return status == 0; + } + /// + inline field_type gen_field_type(enum_field_types typ) { + switch(typ) { + case MYSQL_TYPE_TINY: return TYPE_TINY; + case MYSQL_TYPE_SHORT: return TYPE_SHORT; + case MYSQL_TYPE_LONG: return TYPE_LONG; + case MYSQL_TYPE_LONGLONG: return TYPE_LONGLONG; + case MYSQL_TYPE_FLOAT: return TYPE_FLOAT; + case MYSQL_TYPE_DOUBLE: return TYPE_DOUBLE; + case MYSQL_TYPE_STRING: return TYPE_STRING; + case MYSQL_TYPE_BLOB: return TYPE_BLOB; + case MYSQL_TYPE_TIME: return TYPE_TIME; + case MYSQL_TYPE_DATE: return TYPE_TIME; + case MYSQL_TYPE_DATETIME: return TYPE_TIME; + default: + break; + } + throw std::runtime_error("Invalid data type encountered!"); + } + /// + inline enum_field_types mysql_field_type(field_type typ) { + switch(typ) { + case TYPE_TINY: return MYSQL_TYPE_TINY; + case TYPE_SHORT: return MYSQL_TYPE_SHORT; + case TYPE_LONG: return MYSQL_TYPE_LONG; + case TYPE_LONGLONG: return MYSQL_TYPE_LONGLONG; + case TYPE_FLOAT: return MYSQL_TYPE_FLOAT; + case TYPE_DOUBLE: return MYSQL_TYPE_DOUBLE; + case TYPE_STRING: return MYSQL_TYPE_STRING; + case TYPE_BLOB: return MYSQL_TYPE_BLOB; + case TYPE_TIME: return MYSQL_TYPE_DATETIME; + default: + break; + } + throw std::runtime_error("Invalid data type encountered!"); + } + + /// Technology abstraction layer + /** + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + struct column_bind : public MYSQL_BIND { + public: + /// Bind parameter to null value + int bind_store_null(column_data* d) { + d->null = true; + this->is_null = &d->null; + this->buffer = nullptr; + this->length = nullptr; + this->error = nullptr; + return sqldb::OK; + } + /// Generic parameter binding + int bind_store(column_data* d, const unsigned char* ptr, size_t len) { + if ( ptr ) { + d->bind_pointer(ptr, len); + this->buffer = d->item.string; + this->length = &d->length; + this->is_null = nullptr; + this->error = nullptr; + return sqldb::OK; + } + return this->bind_store_null(d); + } + /// Type specific parameter bind + template <typename T> int bind_store(column_data* d, T& to, const T* from) { + if ( from ) { + to = *from; + d->length = sizeof(T); + this->buffer = &to; + this->length = nullptr; + this->is_null = nullptr; + this->error = nullptr; + return sqldb::OK; + } + return this->bind_store_null(d); + } + /// Bind output field + int bind_fetch(column_data* d, void* to, size_t len) { + d->null = false; + d->error = false; + d->length = 0; + this->buffer = to; + this->length = &d->length; + this->is_null = &d->null; + this->error = &d->error; + this->buffer_length = len; + return sqldb::OK; + } + }; + + /// Technology abstraction layer for a MySQL prepared statement + /** + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + struct statement_backend : public statement::backend { + public: + typedef std::tuple<column_data*, column_bind*, MYSQL_FIELD*> field_t; + std::vector<column_bind> _pbind, _fbind; + std::string _sql_string; + MYSQL_STMT* _imp { nullptr }; + + /// Access to the internal statement handle. Throws exception if the handle is invalid + MYSQL_STMT* handle() const { + if ( this->_imp ) return this->_imp; + throw std::runtime_error("Failed to access statement handle. [Invalid handle]"); + } + std::pair<column_data*, column_bind*> param(int col) { + return std::make_pair(¶ms[col], &_pbind[col]); + } + field_t field(int col) { + return field_t(&this->fields[col], &_fbind[col], &_imp->fields[col]); + } + /// Access error message on failure + virtual std::string errmsg() override { + return ::mysql_stmt_error(this->handle()); + } + /// Character string of the prepared statement + virtual const char* sql() override { + return this->_sql_string.c_str(); + } + /// Access the number of rows affected by the last statement + virtual long rows_affected() override { + return ::mysql_stmt_num_rows(this->handle()); + } + + /// Prepare a new statement and allocate resources. + int prepare(mysql::dbase_t* db, const std::string& sql_str) { + MYSQL_STMT* stmt = nullptr; + if ( _imp ) ::mysql_stmt_reset(_imp); + if ( _imp ) ::mysql_stmt_close(_imp); + stmt = ::mysql_stmt_init(db); + if ( stmt ) { + int ret = ::mysql_stmt_prepare(stmt, sql_str.c_str(), sql_str.length()); + if ( ret == sqldb::OK ) { +#if 0 + unsigned long type = CURSOR_TYPE_READ_ONLY; + int rc = ::mysql_stmt_attr_set(stmt, STMT_ATTR_CURSOR_TYPE, &type); + if ( rc ) { + return this->error(); + } +#endif + int num_param = ::mysql_stmt_param_count(stmt); + int num_field = ::mysql_stmt_field_count(stmt); + this->_pbind.resize(num_param, column_bind()); + this->params.resize(num_param, column_data()); + this->_fbind.resize(num_field, column_bind()); + this->fields.resize(num_field, column_data()); + this->_sql_string = sql_str; + this->_imp = stmt; + this->stmt = this; + return sqldb::OK; + } + return sqldb::ERROR; + } + return invalid_statement(); + } + + /// Execute prepared statement + virtual int execute() { + if( auto* h = this->_imp ) { + if( this->_pbind.size() ) { + if( ::mysql_stmt_bind_param(h, &this->_pbind.front()) ) { + return invalid_statement(::mysql_stmt_error(h)); + } + } + if( sqldb::OK == ::mysql_stmt_execute(h) ) { + if( sqldb::OK != ::mysql_stmt_store_result(h) ) { + if( auto* r = ::mysql_stmt_result_metadata(h) ) { + ::mysql_free_result(r); + return invalid_statement(::mysql_stmt_error(h)); + } + } + return sqldb::OK; + } + return invalid_statement(::mysql_stmt_error(h)); + } + return invalid_statement(); + } + + /// Fetch next result row. Stop if result is NON-zero + virtual int fetch_one() { + if ( this->_imp ) { + size_t nfld = this->_fbind.size(); + if ( nfld > 0 ) { + ::memset(&this->_fbind.front(), 0, nfld*sizeof(column_bind)); + for( size_t i=0; i < nfld; i++ ) { + this->bind_fetch(i); + if ( !mysql_ok(::mysql_stmt_bind_result(this->_imp, &this->_fbind.front())) ) + return invalid_statement(::mysql_stmt_error(this->_imp)); + } + } + int ret = ::mysql_stmt_fetch(this->_imp); + if( ret == MYSQL_NO_DATA ) { + return sqldb::NO_DATA; + } + else if( ret == MYSQL_DATA_TRUNCATED ) { + for( size_t i=0; i < nfld; i++ ) { + auto& d = this->fields[i]; + if( d.error && !d.null && d.length >= sizeof(d.buf) ) { + auto& b = this->_fbind[i]; + d.vbuf.resize(d.length); + d.item.blob = &d.vbuf.front(); + b.buffer = d.item.blob; + b.buffer_length = d.length; + if( !mysql_ok(::mysql_stmt_fetch_column(this->_imp,&b,i,0)) ) { + return invalid_statement(::mysql_stmt_error(this->_imp)); + } + } + } + } + return sqldb::OK; + } + return invalid_statement(); + } + + /// Bind output field identified by column number + int bind_fetch(size_t col) { + auto* p = this; + auto [d, b, f] = p->field(col); + + ::memset(b, 0, sizeof(column_bind)); + b->buffer_type = f->type; + d->type = gen_field_type(f->type); + switch(d->type) { + case TYPE_TINY: return b->bind_fetch(d, &d->item.int8, sizeof(d->item.int8)); + case TYPE_SHORT: return b->bind_fetch(d, &d->item.int16, sizeof(d->item.int16)); + case TYPE_LONG: return b->bind_fetch(d, &d->item.int32, sizeof(d->item.int32)); + case TYPE_LONGLONG: return b->bind_fetch(d, &d->item.int64, sizeof(d->item.int64)); + case TYPE_FLOAT: return b->bind_fetch(d, &d->item.real32, sizeof(d->item.real32)); + case TYPE_DOUBLE: return b->bind_fetch(d, &d->item.real64, sizeof(d->item.real64)); + case TYPE_STRING: return b->bind_fetch(d, d->buf, sizeof(d->buf)); + case TYPE_BLOB: return b->bind_fetch(d, d->buf, sizeof(d->buf)); + case TYPE_TIME: return b->bind_fetch(d, &d->item.time.stamp, sizeof(d->item.int64)); + default: return invalid_statement("Invalid MySQL Field data type"); + } + return sqldb::OK; + } + + /// Bind a single column to the data passed + virtual int bind(size_t col, field_type typ, const void* data, size_t len) override { + pointers_t ptr(data); + auto [d, b] = this->param(col); + + ::memset(&b, 0, sizeof(column_bind)); + d->type = typ; + b->buffer_type = mysql_field_type(typ); + switch(typ) { + case TYPE_NULL: return b->bind_store_null(d); + case TYPE_TINY: return b->bind_store(d, d->item.uint8, ptr.puint8); + case TYPE_SHORT: return b->bind_store(d, d->item.uint16, ptr.puint16); + case TYPE_LONG: return b->bind_store(d, d->item.uint32, ptr.puint32); + case TYPE_LONGLONG: return b->bind_store(d, d->item.uint64, ptr.puint64); + case TYPE_FLOAT: return b->bind_store(d, d->item.real32, ptr.preal32); + case TYPE_DOUBLE: return b->bind_store(d, d->item.real64, ptr.preal64); + case TYPE_STRING: return b->bind_store(d, ptr.puint8, len); + case TYPE_BLOB: return b->bind_store(d, ptr.puint8, len); + /// For now store time stamps as long integers (time_t) + case TYPE_TIME: return b->bind_store(d, d->item.time.stamp, ptr.puint64); + default: return invalid_statement("Invalid MySQL parameter data type"); + } + return sqldb::OK; + } + + /// Action routine for various functions + int action(action_type type) override { + switch(type) { + case EXEC: /// Execute prepared statement + return this->execute(); + case FETCH: /// Fetch next result row. Stop if result is NON-zero + return this->fetch_one(); + case RESET: /// Reset all data to re-execute the statement with new parameters + if( auto* h = this->handle() ) { + if( !mysql_ok(::mysql_stmt_reset(h)) ) + return sqldb::ERROR; + } + return sqldb::OK; + case FINALIZE: /// Finalize and run-down the statement. Release all resources. + if ( this->_imp ) { + ::mysql_stmt_reset(this->_imp); + ::mysql_stmt_close(this->_imp); + this->_imp = nullptr; + } + return sqldb::OK; + case ERRNO: /// Access error number of last failed statement + return ::mysql_stmt_errno(this->handle()); + default: + throw std::runtime_error("Invalid action request received!"); + } + } + }; + + /// Technology abstraction layer for a MySQL database + /** + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + struct dbase_backend : public database::backend { + mysql::dbase_t _imp; + mysql::dbase_t* _handle {nullptr}; + + /// Access to the internal database handle. Throws exception if the handle is invalid + mysql::dbase_t* handle() const { + if ( _handle ) return _handle; + throw std::runtime_error("Invalid MySQL database handle"); + } + /// Access error message on failure + virtual std::string errmsg() override { + return ::mysql_error(this->handle()); + } + /// Access the number of rows affected by the last statement + virtual long rows_affected() override { + return ::mysql_affected_rows(this->handle()); + } + /// Perform multiple actions + virtual int action(action_type type) override { + switch(type) { + case CLOSE: + if ( this->_handle ) { + ::mysql_close(this->_handle); + ::memset(&_imp, 0, sizeof(_imp)); + this->_handle = nullptr; + } + return sqldb::OK; + case BEGIN: /// Start transaction +#define _START "START TRANSACTION;" + return ::mysql_real_query(this->_handle,_START,sizeof(_START)) ? sqldb::ERROR : sqldb::OK; + case COMMIT: /// Commit transaction + return ::mysql_commit(this->handle()); + case ROLLBACK: /// Rollback transaction + return ::mysql_rollback(this->handle()); + case ERRNO: /// Access error number of last failed statement + return ::mysql_errno(this->handle()); + default: + throw std::runtime_error("Invalid action request received!"); + } + } + /// Execute prepared statement + virtual int execute(std::string& err, const std::string& sql) override { + if( this->_handle ) { + if( ::mysql_real_query(this->_handle, sql.c_str(), sql.length()) ) { + err = ::mysql_error(this->_handle); + return sqldb::ERROR; + } + return sqldb::OK; + } + return sqldb::INVALID_HANDLE; + } + /// Prepare a new statement and allocate resources. + int prepare(statement& bs, const std::string& sql_str) { + auto* stm_imp = new statement_backend(); + bs.intern.reset(stm_imp); + return stm_imp->prepare(this->handle(), sql_str); + } + /// Open database + int open(const std::string& connect_string) { + auto args = connection_args().parse(connect_string); + this->_handle = nullptr; + if ( nullptr == ::mysql_real_connect(&_imp, + args["host"].c_str(), + args["user"].c_str(), + args["password"].c_str(), + args["database"].c_str(), + 0, nullptr, 0) ) { + return sqldb::ERROR; + } + this->_handle = &_imp; + this->name = args["database"]; + return sqldb::OK; + } + + /// Initializing constructor + dbase_backend() = default; + }; + } + + /// Open the database connection using all information in passed string + template <> + std::pair<std::shared_ptr<database::backend>, int> + database::backend::open<mysql>(const std::string& connect_string) { + auto db = std::make_shared<mysql_imp::dbase_backend>(); + int ret = db->open(connect_string); + if ( ret == sqldb::OK ) + return make_pair(db, sqldb::OK); + return make_pair(std::make_shared<mysql_imp::dbase_backend>(), ret); + } +} // End namespace sqldb diff --git a/Online/sqldb/src/oracle.cpp b/Online/sqldb/src/oracle.cpp new file mode 100644 index 000000000..eb9ba90b8 --- /dev/null +++ b/Online/sqldb/src/oracle.cpp @@ -0,0 +1,645 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Package : RPC +// +// Author : Markus Frank +//========================================================================== + +/// Framework include files +#include <sqldb/oracle.h> + +/// SQLite3 header +#include "sql.h" +#include "oci.h" +#include "ocidfn.h" + +/// C/C++ headers +#include <stdexcept> + +#define SQLDB_SQLITE_INLINE + +/// SQLDB namespace declaration +namespace sqldb { + + namespace oracle_imp { + + typedef unsigned short result_t; + typedef struct OCIEnv* EnvH; + typedef struct OCISvcCtx* DbcH; + typedef struct OCIStmt* StmH; + typedef struct OCIError* ErrorH; + typedef struct OCIBind* BindH; + typedef struct OCIParam* ParamH; + typedef struct OCIDefine* DefineH; + typedef struct OCILobLocator* LobLocatorH; + + class column_bind; + class statement_backend; + + /// + inline bool oracle_ok(int status) { + return ((status)&(~1)) == 0; + } + /// + inline field_type gen_field_type(int typ) { + switch(typ) { + case SQLT_UIN: + case SQLT_NUM: + case SQLT_INT: return TYPE_LONGLONG; + case SQLT_BFLOAT: + case SQLT_BDOUBLE: + case SQLT_FLT: return TYPE_DOUBLE; + case SQLT_AFC: + case SQLT_AVC: + case SQLT_STR: + case SQLT_VCS: + case SQLT_LVC: return TYPE_STRING; + case SQLT_VBI: + case SQLT_BIN: + case SQLT_LBI: + case SQLT_LVB: return TYPE_BLOB; + default: + break; + } + throw std::runtime_error("Invalid data type encountered!"); + } + /// + inline int oracle_field_type(field_type typ) { + switch(typ) { + case TYPE_TINY: + case TYPE_SHORT: + case TYPE_LONG: + case TYPE_LONGLONG: return SQLT_INT; + case TYPE_FLOAT: + case TYPE_DOUBLE: return SQLT_FLT; + case TYPE_STRING: return SQLT_VCS; + case TYPE_BLOB: return SQLT_LBI; + default: + break; + } + throw std::runtime_error("Invalid data type encountered!"); + } + + std::string error_message(int code, ErrorH err) { + text b[512]; + int c; + std::string msg = ""; + switch (code) { + case OCI_SUCCESS: + msg = "SUCCESS"; + break; + case OCI_SUCCESS_WITH_INFO: + msg = "Error - OCI_SUCCESS_WITH_INFO"; + break; + case OCI_NEED_DATA: + msg = "Error - OCI_NEED_DATA"; + break; + case OCI_NO_DATA: + msg = "Error - OCI_NODATA - "; + ::OCIErrorGet(err,1,0,&c,b,sizeof(b),OCI_HTYPE_ERROR); + msg += (char*)b; + break; + case OCI_ERROR: + msg = "Error - "; + ::OCIErrorGet(err,1,0,&c,b,sizeof(b),OCI_HTYPE_ERROR); + msg += (char*)b; + break; + case OCI_INVALID_HANDLE: + msg = "Error - OCI_INVALID_HANDLE - "; + ::OCIErrorGet(err,1,0,&c,b,sizeof(b),OCI_HTYPE_ERROR); + msg += (char*)b; + break; + case OCI_STILL_EXECUTING: + msg = "Error - OCI_STILL_EXECUTING"; + break; + default: + msg = ""; + break; + } + return msg; + } + + /// Technology abstraction layer + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + class column_bind { + public: + size_t _column { 0 }; + statement_backend* _stm { nullptr }; + BindH _bind { nullptr }; + DefineH _define { nullptr }; + LobLocatorH _lob { nullptr }; + ub2 _sql_type; + + public: + /// Default destructor + ~column_bind(); + /// Generic parameter binding + int bind_str(column_data& d, const char* ptr, size_t len); + /// Generic parameter binding + int bind_blob(column_data& d, const unsigned char* ptr, size_t len); + /// Type specific parameter bind + int bind_int(column_data& d, int32_t from); + /// Type specific parameter bind + int bind_int64(column_data& d, int64_t from); + /// Type specific parameter bind + int bind_flt(column_data& d, double from); + /// Read lob data + int fetch_data(column_data& d); + /// Bind output field + int bind_fetch(column_data& d, void* ptr, size_t len); + }; + + /// Technology abstraction layer + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + struct statement_backend : public statement::backend { + public: + + typedef std::tuple<column_data*, column_bind*, ub2> field_t; + typedef std::vector<column_bind> Binders; + std::vector<ub2> _types; + Binders _pbind, _fbind; + EnvH _env { nullptr }; + DbcH _dbc { nullptr }; + StmH _stmt { nullptr }; + ErrorH _error { nullptr }; + std::string _sql_str { }; + int _error_code { 0 }; + + /// Initializing constructor + statement_backend(EnvH env, DbcH dbc, ErrorH err) : _env(env), _dbc(dbc), _error(err) { + } + + /// Default destructor + virtual ~statement_backend() { + if ( _stmt ) { + ::OCIHandleFree(_stmt, OCI_HTYPE_STMT); + _stmt = nullptr; + } + if ( _error ) { + ::OCIHandleFree(_error, OCI_HTYPE_ERROR); + _error = nullptr; + } + } + /// Access to the internal statement handle. Throws exception if the handle is invalid + StmH handle() const { + if ( this->_stmt ) return this->_stmt; + throw std::runtime_error("Failed to access statement handle. [Invalid handle]"); + } + std::pair<column_data*, column_bind*> param(int col) { + return std::make_pair(¶ms[col], &_pbind[col]); + } + field_t field(int col) { + return field_t(&this->fields[col], &_fbind[col], _types[col]); + } + /// Execute prepared statement + int execute(); + /// Fetch next result row. Stop if result is NON-zero + int fetch_one(); + /// Bind output field identified by column number + int bind_fetch(size_t col); + /// Prepare a new statement and allocate resources. + int prepare(const std::string& sql_str); + /// Access error message on failure + std::string errmsg() override { + return error_message(_error_code, _error); + } + /// Character string of the prepared statement + const char* sql() override { + return _sql_str.c_str(); + } + /// Access the number of rows affected by the last statement + long rows_affected() override { + ub4 row_count = 0; + result_t ret = ::OCIAttrGet(_stmt, OCI_HTYPE_STMT, &row_count, 0, OCI_ATTR_ROW_COUNT, _error); + if ( oracle_ok(ret) ) + return row_count; + _error_code = ret; + return -1; + } + /// Bind a single column to the data passed + virtual int bind(size_t col, field_type typ, const void* data, size_t len) override; + /// + int action(action_type type) override; + }; + + /// Technology abstraction layer for the database + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + struct dbase_backend : public database::backend { + DbcH _dbc; + EnvH _env; + ErrorH _error; + int _error_code; + + /// Initializing constructor + dbase_backend(); + /// Default destructor + virtual ~dbase_backend(); + + /// Access to the internal database handle. Throws exception if the handle is invalid + DbcH handle() const { + if ( _dbc ) return _dbc; + throw std::runtime_error("Invalid MySQL database handle"); + } + /// Prepare a new statement and allocate resources. + int prepare(statement& bs, const std::string& sql_str) override; + /// Open database + int open(const std::string& connect_string); + /// Access error message on failure + std::string errmsg() override { + return error_message(_error_code, _error); + } + /// Access the number of rows affected by the last statement + virtual long rows_affected() override { + return -1; + } + /// Execute prepared statement + virtual int execute(std::string& error, const std::string& sql_str) override; + /// Perform multiple actions + virtual int action(action_type type) override; + }; + + /// Default destructor + column_bind::~column_bind() { + if ( _lob ) { + ::OCIDescriptorFree(_lob, OCI_DTYPE_LOB); + _lob = nullptr; + } + } + + /// Generic parameter binding + int column_bind::bind_str(column_data& d, const char* ptr, size_t len) { + pointers_t p(ptr); + d.bind_pointer(p.puint8, len); + auto ret = ::OCIBindByPos(_stm->_stmt, &_bind, _stm->_error, + _column, p.pointer, len, SQLT_VCS, + 0, 0, 0, 0, 0, OCI_DEFAULT); + return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; + } + /// Generic parameter binding + int column_bind::bind_blob(column_data& d, const unsigned char* ptr, size_t len) { + pointers_t p(ptr); + d.bind_pointer(ptr, len); + auto ret = ::OCIBindByPos(_stm->_stmt, &_bind, _stm->_error, + _column, p.pointer, len, SQLT_LVB, + 0, 0, 0, 0, 0, OCI_DEFAULT); + return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; + } + /// Type specific parameter bind + int column_bind::bind_int(column_data& d, int32_t from) { + d.item.int32 = from; + d.length = sizeof(int); + result_t ret = ::OCIBindByPos(_stm->_stmt, &_bind, _stm->_error, + _column, &d.item.int32, sizeof(d.item.int32), SQLT_INT, + 0, 0, 0, 0, 0, OCI_DEFAULT); + return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; + } + /// Type specific parameter bind + int column_bind::bind_int64(column_data& d, int64_t from) { + d.item.int64 = from; + d.length = sizeof(int64_t); + result_t ret = ::OCIBindByPos(_stm->_stmt, &_bind, _stm->_error, + _column, &d.item.int64, sizeof(d.item.int64), SQLT_INT, + 0, 0, 0, 0, 0, OCI_DEFAULT); + return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; + } + /// Type specific parameter bind + int column_bind::bind_flt(column_data& d, double from) { + d.item.real64 = from; + d.length = sizeof(double); + result_t ret = ::OCIBindByPos(_stm->_stmt, &_bind, _stm->_error, + _column, &d.item.real64, sizeof(d.item.real64), SQLT_FLT, + 0, 0, 0, 0, 0, OCI_DEFAULT); + return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; + } + /// Read lob data + int column_bind::fetch_data(column_data& d) { + result_t rc = OCI_SUCCESS; + DbcH db = _stm->_dbc; + ErrorH err = _stm->_error; + if ( d.type == TYPE_BLOB && nullptr != _lob ) { + rc = ::OCILobOpen(db, err, _lob, OCI_LOB_READONLY); + if ( oracle_ok(rc) ) { + unsigned int len = 0, rd_len = 0; + rc = ::OCILobGetLength(db, err, _lob, &len); + if ( oracle_ok(rc) ) { + d.length = len; + d.vbuf.resize(len); + d.item.pointer = &d.vbuf[0]; + rc = ::OCILobRead(db,err,_lob,&rd_len,1,d.item.blob,len,0,0,0,SQLCS_IMPLICIT); + } + } + ::OCILobClose(db, err, _lob); + } + return oracle_ok(rc) ? sqldb::OK : sqldb::ERROR; + } + /// Bind output field + int column_bind::bind_fetch(column_data& d, void* ptr, size_t len) { + result_t rc = OCI_SUCCESS; + d.null = false; + d.error = false; + d.length = 0; + if ( d.type == TYPE_BLOB ) { + if ( _lob ) { + ::OCIDescriptorFree(_lob, OCI_DTYPE_LOB); + } + rc = ::OCIDescriptorAlloc(_stm->_env, (void**)&_lob, OCI_DTYPE_LOB, 0, 0); + if ( oracle_ok(rc) ) { + len = -1; + ptr = &_lob; + } + } + if ( oracle_ok(rc) ) { + rc = ::OCIDefineByPos(_stm->_stmt, &_define, _stm->_error, + _column, ptr, len, _sql_type, + 0, 0, 0, OCI_DEFAULT); + } + return oracle_ok(rc) ? sqldb::OK : sqldb::ERROR; + } + + /// Fetch next result row. Stop if result is NON-zero + int statement_backend::fetch_one() { + if ( _stmt ) { + size_t nfld = _fbind.size(); + if ( nfld > 0 ) { + for( size_t i=0; i < nfld; i++ ) { + if ( bind_fetch(i) != sqldb::OK ) { + return invalid_statement(errmsg()); + } + } + } + result_t ret = OCIStmtFetch(_stmt, _error, 1, OCI_FETCH_NEXT, OCI_DEFAULT); + if( ret == OCI_SUCCESS ) { + for( size_t i=0; i < nfld; i++ ) { + auto& d = fields[i]; + if ( d.type == TYPE_BLOB ) { + if ( sqldb::OK != _fbind[i].fetch_data(d) ) { + return invalid_statement(errmsg()); + } + } + } + } + return sqldb::OK; + } + return invalid_statement(); + } + + /// Bind output field identified by column number + int statement_backend::bind_fetch(size_t col) { + auto* p = this; + auto [d, b, type] = p->field(col); + b->_column = col; + b->_stm = this; + b->_bind = nullptr; + b->_define = nullptr; + b->_lob = nullptr; + b->_sql_type = type; + d->type = gen_field_type(type); + switch(type) { + case TYPE_TINY: return b->bind_fetch(*d, &d->item.int8, sizeof(d->item.int8)); + case TYPE_SHORT: return b->bind_fetch(*d, &d->item.int16, sizeof(d->item.int16)); + case TYPE_LONG: return b->bind_fetch(*d, &d->item.int32, sizeof(d->item.int32)); + case TYPE_LONGLONG: return b->bind_fetch(*d, &d->item.int64, sizeof(d->item.int64)); + case TYPE_FLOAT: return b->bind_fetch(*d, &d->item.real32, sizeof(d->item.real32)); + case TYPE_DOUBLE: return b->bind_fetch(*d, &d->item.real64, sizeof(d->item.real64)); + case TYPE_STRING: return b->bind_fetch(*d, nullptr, sizeof(d->buf)); + case TYPE_BLOB: return b->bind_fetch(*d, nullptr, sizeof(d->buf)); + default: return invalid_statement("Invalid MySQL Field data type"); + } + return sqldb::OK; + } + + /// Bind a single column to the data passed + int statement_backend::bind(size_t col, field_type typ, const void* data, size_t len) { + pointers_t ptr(data); + auto [d, b] = param(col); + d->type = typ; + b->_column = col; + b->_stm = this; + b->_bind = nullptr; + b->_define = nullptr; + b->_lob = nullptr; + b->_sql_type = oracle_field_type(typ); + switch(typ) { + case TYPE_TINY: return b->bind_int (*d, *ptr.puint8); + case TYPE_SHORT: return b->bind_int (*d, *ptr.puint16); + case TYPE_LONG: return b->bind_int (*d, *ptr.puint32); + case TYPE_LONGLONG: return b->bind_int64(*d, *ptr.puint64); + case TYPE_FLOAT: return b->bind_flt (*d, *ptr.preal32); + case TYPE_DOUBLE: return b->bind_flt (*d, *ptr.preal64); + case TYPE_STRING: return b->bind_str (*d, ptr.pstring, len); + case TYPE_BLOB: return b->bind_blob (*d, ptr.puint8, len); + default: return invalid_statement("Invalid MySQL parameter data type"); + } + return sqldb::OK; + } + + /// Prepare a new statement and allocate resources. + int statement_backend::prepare(const std::string& sql_str) { + StmH stm; + ub4 count; + if ( _stmt ) { + ::OCIHandleFree(_stmt, OCI_HTYPE_STMT); + _stmt = nullptr; + } + result_t ret = ::OCIHandleAlloc(_env, (void**)&stm, OCI_HTYPE_STMT , 0, 0); + if ( !oracle_ok(ret) ) { + _error_code = ret; + return sqldb::ERROR; + } +#define CHECK(ret) if ( !oracle_ok(ret) ) { \ + ::OCIHandleFree(stm, OCI_HTYPE_STMT); \ + _types.clear(); \ + _error_code = ret; \ + return sqldb::ERROR; } + + ret = ::OCIStmtPrepare(stm, _error, + (OraText*)sql_str.c_str(), sql_str.length(), + OCI_NTV_SYNTAX, OCI_DEFAULT); + CHECK(ret); + ret = ::OCIAttrGet(stm, OCI_HTYPE_STMT, &count, nullptr, + OCI_ATTR_BIND_COUNT, _error); + CHECK(ret); + params.resize(count, column_data()); + ret = ::OCIAttrGet(stm, OCI_HTYPE_STMT, &count, nullptr, + OCI_ATTR_PARAM_COUNT, _error); + CHECK(ret); + _types.clear(); + for(size_t i=0; i<count; ++i) { + ub2 type; + ParamH colhd { nullptr }; + ret = ::OCIParamGet(stm, OCI_HTYPE_STMT, _error, (void**)&colhd, i); + CHECK(ret); + ret = ::OCIAttrGet(colhd, OCI_DTYPE_PARAM, &type, + nullptr, OCI_ATTR_DATA_TYPE, _error); + CHECK(ret); + _types.push_back(type); + } + fields.resize(count, column_data()); + _sql_str = sql_str; + _stmt = stm; + stmt = this; + return sqldb::OK; + } + /// + int statement_backend::action(action_type type) { + switch(type) { + case RESET: /// Reset all data to re-execute the statement with new parameters + return action(EXEC); + case FINALIZE: /// Finalize and run-down the statement. Release all resources. + if( _stmt ) { + ::OCIHandleFree(_stmt, OCI_HTYPE_STMT); + _stmt = nullptr; + } + return sqldb::OK; + case EXEC: /// Execute prepared statement + if( _stmt ) { + long iters = 1; + _error_code = ::OCIStmtExecute(_dbc,_stmt,_error,iters,0,0,0,OCI_DEFAULT); + switch( _error_code ) { + case 995: return sqldb::NO_DATA; // ORA-00955: name is already used by an existing object + case 1405: return sqldb::NO_DATA; // ORA-01405: fetched column value is NULL + default: return oracle_ok(_error_code) ? sqldb::OK : sqldb::ERROR; + } + } + return sqldb::ERROR; + case FETCH: /// Fetch next result row. Stop if result is NON-zero + return fetch_one(); + case ERRNO: /// Access error number of last failed statement + return _error_code; + default: + throw std::runtime_error("Invalid action request received!"); + } + } + + dbase_backend::dbase_backend() { + result_t rc; + rc = OCIEnvCreate(&_env, OCI_OBJECT, 0, 0, 0, 0, 0, 0); + if ( oracle_ok(rc) ) { + rc = OCIHandleAlloc (_env, (void**)&_error, OCI_HTYPE_ERROR , 0, 0); + if ( oracle_ok(rc) ) { + _error_code = OCI_SUCCESS; + return; + } + } + _error_code = rc; + } + /// Default destructor + dbase_backend::~dbase_backend() { + if ( _dbc ) { + action(CLOSE); + } + ::OCIHandleFree(_error, OCI_HTYPE_ERROR); + _error = nullptr; + } + /// Execute prepared statement + int dbase_backend::execute(std::string& error, const std::string& sql_str) { + char* err = nullptr; + int ret = OCI_SUCCESS; //do_exec(handle(), sql_str.c_str(), &err); + if ( !sql_str.empty() ) { + if ( ret != sqldb::OK && err != nullptr ) error = err; + } + return ret; + } + /// Prepare a new statement and allocate resources. + int dbase_backend::prepare(statement& bs, const std::string& sql_str) { + ErrorH err {nullptr}; + if ( bs.intern.get() ) { + bs.intern.reset(); + } + result_t rc = ::OCIHandleAlloc(_env, (void**)&err, OCI_HTYPE_ERROR, 0, 0); + if ( oracle_ok(rc) ) { + auto* stm = new statement_backend(_env, _dbc, err); + bs.intern.reset(stm); + return stm->prepare(sql_str); + } + _error_code = rc; + return sqldb::ERROR; + } + /// Open database + int dbase_backend::open(const std::string& connect_string) { + auto args = connection_args().parse(connect_string); + if ( _dbc ) { + ::OCILogoff(_dbc, _error); + ::OCIHandleFree(_dbc, OCI_HTYPE_SVCCTX); + } + DbcH db = nullptr; + const std::string& pwd = args["password"]; + const std::string& user = args["user"]; + const std::string& dbase = args["database"]; + result_t rc = ::OCILogon(_env, _error, &db, + (OraText*)user.c_str(), user.length(), + (OraText*)pwd.c_str(), pwd.length(), + (OraText*)dbase.c_str(),dbase.length()); + if ( !oracle_ok(rc) ) { + _error_code = rc; + return sqldb::ERROR; + } + _dbc = db; + name = dbase; + _error_code = OCI_SUCCESS; + return sqldb::OK; + } + /// Perform multiple actions + int dbase_backend::action(action_type type) { + result_t rc = OCI_SUCCESS; + switch(type) { + case BEGIN: /// Start transaction + rc = _dbc ? ::OCITransStart(_dbc, _error, 60, OCI_TRANS_NEW) : OCI_ERROR; + break; + case COMMIT: /// Commit transaction + rc = _dbc ? ::OCITransCommit(_dbc, _error, OCI_DEFAULT) : OCI_ERROR; + break; + case ROLLBACK: /// Rollback transaction + rc = _dbc ? ::OCITransRollback(_dbc, _error, OCI_DEFAULT) : OCI_ERROR; + break; + case CLOSE: /// Close database access + if ( _dbc ) { + ::OCILogoff(_dbc, _error); + ::OCIHandleFree(_dbc, OCI_HTYPE_SVCCTX); + } + _dbc = nullptr; + _error_code = OCI_SUCCESS; + return sqldb::OK; + case ERRNO: /// Access error number of last failed statement + return _error_code; + default: + throw std::runtime_error("Invalid actoin request received!"); + } + _error_code = rc; + return sqldb::ERROR; + } + } + + /// Open the database connection using all information in passed string + template <> std::pair<std::shared_ptr<database::backend>, int> + database::backend::open<oracle>(const std::string& connect_string) { + auto db = std::make_shared<oracle_imp::dbase_backend>(); + int ret = db->open(connect_string); + if ( ret == sqldb::OK ) + return make_pair(db, sqldb::OK); + return make_pair(std::make_shared<oracle_imp::dbase_backend>(), ret); + } +} // End namespace sqldb diff --git a/Online/sqldb/src/sql.h b/Online/sqldb/src/sql.h new file mode 100755 index 000000000..8449a9fbb --- /dev/null +++ b/Online/sqldb/src/sql.h @@ -0,0 +1,823 @@ +/***************************************************************** +** SQL.H - This is the the main include for ODBC Core functions. +** +** preconditions: +** #include "windows.h" +** +** (C) Copyright 1990 - 1996 By Microsoft Corp. +** +** Updated 5/12/93 for 2.00 specification +** Updated 5/23/94 for 2.01 specification +** Updated 11/10/94 for 2.10 specification +** Updated 04/10/95 for 2.50 specification +** Updated 6/6/95 for 3.00 specification +*********************************************************************/ + + +#ifndef __SQL +#define __SQL + +/* +* ODBCVER ODBC version number (0x0300). To exclude +* definitions introduced in version 3.0 (or above) +* #define ODBCVER 0x0250 before #including <sql.h> +*/ +#ifndef ODBCVER +#define ODBCVER 0x0300 +#endif + +#ifndef __SQLTYPES +#include "sqltypes.h" +#endif + +#ifdef __cplusplus +extern "C" { /* Assume C declarations for C++ */ +#endif /* __cplusplus */ + +/* special length/indicator values */ +#define SQL_NULL_DATA (-1) +#define SQL_DATA_AT_EXEC (-2) + + +/* return values from functions */ +#define SQL_SUCCESS 0 +#define SQL_SUCCESS_WITH_INFO 1 +#if (ODBCVER >= 0x0300) +#define SQL_NO_DATA 100 +#endif +#define SQL_ERROR (-1) +#define SQL_INVALID_HANDLE (-2) + +#define SQL_STILL_EXECUTING 2 +#define SQL_NEED_DATA 99 + +/* test for SQL_SUCCESS or SQL_SUCCESS_WITH_INFO */ +#define SQL_SUCCEEDED(rc) (((rc)&(~1))==0) + +/* flags for null-terminated string */ +#define SQL_NTS (-3) +#define SQL_NTSL (-3L) + +/* maximum message length */ +#define SQL_MAX_MESSAGE_LENGTH 512 + +/* date/time length constants */ +#if (ODBCVER >= 0x0300) +#define SQL_DATE_LEN 10 +#define SQL_TIME_LEN 8 /* add P+1 if precision is nonzero */ +#define SQL_TIMESTAMP_LEN 19 /* add P+1 if precision is nonzero */ +#endif + +/* handle type identifiers */ +#if (ODBCVER >= 0x0300) +#define SQL_HANDLE_ENV 1 +#define SQL_HANDLE_DBC 2 +#define SQL_HANDLE_STMT 3 +#define SQL_HANDLE_DESC 4 +#endif + +/* environment attribute */ +#if (ODBCVER >= 0x0300) +#define SQL_ATTR_OUTPUT_NTS 10001 +#endif + +/* connection attributes */ +#if (ODBCVER >= 0x0300) +#define SQL_ATTR_AUTO_IPD 10001 +#define SQL_ATTR_METADATA_ID 10014 +#endif /* ODBCVER >= 0x0300 */ + +/* statement attributes */ +#if (ODBCVER >= 0x0300) +#define SQL_ATTR_APP_ROW_DESC 10010 +#define SQL_ATTR_APP_PARAM_DESC 10011 +#define SQL_ATTR_IMP_ROW_DESC 10012 +#define SQL_ATTR_IMP_PARAM_DESC 10013 +#define SQL_ATTR_CURSOR_SCROLLABLE (-1) +#define SQL_ATTR_CURSOR_SENSITIVITY (-2) +#endif + +/* SQL_ATTR_CURSOR_SCROLLABLE values */ +#if (ODBCVER >= 0x0300) +#define SQL_NONSCROLLABLE 0 +#define SQL_SCROLLABLE 1 +#endif /* ODBCVER >= 0x0300 */ + +/* identifiers of fields in the SQL descriptor */ +#if (ODBCVER >= 0x0300) +#define SQL_DESC_COUNT 1001 +#define SQL_DESC_TYPE 1002 +#define SQL_DESC_LENGTH 1003 +#define SQL_DESC_OCTET_LENGTH_PTR 1004 +#define SQL_DESC_PRECISION 1005 +#define SQL_DESC_SCALE 1006 +#define SQL_DESC_DATETIME_INTERVAL_CODE 1007 +#define SQL_DESC_NULLABLE 1008 +#define SQL_DESC_INDICATOR_PTR 1009 +#define SQL_DESC_DATA_PTR 1010 +#define SQL_DESC_NAME 1011 +#define SQL_DESC_UNNAMED 1012 +#define SQL_DESC_OCTET_LENGTH 1013 +#define SQL_DESC_ALLOC_TYPE 1099 +#endif + +/* identifiers of fields in the diagnostics area */ +#if (ODBCVER >= 0x0300) +#define SQL_DIAG_RETURNCODE 1 +#define SQL_DIAG_NUMBER 2 +#define SQL_DIAG_ROW_COUNT 3 +#define SQL_DIAG_SQLSTATE 4 +#define SQL_DIAG_NATIVE 5 +#define SQL_DIAG_MESSAGE_TEXT 6 +#define SQL_DIAG_DYNAMIC_FUNCTION 7 +#define SQL_DIAG_CLASS_ORIGIN 8 +#define SQL_DIAG_SUBCLASS_ORIGIN 9 +#define SQL_DIAG_CONNECTION_NAME 10 +#define SQL_DIAG_SERVER_NAME 11 +#define SQL_DIAG_DYNAMIC_FUNCTION_CODE 12 +#endif + +/* dynamic function codes */ +#if (ODBCVER >= 0x0300) +#define SQL_DIAG_ALTER_TABLE 4 +#define SQL_DIAG_CREATE_INDEX (-1) +#define SQL_DIAG_CREATE_TABLE 77 +#define SQL_DIAG_CREATE_VIEW 84 +#define SQL_DIAG_DELETE_WHERE 19 +#define SQL_DIAG_DROP_INDEX (-2) +#define SQL_DIAG_DROP_TABLE 32 +#define SQL_DIAG_DROP_VIEW 36 +#define SQL_DIAG_DYNAMIC_DELETE_CURSOR 38 +#define SQL_DIAG_DYNAMIC_UPDATE_CURSOR 81 +#define SQL_DIAG_GRANT 48 +#define SQL_DIAG_INSERT 50 +#define SQL_DIAG_REVOKE 59 +#define SQL_DIAG_SELECT_CURSOR 85 +#define SQL_DIAG_UNKNOWN_STATEMENT 0 +#define SQL_DIAG_UPDATE_WHERE 82 +#endif /* ODBCVER >= 0x0300 */ + +/* SQL data type codes */ +#define SQL_UNKNOWN_TYPE 0 +#define SQL_CHAR 1 +#define SQL_NUMERIC 2 +#define SQL_DECIMAL 3 +#define SQL_INTEGER 4 +#define SQL_SMALLINT 5 +#define SQL_FLOAT 6 +#define SQL_REAL 7 +#define SQL_DOUBLE 8 +#if (ODBCVER >= 0x0300) +#define SQL_DATETIME 9 +#endif +#define SQL_VARCHAR 12 + +/* One-parameter shortcuts for date/time data types */ +#if (ODBCVER >= 0x0300) +#define SQL_TYPE_DATE 91 +#define SQL_TYPE_TIME 92 +#define SQL_TYPE_TIMESTAMP 93 +#endif + +/* Statement attribute values for cursor sensitivity */ +#if (ODBCVER >= 0x0300) +#define SQL_UNSPECIFIED 0 +#define SQL_INSENSITIVE 1 +#define SQL_SENSITIVE 2 +#endif + +/* GetTypeInfo() request for all data types */ +#define SQL_ALL_TYPES 0 + +/* Default conversion code for SQLBindCol(), SQLBindParam() and SQLGetData() */ +#if (ODBCVER >= 0x0300) +#define SQL_DEFAULT 99 +#endif + +/* SQLGetData() code indicating that the application row descriptor + * specifies the data type + */ +#if (ODBCVER >= 0x0300) +#define SQL_ARD_TYPE (-99) +#endif + +/* SQL date/time type subcodes */ +#if (ODBCVER >= 0x0300) +#define SQL_CODE_DATE 1 +#define SQL_CODE_TIME 2 +#define SQL_CODE_TIMESTAMP 3 +#endif + +/* CLI option values */ +#if (ODBCVER >= 0x0300) +#define SQL_FALSE 0 +#define SQL_TRUE 1 +#endif + +/* values of NULLABLE field in descriptor */ +#define SQL_NO_NULLS 0 +#define SQL_NULLABLE 1 + +/* Value returned by SQLGetTypeInfo() to denote that it is + * not known whether or not a data type supports null values. + */ +#define SQL_NULLABLE_UNKNOWN 2 + +/* Values returned by SQLGetTypeInfo() to show WHERE clause + * supported + */ +#if (ODBCVER >= 0x0300) +#define SQL_PRED_NONE 0 +#define SQL_PRED_CHAR 1 +#define SQL_PRED_BASIC 2 +#endif + +/* values of UNNAMED field in descriptor */ +#if (ODBCVER >= 0x0300) +#define SQL_NAMED 0 +#define SQL_UNNAMED 1 +#endif + +/* values of ALLOC_TYPE field in descriptor */ +#if (ODBCVER >= 0x0300) +#define SQL_DESC_ALLOC_AUTO 1 +#define SQL_DESC_ALLOC_USER 2 +#endif + +/* FreeStmt() options */ +#define SQL_CLOSE 0 +#define SQL_DROP 1 +#define SQL_UNBIND 2 +#define SQL_RESET_PARAMS 3 + +/* Codes used for FetchOrientation in SQLFetchScroll(), + and in SQLDataSources() +*/ +#define SQL_FETCH_NEXT 1 +#define SQL_FETCH_FIRST 2 + +/* Other codes used for FetchOrientation in SQLFetchScroll() */ +#define SQL_FETCH_LAST 3 +#define SQL_FETCH_PRIOR 4 +#define SQL_FETCH_ABSOLUTE 5 +#define SQL_FETCH_RELATIVE 6 + +/* SQLEndTran() options */ +#define SQL_COMMIT 0 +#define SQL_ROLLBACK 1 + +/* null handles returned by SQLAllocHandle() */ +#define SQL_NULL_HENV 0 +#define SQL_NULL_HDBC 0 +#define SQL_NULL_HSTMT 0 +#if (ODBCVER >= 0x0300) +#define SQL_NULL_HDESC 0 +#endif + +/* null handle used in place of parent handle when allocating HENV */ +#if (ODBCVER >= 0x0300) +#define SQL_NULL_HANDLE 0L +#endif + +/* Values that may appear in the result set of SQLSpecialColumns() */ +#define SQL_SCOPE_CURROW 0 +#define SQL_SCOPE_TRANSACTION 1 +#define SQL_SCOPE_SESSION 2 + +#define SQL_PC_UNKNOWN 0 +#if (ODBCVER >= 0x0300) +#define SQL_PC_NON_PSEUDO 1 +#endif +#define SQL_PC_PSEUDO 2 + +/* Reserved value for the IdentifierType argument of SQLSpecialColumns() */ +#if (ODBCVER >= 0x0300) +#define SQL_ROW_IDENTIFIER 1 +#endif + +/* Reserved values for UNIQUE argument of SQLStatistics() */ +#define SQL_INDEX_UNIQUE 0 +#define SQL_INDEX_ALL 1 + +/* Values that may appear in the result set of SQLStatistics() */ +#define SQL_INDEX_CLUSTERED 1 +#define SQL_INDEX_HASHED 2 +#define SQL_INDEX_OTHER 3 + +/* SQLGetFunctions() values to identify ODBC APIs */ +#define SQL_API_SQLALLOCCONNECT 1 +#define SQL_API_SQLALLOCENV 2 +#if (ODBCVER >= 0x0300) +#define SQL_API_SQLALLOCHANDLE 1001 +#endif +#define SQL_API_SQLALLOCSTMT 3 +#define SQL_API_SQLBINDCOL 4 +#if (ODBCVER >= 0x0300) +#define SQL_API_SQLBINDPARAM 1002 +#endif +#define SQL_API_SQLCANCEL 5 +#if (ODBCVER >= 0x0300) +#define SQL_API_SQLCLOSECURSOR 1003 +#define SQL_API_SQLCOLATTRIBUTE 6 +#endif +#define SQL_API_SQLCOLUMNS 40 +#define SQL_API_SQLCONNECT 7 +#if (ODBCVER >= 0x0300) +#define SQL_API_SQLCOPYDESC 1004 +#endif +#define SQL_API_SQLDATASOURCES 57 +#define SQL_API_SQLDESCRIBECOL 8 +#define SQL_API_SQLDISCONNECT 9 +#if (ODBCVER >= 0x0300) +#define SQL_API_SQLENDTRAN 1005 +#endif +#define SQL_API_SQLERROR 10 +#define SQL_API_SQLEXECDIRECT 11 +#define SQL_API_SQLEXECUTE 12 +#define SQL_API_SQLFETCH 13 +#if (ODBCVER >= 0x0300) +#define SQL_API_SQLFETCHSCROLL 1021 +#endif +#define SQL_API_SQLFREECONNECT 14 +#define SQL_API_SQLFREEENV 15 +#if (ODBCVER >= 0x0300) +#define SQL_API_SQLFREEHANDLE 1006 +#endif +#define SQL_API_SQLFREESTMT 16 +#if (ODBCVER >= 0x0300) +#define SQL_API_SQLGETCONNECTATTR 1007 +#endif +#define SQL_API_SQLGETCONNECTOPTION 42 +#define SQL_API_SQLGETCURSORNAME 17 +#define SQL_API_SQLGETDATA 43 +#if (ODBCVER >= 0x0300) +#define SQL_API_SQLGETDESCFIELD 1008 +#define SQL_API_SQLGETDESCREC 1009 +#define SQL_API_SQLGETDIAGFIELD 1010 +#define SQL_API_SQLGETDIAGREC 1011 +#define SQL_API_SQLGETENVATTR 1012 +#endif +#define SQL_API_SQLGETFUNCTIONS 44 +#define SQL_API_SQLGETINFO 45 +#if (ODBCVER >= 0x0300) +#define SQL_API_SQLGETSTMTATTR 1014 +#endif +#define SQL_API_SQLGETSTMTOPTION 46 +#define SQL_API_SQLGETTYPEINFO 47 +#define SQL_API_SQLNUMRESULTCOLS 18 +#define SQL_API_SQLPARAMDATA 48 +#define SQL_API_SQLPREPARE 19 +#define SQL_API_SQLPUTDATA 49 +#define SQL_API_SQLROWCOUNT 20 +#if (ODBCVER >= 0x0300) +#define SQL_API_SQLSETCONNECTATTR 1016 +#endif +#define SQL_API_SQLSETCONNECTOPTION 50 +#define SQL_API_SQLSETCURSORNAME 21 +#if (ODBCVER >= 0x0300) +#define SQL_API_SQLSETDESCFIELD 1017 +#define SQL_API_SQLSETDESCREC 1018 +#define SQL_API_SQLSETENVATTR 1019 +#endif +#define SQL_API_SQLSETPARAM 22 +#if (ODBCVER >= 0x0300) +#define SQL_API_SQLSETSTMTATTR 1020 +#endif +#define SQL_API_SQLSETSTMTOPTION 51 +#define SQL_API_SQLSPECIALCOLUMNS 52 +#define SQL_API_SQLSTATISTICS 53 +#define SQL_API_SQLTABLES 54 +#define SQL_API_SQLTRANSACT 23 + +/* Information requested by SQLGetInfo() */ +#if (ODBCVER >= 0x0300) +#define SQL_MAX_DRIVER_CONNECTIONS 0 +#define SQL_MAXIMUM_DRIVER_CONNECTIONS SQL_MAX_DRIVER_CONNECTIONS +#define SQL_MAX_CONCURRENT_ACTIVITIES 1 +#define SQL_MAXIMUM_CONCURRENT_ACTIVITIES SQL_MAX_CONCURRENT_ACTIVITIES +#endif +#define SQL_DATA_SOURCE_NAME 2 +#define SQL_FETCH_DIRECTION 8 +#define SQL_SERVER_NAME 13 +#define SQL_SEARCH_PATTERN_ESCAPE 14 +#define SQL_DBMS_NAME 17 +#define SQL_DBMS_VER 18 +#define SQL_ACCESSIBLE_TABLES 19 +#define SQL_ACCESSIBLE_PROCEDURES 20 +#define SQL_CURSOR_COMMIT_BEHAVIOR 23 +#define SQL_DATA_SOURCE_READ_ONLY 25 +#define SQL_DEFAULT_TXN_ISOLATION 26 +#define SQL_IDENTIFIER_CASE 28 +#define SQL_IDENTIFIER_QUOTE_CHAR 29 +#define SQL_MAX_COLUMN_NAME_LEN 30 +#define SQL_MAXIMUM_COLUMN_NAME_LENGTH SQL_MAX_COLUMN_NAME_LEN +#define SQL_MAX_CURSOR_NAME_LEN 31 +#define SQL_MAXIMUM_CURSOR_NAME_LENGTH SQL_MAX_CURSOR_NAME_LEN +#define SQL_MAX_SCHEMA_NAME_LEN 32 +#define SQL_MAXIMUM_SCHEMA_NAME_LENGTH SQL_MAX_SCHEMA_NAME_LEN +#define SQL_MAX_CATALOG_NAME_LEN 34 +#define SQL_MAXIMUM_CATALOG_NAME_LENGTH SQL_MAX_CATALOG_NAME_LEN +#define SQL_MAX_TABLE_NAME_LEN 35 +#define SQL_SCROLL_CONCURRENCY 43 +#define SQL_TXN_CAPABLE 46 +#define SQL_TRANSACTION_CAPABLE SQL_TXN_CAPABLE +#define SQL_USER_NAME 47 +#define SQL_TXN_ISOLATION_OPTION 72 +#define SQL_TRANSACTION_ISOLATION_OPTION SQL_TXN_ISOLATION_OPTION +#define SQL_INTEGRITY 73 +#define SQL_GETDATA_EXTENSIONS 81 +#define SQL_NULL_COLLATION 85 +#define SQL_ALTER_TABLE 86 +#define SQL_ORDER_BY_COLUMNS_IN_SELECT 90 +#define SQL_SPECIAL_CHARACTERS 94 +#define SQL_MAX_COLUMNS_IN_GROUP_BY 97 +#define SQL_MAXIMUM_COLUMNS_IN_GROUP_BY SQL_MAX_COLUMNS_IN_GROUP_BY +#define SQL_MAX_COLUMNS_IN_INDEX 98 +#define SQL_MAXIMUM_COLUMNS_IN_INDEX SQL_MAX_COLUMNS_IN_INDEX +#define SQL_MAX_COLUMNS_IN_ORDER_BY 99 +#define SQL_MAXIMUM_COLUMNS_IN_ORDER_BY SQL_MAX_COLUMNS_IN_ORDER_BY +#define SQL_MAX_COLUMNS_IN_SELECT 100 +#define SQL_MAXIMUM_COLUMNS_IN_SELECT SQL_MAX_COLUMNS_IN_SELECT +#define SQL_MAX_COLUMNS_IN_TABLE 101 +#define SQL_MAX_INDEX_SIZE 102 +#define SQL_MAXIMUM_INDEX_SIZE SQL_MAX_INDEX_SIZE +#define SQL_MAX_ROW_SIZE 104 +#define SQL_MAXIMUM_ROW_SIZE SQL_MAX_ROW_SIZE +#define SQL_MAX_STATEMENT_LEN 105 +#define SQL_MAXIMUM_STATEMENT_LENGTH SQL_MAX_STATEMENT_LEN +#define SQL_MAX_TABLES_IN_SELECT 106 +#define SQL_MAXIMUM_TABLES_IN_SELECT SQL_MAX_TABLES_IN_SELECT +#define SQL_MAX_USER_NAME_LEN 107 +#define SQL_MAXIMUM_USER_NAME_LENGTH SQL_MAX_USER_NAME_LEN +#if (ODBCVER >= 0x0300) +#define SQL_OJ_CAPABILITIES 115 +#define SQL_OUTER_JOIN_CAPABILITIES SQL_OJ_CAPABILITIES +#endif /* ODBCVER >= 0x0300 */ + +#if (ODBCVER >= 0x0300) +#define SQL_XOPEN_CLI_YEAR 10000 +#define SQL_CURSOR_SENSITIVITY 10001 +#define SQL_DESCRIBE_PARAMETER 10002 +#define SQL_CATALOG_NAME 10003 +#define SQL_COLLATION_SEQ 10004 +#define SQL_MAX_IDENTIFIER_LEN 10005 +#define SQL_MAXIMUM_IDENTIFIER_LENGTH SQL_MAX_IDENTIFIER_LEN +#endif /* ODBCVER >= 0x0300 */ + +/* SQL_ALTER_TABLE bitmasks */ +#if (ODBCVER >= 0x0200) +#define SQL_AT_ADD_COLUMN 0x00000001L +#define SQL_AT_DROP_COLUMN 0x00000002L +#endif /* ODBCVER >= 0x0200 */ + +#if (ODBCVER >= 0x0300) +#define SQL_AT_ADD_CONSTRAINT 0x00000008L + +/* The following bitmasks are ODBC extensions and defined in sqlext.h +*#define SQL_AT_COLUMN_SINGLE 0x00000020L +*#define SQL_AT_ADD_COLUMN_DEFAULT 0x00000040L +*#define SQL_AT_ADD_COLUMN_COLLATION 0x00000080L +*#define SQL_AT_SET_COLUMN_DEFAULT 0x00000100L +*#define SQL_AT_DROP_COLUMN_DEFAULT 0x00000200L +*#define SQL_AT_DROP_COLUMN_CASCADE 0x00000400L +*#define SQL_AT_DROP_COLUMN_RESTRICT 0x00000800L +*#define SQL_AT_ADD_TABLE_CONSTRAINT 0x00001000L +*#define SQL_AT_DROP_TABLE_CONSTRAINT_CASCADE 0x00002000L +*#define SQL_AT_DROP_TABLE_CONSTRAINT_RESTRICT 0x00004000L +*#define SQL_AT_CONSTRAINT_NAME_DEFINITION 0x00008000L +*#define SQL_AT_CONSTRAINT_INITIALLY_DEFERRED 0x00010000L +*#define SQL_AT_CONSTRAINT_INITIALLY_IMMEDIATE 0x00020000L +*#define SQL_AT_CONSTRAINT_DEFERRABLE 0x00040000L +*#define SQL_AT_CONSTRAINT_NON_DEFERRABLE 0x00080000L +*/ +#endif /* ODBCVER >= 0x0300 */ + + +/* SQL_ASYNC_MODE values */ +#if (ODBCVER >= 0x0300) +#define SQL_AM_NONE 0 +#define SQL_AM_CONNECTION 1 +#define SQL_AM_STATEMENT 2 +#endif + +/* SQL_CURSOR_COMMIT_BEHAVIOR values */ +#define SQL_CB_DELETE 0 +#define SQL_CB_CLOSE 1 +#define SQL_CB_PRESERVE 2 + +/* SQL_FETCH_DIRECTION bitmasks */ +#define SQL_FD_FETCH_NEXT 0x00000001L +#define SQL_FD_FETCH_FIRST 0x00000002L +#define SQL_FD_FETCH_LAST 0x00000004L +#define SQL_FD_FETCH_PRIOR 0x00000008L +#define SQL_FD_FETCH_ABSOLUTE 0x00000010L +#define SQL_FD_FETCH_RELATIVE 0x00000020L + +/* SQL_GETDATA_EXTENSIONS bitmasks */ +#define SQL_GD_ANY_COLUMN 0x00000001L +#define SQL_GD_ANY_ORDER 0x00000002L + +/* SQL_IDENTIFIER_CASE values */ +#define SQL_IC_UPPER 1 +#define SQL_IC_LOWER 2 +#define SQL_IC_SENSITIVE 3 +#define SQL_IC_MIXED 4 + +/* SQL_OJ_CAPABILITIES bitmasks */ +/* NB: this means 'outer join', not what you may be thinking */ + + +#if (ODBCVER >= 0x0201) +#define SQL_OJ_LEFT 0x00000001L +#define SQL_OJ_RIGHT 0x00000002L +#define SQL_OJ_FULL 0x00000004L +#define SQL_OJ_NESTED 0x00000008L +#define SQL_OJ_NOT_ORDERED 0x00000010L +#define SQL_OJ_INNER 0x00000020L +#define SQL_OJ_ALL_COMPARISON_OPS 0x00000040L +#endif + +/* SQL_SCROLL_CONCURRENCY bitmasks */ +#define SQL_SCCO_READ_ONLY 0x00000001L +#define SQL_SCCO_LOCK 0x00000002L +#define SQL_SCCO_OPT_ROWVER 0x00000004L +#define SQL_SCCO_OPT_VALUES 0x00000008L + +/* SQL_TXN_CAPABLE values */ +#define SQL_TC_NONE 0 +#define SQL_TC_DML 1 +#define SQL_TC_ALL 2 +#define SQL_TC_DDL_COMMIT 3 +#define SQL_TC_DDL_IGNORE 4 + +/* SQL_TXN_ISOLATION_OPTION bitmasks */ +#define SQL_TXN_READ_UNCOMMITTED 0x00000001L +#define SQL_TRANSACTION_READ_UNCOMMITTED SQL_TXN_READ_UNCOMMITTED +#define SQL_TXN_READ_COMMITTED 0x00000002L +#define SQL_TRANSACTION_READ_COMMITTED SQL_TXN_READ_COMMITTED +#define SQL_TXN_REPEATABLE_READ 0x00000004L +#define SQL_TRANSACTION_REPEATABLE_READ SQL_TXN_REPEATABLE_READ +#define SQL_TXN_SERIALIZABLE 0x00000008L +#define SQL_TRANSACTION_SERIALIZABLE SQL_TXN_SERIALIZABLE + +/* SQL_NULL_COLLATION values */ +#define SQL_NC_HIGH 0 +#define SQL_NC_LOW 1 + +#ifndef RC_INVOKED + +SQLRETURN SQL_API SQLAllocConnect(SQLHENV EnvironmentHandle, + SQLHDBC *ConnectionHandle); + +SQLRETURN SQL_API SQLAllocEnv(SQLHENV *EnvironmentHandle); + +#if (ODBCVER >= 0x0300) +SQLRETURN SQL_API SQLAllocHandle(SQLSMALLINT HandleType, + SQLHANDLE InputHandle, SQLHANDLE *OutputHandle); +#endif + +SQLRETURN SQL_API SQLAllocStmt(SQLHDBC ConnectionHandle, + SQLHSTMT *StatementHandle); + +SQLRETURN SQL_API SQLBindCol(SQLHSTMT StatementHandle, + SQLUSMALLINT ColumnNumber, SQLSMALLINT TargetType, + SQLPOINTER TargetValue, SQLINTEGER BufferLength, + SQLINTEGER *StrLen_or_Ind); + +#if (ODBCVER >= 0x0300) +SQLRETURN SQL_API SQLBindParam(SQLHSTMT StatementHandle, + SQLUSMALLINT ParameterNumber, SQLSMALLINT ValueType, + SQLSMALLINT ParameterType, SQLUINTEGER LengthPrecision, + SQLSMALLINT ParameterScale, SQLPOINTER ParameterValue, + SQLINTEGER *StrLen_or_Ind); +#endif + +SQLRETURN SQL_API SQLCancel(SQLHSTMT StatementHandle); + +#if (ODBCVER >= 0x0300) +SQLRETURN SQL_API SQLCloseCursor(SQLHSTMT StatementHandle); + +SQLRETURN SQL_API SQLColAttribute (SQLHSTMT StatementHandle, + SQLUSMALLINT ColumnNumber, SQLUSMALLINT FieldIdentifier, + SQLPOINTER CharacterAttribute, SQLSMALLINT BufferLength, + SQLSMALLINT *StringLength, SQLPOINTER NumericAttribute); +#endif + + +SQLRETURN SQL_API SQLColumns(SQLHSTMT StatementHandle, + SQLCHAR *CatalogName, SQLSMALLINT NameLength1, + SQLCHAR *SchemaName, SQLSMALLINT NameLength2, + SQLCHAR *TableName, SQLSMALLINT NameLength3, + SQLCHAR *ColumnName, SQLSMALLINT NameLength4); + + +SQLRETURN SQL_API SQLConnect(SQLHDBC ConnectionHandle, + SQLCHAR *ServerName, SQLSMALLINT NameLength1, + SQLCHAR *UserName, SQLSMALLINT NameLength2, + SQLCHAR *Authentication, SQLSMALLINT NameLength3); + +#if (ODBCVER >= 0x0300) +SQLRETURN SQL_API SQLCopyDesc(SQLHDESC SourceDescHandle, + SQLHDESC TargetDescHandle); +#endif + +SQLRETURN SQL_API SQLDataSources(SQLHENV EnvironmentHandle, + SQLUSMALLINT Direction, SQLCHAR *ServerName, + SQLSMALLINT BufferLength1, SQLSMALLINT *NameLength1, + SQLCHAR *Description, SQLSMALLINT BufferLength2, + SQLSMALLINT *NameLength2); + +SQLRETURN SQL_API SQLDescribeCol(SQLHSTMT StatementHandle, + SQLUSMALLINT ColumnNumber, SQLCHAR *ColumnName, + SQLSMALLINT BufferLength, SQLSMALLINT *NameLength, + SQLSMALLINT *DataType, SQLUINTEGER *ColumnSize, + SQLSMALLINT *DecimalDigits, SQLSMALLINT *Nullable); + +SQLRETURN SQL_API SQLDisconnect(SQLHDBC ConnectionHandle); + +#if (ODBCVER >= 0x0300) +SQLRETURN SQL_API SQLEndTran(SQLSMALLINT HandleType, SQLHANDLE Handle, + SQLSMALLINT CompletionType); +#endif + +SQLRETURN SQL_API SQLError(SQLHENV EnvironmentHandle, + SQLHDBC ConnectionHandle, SQLHSTMT StatementHandle, + SQLCHAR *Sqlstate, SQLINTEGER *NativeError, + SQLCHAR *MessageText, SQLSMALLINT BufferLength, + SQLSMALLINT *TextLength); + +SQLRETURN SQL_API SQLExecDirect(SQLHSTMT StatementHandle, + SQLCHAR *StatementText, SQLINTEGER TextLength); + +SQLRETURN SQL_API SQLExecute(SQLHSTMT StatementHandle); + +SQLRETURN SQL_API SQLFetch(SQLHSTMT StatementHandle); + +#if (ODBCVER >= 0x0300) +SQLRETURN SQL_API SQLFetchScroll(SQLHSTMT StatementHandle, + SQLSMALLINT FetchOrientation, SQLINTEGER FetchOffset); +#endif + +SQLRETURN SQL_API SQLFreeConnect(SQLHDBC ConnectionHandle); + +SQLRETURN SQL_API SQLFreeEnv(SQLHENV EnvironmentHandle); + +#if (ODBCVER >= 0x0300) +SQLRETURN SQL_API SQLFreeHandle(SQLSMALLINT HandleType, SQLHANDLE Handle); +#endif + +SQLRETURN SQL_API SQLFreeStmt(SQLHSTMT StatementHandle, + SQLUSMALLINT Option); + +#if (ODBCVER >= 0x0300) +SQLRETURN SQL_API SQLGetConnectAttr(SQLHDBC ConnectionHandle, + SQLINTEGER Attribute, SQLPOINTER Value, + SQLINTEGER BufferLength, SQLINTEGER *StringLength); +#endif + +SQLRETURN SQL_API SQLGetConnectOption(SQLHDBC ConnectionHandle, + SQLUSMALLINT Option, SQLPOINTER Value); + +SQLRETURN SQL_API SQLGetCursorName(SQLHSTMT StatementHandle, + SQLCHAR *CursorName, SQLSMALLINT BufferLength, + SQLSMALLINT *NameLength); + +SQLRETURN SQL_API SQLGetData(SQLHSTMT StatementHandle, + SQLUSMALLINT ColumnNumber, SQLSMALLINT TargetType, + SQLPOINTER TargetValue, SQLINTEGER BufferLength, + SQLINTEGER *StrLen_or_Ind); + +#if (ODBCVER >= 0x0300) +SQLRETURN SQL_API SQLGetDescField(SQLHDESC DescriptorHandle, + SQLSMALLINT RecNumber, SQLSMALLINT FieldIdentifier, + SQLPOINTER Value, SQLINTEGER BufferLength, + SQLINTEGER *StringLength); + +SQLRETURN SQL_API SQLGetDescRec(SQLHDESC DescriptorHandle, + SQLSMALLINT RecNumber, SQLCHAR *Name, + SQLSMALLINT BufferLength, SQLSMALLINT *StringLength, + SQLSMALLINT *Type, SQLSMALLINT *SubType, + SQLINTEGER *Length, SQLSMALLINT *Precision, + SQLSMALLINT *Scale, SQLSMALLINT *Nullable); + +SQLRETURN SQL_API SQLGetDiagField(SQLSMALLINT HandleType, SQLHANDLE Handle, + SQLSMALLINT RecNumber, SQLSMALLINT DiagIdentifier, + SQLPOINTER DiagInfo, SQLSMALLINT BufferLength, + SQLSMALLINT *StringLength); + +SQLRETURN SQL_API SQLGetDiagRec(SQLSMALLINT HandleType, SQLHANDLE Handle, + SQLSMALLINT RecNumber, SQLCHAR *Sqlstate, + SQLINTEGER *NativeError, SQLCHAR *MessageText, + SQLSMALLINT BufferLength, SQLSMALLINT *TextLength); + +SQLRETURN SQL_API SQLGetEnvAttr(SQLHENV EnvironmentHandle, + SQLINTEGER Attribute, SQLPOINTER Value, + SQLINTEGER BufferLength, SQLINTEGER *StringLength); +#endif /* ODBCVER >= 0x0300 */ + +SQLRETURN SQL_API SQLGetFunctions(SQLHDBC ConnectionHandle, + SQLUSMALLINT FunctionId, SQLUSMALLINT *Supported); + +SQLRETURN SQL_API SQLGetInfo(SQLHDBC ConnectionHandle, + SQLUSMALLINT InfoType, SQLPOINTER InfoValue, + SQLSMALLINT BufferLength, SQLSMALLINT *StringLength); + +#if (ODBCVER >= 0x0300) +SQLRETURN SQL_API SQLGetStmtAttr(SQLHSTMT StatementHandle, + SQLINTEGER Attribute, SQLPOINTER Value, + SQLINTEGER BufferLength, SQLINTEGER *StringLength); +#endif /* ODBCVER >= 0x0300 */ + +SQLRETURN SQL_API SQLGetStmtOption(SQLHSTMT StatementHandle, + SQLUSMALLINT Option, SQLPOINTER Value); + +SQLRETURN SQL_API SQLGetTypeInfo(SQLHSTMT StatementHandle, + SQLSMALLINT DataType); + +SQLRETURN SQL_API SQLNumResultCols(SQLHSTMT StatementHandle, + SQLSMALLINT *ColumnCount); + +SQLRETURN SQL_API SQLParamData(SQLHSTMT StatementHandle, + SQLPOINTER *Value); + +SQLRETURN SQL_API SQLPrepare(SQLHSTMT StatementHandle, + SQLCHAR *StatementText, SQLINTEGER TextLength); + +SQLRETURN SQL_API SQLPutData(SQLHSTMT StatementHandle, + SQLPOINTER Data, SQLINTEGER StrLen_or_Ind); + +SQLRETURN SQL_API SQLRowCount(SQLHSTMT StatementHandle, + SQLINTEGER *RowCount); + +#if (ODBCVER >= 0x0300) +SQLRETURN SQL_API SQLSetConnectAttr(SQLHDBC ConnectionHandle, + SQLINTEGER Attribute, SQLPOINTER Value, + SQLINTEGER StringLength); +#endif /* ODBCVER >= 0x0300 */ + +SQLRETURN SQL_API SQLSetConnectOption(SQLHDBC ConnectionHandle, + SQLUSMALLINT Option, SQLUINTEGER Value); + +SQLRETURN SQL_API SQLSetCursorName(SQLHSTMT StatementHandle, + SQLCHAR *CursorName, SQLSMALLINT NameLength); + +#if (ODBCVER >= 0x0300) +SQLRETURN SQL_API SQLSetDescField(SQLHDESC DescriptorHandle, + SQLSMALLINT RecNumber, SQLSMALLINT FieldIdentifier, + SQLPOINTER Value, SQLINTEGER BufferLength); + +SQLRETURN SQL_API SQLSetDescRec(SQLHDESC DescriptorHandle, + SQLSMALLINT RecNumber, SQLSMALLINT Type, + SQLSMALLINT SubType, SQLINTEGER Length, + SQLSMALLINT Precision, SQLSMALLINT Scale, + SQLPOINTER Data, SQLINTEGER *StringLength, + SQLINTEGER *Indicator); + +SQLRETURN SQL_API SQLSetEnvAttr(SQLHENV EnvironmentHandle, + SQLINTEGER Attribute, SQLPOINTER Value, + SQLINTEGER StringLength); +#endif /* ODBCVER >= 0x0300 */ + +SQLRETURN SQL_API SQLSetParam(SQLHSTMT StatementHandle, + SQLUSMALLINT ParameterNumber, SQLSMALLINT ValueType, + SQLSMALLINT ParameterType, SQLUINTEGER LengthPrecision, + SQLSMALLINT ParameterScale, SQLPOINTER ParameterValue, + SQLINTEGER *StrLen_or_Ind); + +#if (ODBCVER >= 0x0300) +SQLRETURN SQL_API SQLSetStmtAttr(SQLHSTMT StatementHandle, + SQLINTEGER Attribute, SQLPOINTER Value, + SQLINTEGER StringLength); +#endif + +SQLRETURN SQL_API SQLSetStmtOption(SQLHSTMT StatementHandle, + SQLUSMALLINT Option, SQLUINTEGER Value); + +SQLRETURN SQL_API SQLSpecialColumns(SQLHSTMT StatementHandle, + SQLUSMALLINT IdentifierType, SQLCHAR *CatalogName, + SQLSMALLINT NameLength1, SQLCHAR *SchemaName, + SQLSMALLINT NameLength2, SQLCHAR *TableName, + SQLSMALLINT NameLength3, SQLUSMALLINT Scope, + SQLUSMALLINT Nullable); + +SQLRETURN SQL_API SQLStatistics(SQLHSTMT StatementHandle, + SQLCHAR *CatalogName, SQLSMALLINT NameLength1, + SQLCHAR *SchemaName, SQLSMALLINT NameLength2, + SQLCHAR *TableName, SQLSMALLINT NameLength3, + SQLUSMALLINT Unique, SQLUSMALLINT Reserved); + +SQLRETURN SQL_API SQLTables(SQLHSTMT StatementHandle, + SQLCHAR *CatalogName, SQLSMALLINT NameLength1, + SQLCHAR *SchemaName, SQLSMALLINT NameLength2, + SQLCHAR *TableName, SQLSMALLINT NameLength3, + SQLCHAR *TableType, SQLSMALLINT NameLength4); + +SQLRETURN SQL_API SQLTransact(SQLHENV EnvironmentHandle, + SQLHDBC ConnectionHandle, SQLUSMALLINT CompletionType); + +#endif /* RC_INVOKED */ + +#ifdef __cplusplus +} /* End of extern "C" { */ +#endif /* __cplusplus */ +#endif /* #ifndef __SQL */ diff --git a/Online/sqldb/src/sqldb.cpp b/Online/sqldb/src/sqldb.cpp new file mode 100644 index 000000000..4d884e95a --- /dev/null +++ b/Online/sqldb/src/sqldb.cpp @@ -0,0 +1,327 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Package : RPC +// +// Author : Markus Frank +//========================================================================== + +/// Framework include files +#include <sqldb/sqldb.h> + +#ifndef __INLINE__ +#define __INLINE__ +#endif + +/// SQLDB namespace declaration +namespace sqldb { + + /// Call on failure: Throws exception + __INLINE__ int invalid_statement(const char* msg) { + throw std::runtime_error(std::string("Statement Error: ") + (msg ? msg : " [Unknwon Error]")); + } + + /// Access the name of a database + __INLINE__ std::string_view database::name() const { + return intern.get() ? intern->name.c_str() : ""; + } + + /// Access the name of a database + __INLINE__ std::string_view database::name(const database& dbase) { + return dbase.name(); + } + + /// Access error message on failure + __INLINE__ std::string database::errmsg() const { + return intern.get() ? intern->errmsg() : " --- no database handle --- "; + } + + /// Access error message on failure + __INLINE__ std::string database::errmsg(const database& dbase) { + return dbase.errmsg(); + } + + /// Close the connection to the database + __INLINE__ int database::close() { + if ( intern.get() ) { + intern->action(backend::CLOSE); + intern->db = nullptr; + intern.reset(); + } + return sqldb::OK; + } + + /// Start transaction + __INLINE__ int database::begin() const { + return intern->action(backend::BEGIN); + } + + /// Commit transaction + __INLINE__ int database::commit() const { + return intern->action(backend::COMMIT); + } + + /// Roll-back transaction + __INLINE__ int database::rollback() const { + return intern->action(backend::ROLLBACK); + } + + /// Access the number of rows affected by the last changes + __INLINE__ long database::rows_affected() const { + return intern->rows_affected(); + } + + /// Execute SQL statement + __INLINE__ int database::execute(std::string& error, const char* sql) const { + return intern->execute(error, sql); + } + + /// Execute SQL statement + __INLINE__ int database::execute_sql(std::string& error, const char* fmt, ...) const { + char str[4096]; + va_list args; + va_start(args, fmt); + ::vsnprintf(str, sizeof(str), fmt, args); + va_end(args); + return this->execute_sql(error, str); + } + + /// Prepare a new statement and allocate resources. + __INLINE__ int database::prepare(basic_statement& stmt, const std::string& sql) const { + return intern->prepare(stmt, sql); + } + + /// Fetch next result row. Stop if result is <> traits::OK + __INLINE__ int record_set::fetch_one() const { + if ( this->stmt ) return this->stmt->fetch_one(); + return invalid_statement("Invalid recordset statement handle"); + } + + template <> __INLINE__ blob_t record_set::get<blob_t>(size_t col) const { + auto& column = this->field(col); + return blob_t(column.item.blob, column.length); + } + + template <> __INLINE__ vblob_t record_set::get<vblob_t>(size_t col) const { + const auto& column = this->field(col); + if ( column.item.blob ) + return vblob_t(column.item.blob, column.item.blob+column.length); + return vblob_t(); + } + + template <> __INLINE__ std::string record_set::get<std::string>(size_t col) const { + const auto& column = this->field(col); + return column.item.string ? column.item.string : ""; + } + + template <> __INLINE__ signed char record_set::get<signed char>(size_t col) const { + return this->field(col).item.int8; + } + + template <> __INLINE__ char record_set::get<char>(size_t col) const { + return this->field(col).item.int8; + } + + template <> __INLINE__ unsigned char record_set::get<unsigned char>(size_t col) const { + return this->field(col).item.uint8; + } + + template <> __INLINE__ short record_set::get<short>(size_t col) const { + return this->field(col).item.int16; + } + + template <> __INLINE__ unsigned short record_set::get<unsigned short>(size_t col) const { + return this->field(col).item.uint16; + } + + template <> __INLINE__ int record_set::get<int>(size_t col) const { + return this->field(col).item.int32; + } + + template <> __INLINE__ unsigned int record_set::get<unsigned int>(size_t col) const { + return this->field(col).item.uint32; + } + + template <> __INLINE__ long record_set::get<long>(size_t col) const { + return this->field(col).item.int64; + } + + template <> __INLINE__ unsigned long record_set::get<unsigned long>(size_t col) const { + return this->field(col).item.uint64; + } + + template <> __INLINE__ float record_set::get<float>(size_t col) const { + return this->field(col).item.real32; + } + + template <> __INLINE__ double record_set::get<double>(size_t col) const { + return this->field(col).item.real64; + } + + /// Create fully prepared statement in a database context + __INLINE__ basic_statement basic_statement::prepare(const database& db, const std::string& sql) { + basic_statement stmt; + if ( db.prepare(stmt, sql) == sqldb::OK ) + return stmt; + return basic_statement(); + } + + /// Execute SQL statement with variable number of arguments + __INLINE__ basic_statement basic_statement::prepare(const database& db, const char* fmt, ...) { + char sql[4096]; + va_list args; + va_start(args, fmt); + ::vsnprintf(sql, sizeof(sql), fmt, args); + va_end(args); + basic_statement stmt; + if ( db.prepare(stmt, sql) == sqldb::OK ) + return stmt; + return basic_statement(); + } + + /// Execute prepared statement + __INLINE__ int basic_statement::execute() const { + return intern->execute(); + } + + /// Fetch next result row. Stop if result is <> traits::OK + __INLINE__ int basic_statement::fetch_one() const { + return intern->fetch_one(); + } + + /// Reset SQL statement: Reset all data to re-execute the statement with new parameters + __INLINE__ int basic_statement::reset() const { + return intern->reset(); + } + + /// Finalize and run-down the statement. Release all resources. + __INLINE__ int basic_statement::finalize() { + if ( intern.get() ) { + intern->finalize(); + intern.reset(); + } + return sqldb::OK; + } + + template <> __INLINE__ blob_t basic_statement::get<blob_t>(size_t col) const { + auto& column = this->field(col); + return blob_t(column.item.blob, column.length); + } + + template <> __INLINE__ vblob_t basic_statement::get<vblob_t>(size_t col) const { + const auto& column = this->field(col); + if ( column.item.blob ) + return vblob_t(column.item.blob, column.item.blob+column.length); + return vblob_t(); + } + + template <> __INLINE__ std::string basic_statement::get<std::string>(size_t col) const { + const auto& column = this->field(col); + return column.item.string ? column.item.string : ""; + } + + template <> __INLINE__ signed char basic_statement::get<signed char>(size_t col) const { + return this->field(col).item.int8; + } + + template <> __INLINE__ char basic_statement::get<char>(size_t col) const { + return this->field(col).item.int8; + } + + template <> __INLINE__ unsigned char basic_statement::get<unsigned char>(size_t col) const { + return this->field(col).item.uint8; + } + + template <> __INLINE__ short basic_statement::get<short>(size_t col) const { + return this->field(col).item.int16; + } + + template <> __INLINE__ unsigned short basic_statement::get<unsigned short>(size_t col) const { + return this->field(col).item.uint16; + } + + template <> __INLINE__ int basic_statement::get<int>(size_t col) const { + return this->field(col).item.int32; + } + + template <> __INLINE__ unsigned int basic_statement::get<unsigned int>(size_t col) const { + return this->field(col).item.uint32; + } + + template <> __INLINE__ long basic_statement::get<long>(size_t col) const { + return this->field(col).item.int64; + } + + template <> __INLINE__ unsigned long basic_statement::get<unsigned long>(size_t col) const { + return this->field(col).item.uint64; + } + + template <> __INLINE__ float basic_statement::get<float>(size_t col) const { + return this->field(col).item.real32; + } + + template <> __INLINE__ double basic_statement::get<double>(size_t col) const { + return this->field(col).item.real64; + } + + /// Generic bind of a single parameter column + inline int basic_statement::bind (size_t column, field_type typ, const void* data, size_t len) const + { return intern->bind(column, typ, data, len); } + + template <> __INLINE__ int basic_statement::bind<char>(size_t col, char data) const + { return intern->bind(col, TYPE_TINY, &data, sizeof(data)); } + + template <> __INLINE__ int basic_statement::bind<signed char>(size_t col, signed char data) const + { return intern->bind(col, TYPE_TINY, &data, sizeof(data)); } + + template <> __INLINE__ int basic_statement::bind<unsigned char>(size_t col, unsigned char data) const + { return intern->bind(col, TYPE_TINY, &data, sizeof(data)); } + + template <> __INLINE__ int basic_statement::bind<short>(size_t col, short data) const + { return intern->bind(col, TYPE_SHORT, &data, sizeof(data)); } + + template <> __INLINE__ int basic_statement::bind<unsigned short>(size_t col, unsigned short data) const + { return intern->bind(col, TYPE_SHORT, &data, sizeof(data)); } + + template <> __INLINE__ int basic_statement::bind<int>(size_t col, int data) const + { return intern->bind(col, TYPE_LONG, &data, sizeof(data)); } + + template <> __INLINE__ int basic_statement::bind<unsigned int>(size_t col, unsigned int data) const + { return intern->bind(col, TYPE_LONG, &data, sizeof(data)); } + + template <> __INLINE__ int basic_statement::bind<long>(size_t col, long data) const + { return intern->bind(col, TYPE_LONGLONG, &data, sizeof(data)); } + + template <> __INLINE__ int basic_statement::bind<unsigned long>(size_t col, unsigned long data) const + { return intern->bind(col, TYPE_LONGLONG, &data, sizeof(data)); } + + template <> __INLINE__ int basic_statement::bind<float>(size_t col, float data) const + { return intern->bind(col, TYPE_FLOAT, &data, sizeof(data)); } + + template <> __INLINE__ int basic_statement::bind<double>(size_t col, double data) const + { return intern->bind(col, TYPE_DOUBLE, &data, sizeof(data)); } + + template <> __INLINE__ int basic_statement::bind<char* const>(size_t col, char* const data) const + { return intern->bind(col, TYPE_STRING, data ? data : "", data ? ::strlen(data) : 0); } + + template <> __INLINE__ int basic_statement::bind<const char*>(size_t col, const char* data) const + { return intern->bind(col, TYPE_STRING, data ? data : "", data ? ::strlen(data) : 0); } + + inline int basic_statement::bind(size_t col, const blob_t& data) const + { return intern->bind(col, TYPE_BLOB, data.first, data.second); } + + inline int basic_statement::bind(size_t col, const vblob_t& data) const + { return intern->bind(col, TYPE_BLOB, &data.at(0), data.size()); } + + inline int basic_statement::bind(size_t col, const uint8_t* data, size_t len) const + { return intern->bind(col, TYPE_BLOB, data, len); } + +} /// End namespace sqldb + diff --git a/Online/sqldb/src/sqlext.h b/Online/sqldb/src/sqlext.h new file mode 100755 index 000000000..645db7a93 --- /dev/null +++ b/Online/sqldb/src/sqlext.h @@ -0,0 +1,1975 @@ +/***************************************************************** +** SQLEXT.H - This is the include for applications using +** the Microsoft SQL Extensions +** +** (C) Copyright 1990 - 1996 By Microsoft Corp. +** +** Updated 05/12/93 for 2.00 specification +** Updated 05/23/94 for 2.01 specification +** Updated 10/27/94 for 2.10 specification +** Updated 04/10/95 for 2.50 specification +** Updated 07/25/95 for 3.00 specification +** Updated 01/12/96 for 3.00 preliminary release +** Updated 09/16/96 for 3.00 SDK release +** Updated 11/21/96 for bug #4436 +*********************************************************************/ + +#ifndef __SQLEXT +#define __SQLEXT + +#ifndef __SQL +#include "sql.h" +#endif + +#ifdef __cplusplus +extern "C" { /* Assume C declarations for C++ */ +#endif /* __cplusplus */ + +/* generally useful constants */ +#define SQL_SPEC_MAJOR 3 /* Major version of specification */ +#define SQL_SPEC_MINOR 00 /* Minor version of specification */ +#define SQL_SPEC_STRING "03.00" /* String constant for version */ + +#define SQL_SQLSTATE_SIZE 5 /* size of SQLSTATE */ +#define SQL_MAX_DSN_LENGTH 32 /* maximum data source name size */ + +#define SQL_MAX_OPTION_STRING_LENGTH 256 + +/* return code SQL_NO_DATA_FOUND is the same as SQL_NO_DATA */ +#if (ODBCVER < 0x0300) +#define SQL_NO_DATA_FOUND 100 +#else +#define SQL_NO_DATA_FOUND SQL_NO_DATA +#endif + +/* an end handle type */ +#if (ODBCVER >= 0x0300) +#define SQL_HANDLE_SENV 5 +#endif /* ODBCVER >= 0x0300 */ + +/* env attribute */ +#if (ODBCVER >= 0x0300) +#define SQL_ATTR_ODBC_VERSION 200 +#define SQL_ATTR_CONNECTION_POOLING 201 +#define SQL_ATTR_CP_MATCH 202 +#endif /* ODBCVER >= 0x0300 */ + +#if (ODBCVER >= 0x0300) +/* values for SQL_ATTR_CONNECTION_POOLING */ +#define SQL_CP_OFF 0UL +#define SQL_CP_ONE_PER_DRIVER 1UL +#define SQL_CP_ONE_PER_HENV 2UL +#define SQL_CP_DEFAULT SQL_CP_OFF + +/* values for SQL_ATTR_CP_MATCH */ +#define SQL_CP_STRICT_MATCH 0UL +#define SQL_CP_RELAXED_MATCH 1UL +#define SQL_CP_MATCH_DEFAULT SQL_CP_STRICT_MATCH + +/* values for SQL_ATTR_ODBC_VERSION */ +#define SQL_OV_ODBC2 2UL +#define SQL_OV_ODBC3 3UL +#endif /* ODBCVER >= 0x0300 */ + +/* connection attributes */ +#define SQL_ACCESS_MODE 101 +#define SQL_AUTOCOMMIT 102 +#define SQL_LOGIN_TIMEOUT 103 +#define SQL_OPT_TRACE 104 +#define SQL_OPT_TRACEFILE 105 +#define SQL_TRANSLATE_DLL 106 +#define SQL_TRANSLATE_OPTION 107 +#define SQL_TXN_ISOLATION 108 +#define SQL_CURRENT_QUALIFIER 109 +#define SQL_ODBC_CURSORS 110 +#define SQL_QUIET_MODE 111 +#define SQL_PACKET_SIZE 112 + +/* connection attributes with new names */ +#if (ODBCVER >= 0x0300) +#define SQL_ATTR_ACCESS_MODE SQL_ACCESS_MODE +#define SQL_ATTR_AUTOCOMMIT SQL_AUTOCOMMIT +#define SQL_ATTR_CONNECTION_TIMEOUT 113 +#define SQL_ATTR_CURRENT_CATALOG SQL_CURRENT_QUALIFIER +#define SQL_ATTR_DISCONNECT_BEHAVIOR 114 +#define SQL_ATTR_ENLIST_IN_DTC 1207 +#define SQL_ATTR_ENLIST_IN_XA 1208 +#define SQL_ATTR_LOGIN_TIMEOUT SQL_LOGIN_TIMEOUT +#define SQL_ATTR_ODBC_CURSORS SQL_ODBC_CURSORS +#define SQL_ATTR_PACKET_SIZE SQL_PACKET_SIZE +#define SQL_ATTR_QUIET_MODE SQL_QUIET_MODE +#define SQL_ATTR_TRACE SQL_OPT_TRACE +#define SQL_ATTR_TRACEFILE SQL_OPT_TRACEFILE +#define SQL_ATTR_TRANSLATE_LIB SQL_TRANSLATE_DLL +#define SQL_ATTR_TRANSLATE_OPTION SQL_TRANSLATE_OPTION +#define SQL_ATTR_TXN_ISOLATION SQL_TXN_ISOLATION +#endif /* ODBCVER >= 0x0300 */ + +/* SQL_CONNECT_OPT_DRVR_START is not meaningful for 3.0 driver */ +#if (ODBCVER < 0x0300) +#define SQL_CONNECT_OPT_DRVR_START 1000 +#endif /* ODBCVER < 0x0300 */ + +#if (ODBCVER < 0x0300) +#define SQL_CONN_OPT_MAX SQL_PACKET_SIZE +#define SQL_CONN_OPT_MIN SQL_ACCESS_MODE +#endif /* ODBCVER < 0x0300 */ + +/* SQL_ACCESS_MODE options */ +#define SQL_MODE_READ_WRITE 0UL +#define SQL_MODE_READ_ONLY 1UL +#define SQL_MODE_DEFAULT SQL_MODE_READ_WRITE + +/* SQL_AUTOCOMMIT options */ +#define SQL_AUTOCOMMIT_OFF 0UL +#define SQL_AUTOCOMMIT_ON 1UL +#define SQL_AUTOCOMMIT_DEFAULT SQL_AUTOCOMMIT_ON + +/* SQL_LOGIN_TIMEOUT options */ +#define SQL_LOGIN_TIMEOUT_DEFAULT 15UL + +/* SQL_OPT_TRACE options */ +#define SQL_OPT_TRACE_OFF 0UL +#define SQL_OPT_TRACE_ON 1UL +#define SQL_OPT_TRACE_DEFAULT SQL_OPT_TRACE_OFF +#define SQL_OPT_TRACE_FILE_DEFAULT "\\SQL.LOG" + +/* SQL_ODBC_CURSORS options */ +#define SQL_CUR_USE_IF_NEEDED 0UL +#define SQL_CUR_USE_ODBC 1UL +#define SQL_CUR_USE_DRIVER 2UL +#define SQL_CUR_DEFAULT SQL_CUR_USE_DRIVER + +#if (ODBCVER >= 0x0300) +/* values for SQL_ATTR_DISCONNECT_BEHAVIOR */ +#define SQL_DB_RETURN_TO_POOL 0UL +#define SQL_DB_DISCONNECT 1UL +#define SQL_DB_DEFAULT SQL_DB_RETURN_TO_POOL + +/* values for SQL_ATTR_ENLIST_IN_DTC */ +#define SQL_DTC_DONE 0L +#endif /* ODBCVER >= 0x0300 */ + +/* statement attributes */ +#define SQL_QUERY_TIMEOUT 0 +#define SQL_MAX_ROWS 1 +#define SQL_NOSCAN 2 +#define SQL_MAX_LENGTH 3 +#define SQL_ASYNC_ENABLE 4 /* same as SQL_ATTR_ASYNC_ENABLE */ +#define SQL_BIND_TYPE 5 +#define SQL_CURSOR_TYPE 6 +#define SQL_CONCURRENCY 7 +#define SQL_KEYSET_SIZE 8 +#define SQL_ROWSET_SIZE 9 +#define SQL_SIMULATE_CURSOR 10 +#define SQL_RETRIEVE_DATA 11 +#define SQL_USE_BOOKMARKS 12 +#define SQL_GET_BOOKMARK 13 /* GetStmtOption Only */ +#define SQL_ROW_NUMBER 14 /* GetStmtOption Only */ + +/* statement attributes for ODBC 3.0 */ +#if (ODBCVER >= 0x0300) +#define SQL_ATTR_ASYNC_ENABLE 4 +#define SQL_ATTR_CONCURRENCY SQL_CONCURRENCY +#define SQL_ATTR_CURSOR_TYPE SQL_CURSOR_TYPE +#define SQL_ATTR_ENABLE_AUTO_IPD 15 +#define SQL_ATTR_FETCH_BOOKMARK_PTR 16 +#define SQL_ATTR_KEYSET_SIZE SQL_KEYSET_SIZE +#define SQL_ATTR_MAX_LENGTH SQL_MAX_LENGTH +#define SQL_ATTR_MAX_ROWS SQL_MAX_ROWS +#define SQL_ATTR_NOSCAN SQL_NOSCAN +#define SQL_ATTR_PARAM_BIND_OFFSET_PTR 17 +#define SQL_ATTR_PARAM_BIND_TYPE 18 +#define SQL_ATTR_PARAM_OPERATION_PTR 19 +#define SQL_ATTR_PARAM_STATUS_PTR 20 +#define SQL_ATTR_PARAMS_PROCESSED_PTR 21 +#define SQL_ATTR_PARAMSET_SIZE 22 +#define SQL_ATTR_QUERY_TIMEOUT SQL_QUERY_TIMEOUT +#define SQL_ATTR_RETRIEVE_DATA SQL_RETRIEVE_DATA +#define SQL_ATTR_ROW_BIND_OFFSET_PTR 23 +#define SQL_ATTR_ROW_BIND_TYPE SQL_BIND_TYPE +#define SQL_ATTR_ROW_NUMBER SQL_ROW_NUMBER /*GetStmtAttr*/ +#define SQL_ATTR_ROW_OPERATION_PTR 24 +#define SQL_ATTR_ROW_STATUS_PTR 25 +#define SQL_ATTR_ROWS_FETCHED_PTR 26 +#define SQL_ATTR_ROW_ARRAY_SIZE 27 +#define SQL_ATTR_SIMULATE_CURSOR SQL_SIMULATE_CURSOR +#define SQL_ATTR_USE_BOOKMARKS SQL_USE_BOOKMARKS + +#endif /* ODBCVER >= 0x0300 */ + +#if (ODBCVER < 0x0300) +#define SQL_STMT_OPT_MAX SQL_ROW_NUMBER +#define SQL_STMT_OPT_MIN SQL_QUERY_TIMEOUT +#endif /* ODBCVER < 0x0300 */ + +/* New defines for SEARCHABLE column in SQLGetTypeInfo */ + +#if (ODBCVER >= 0x0300) +#define SQL_COL_PRED_CHAR SQL_LIKE_ONLY +#define SQL_COL_PRED_BASIC SQL_ALL_EXCEPT_LIKE +#endif /* ODBCVER >= 0x0300 */ + + + +/* whether an attribute is a pointer or not */ +#if (ODBCVER >= 0x0300) +#define SQL_IS_POINTER (-4) +#define SQL_IS_UINTEGER (-5) +#define SQL_IS_INTEGER (-6) +#define SQL_IS_USMALLINT (-7) +#define SQL_IS_SMALLINT (-8) +#endif /* ODBCVER >= 0x0300 */ + +/* the value of SQL_ATTR_PARAM_BIND_TYPE */ +#if (ODBCVER >= 0x0300) +#define SQL_PARAM_BIND_BY_COLUMN 0UL +#define SQL_PARAM_BIND_TYPE_DEFAULT SQL_PARAM_BIND_BY_COLUMN +#endif /* ODBCVER >= 0x0300 */ + +/* SQL_QUERY_TIMEOUT options */ +#define SQL_QUERY_TIMEOUT_DEFAULT 0UL + +/* SQL_MAX_ROWS options */ +#define SQL_MAX_ROWS_DEFAULT 0UL + +/* SQL_NOSCAN options */ +#define SQL_NOSCAN_OFF 0UL /* 1.0 FALSE */ +#define SQL_NOSCAN_ON 1UL /* 1.0 TRUE */ +#define SQL_NOSCAN_DEFAULT SQL_NOSCAN_OFF + +/* SQL_MAX_LENGTH options */ +#define SQL_MAX_LENGTH_DEFAULT 0UL + +/* values for SQL_ATTR_ASYNC_ENABLE */ +#define SQL_ASYNC_ENABLE_OFF 0UL +#define SQL_ASYNC_ENABLE_ON 1UL +#define SQL_ASYNC_ENABLE_DEFAULT SQL_ASYNC_ENABLE_OFF + +/* SQL_BIND_TYPE options */ +#define SQL_BIND_BY_COLUMN 0UL +#define SQL_BIND_TYPE_DEFAULT SQL_BIND_BY_COLUMN /* Default value */ + +/* SQL_CONCURRENCY options */ +#define SQL_CONCUR_READ_ONLY 1 +#define SQL_CONCUR_LOCK 2 +#define SQL_CONCUR_ROWVER 3 +#define SQL_CONCUR_VALUES 4 +#define SQL_CONCUR_DEFAULT SQL_CONCUR_READ_ONLY /* Default value */ + +/* SQL_CURSOR_TYPE options */ +#define SQL_CURSOR_FORWARD_ONLY 0UL +#define SQL_CURSOR_KEYSET_DRIVEN 1UL +#define SQL_CURSOR_DYNAMIC 2UL +#define SQL_CURSOR_STATIC 3UL +#define SQL_CURSOR_TYPE_DEFAULT SQL_CURSOR_FORWARD_ONLY /* Default value */ + +/* SQL_ROWSET_SIZE options */ +#define SQL_ROWSET_SIZE_DEFAULT 1UL + +/* SQL_KEYSET_SIZE options */ +#define SQL_KEYSET_SIZE_DEFAULT 0UL + +/* SQL_SIMULATE_CURSOR options */ +#define SQL_SC_NON_UNIQUE 0UL +#define SQL_SC_TRY_UNIQUE 1UL +#define SQL_SC_UNIQUE 2UL + +/* SQL_RETRIEVE_DATA options */ +#define SQL_RD_OFF 0UL +#define SQL_RD_ON 1UL +#define SQL_RD_DEFAULT SQL_RD_ON + +/* SQL_USE_BOOKMARKS options */ +#define SQL_UB_OFF 0UL +#define SQL_UB_ON 01UL +#define SQL_UB_DEFAULT SQL_UB_OFF + +/* New values for SQL_USE_BOOKMARKS attribute */ +#if (ODBCVER >= 0x0300) +#define SQL_UB_FIXED SQL_UB_ON +#define SQL_UB_VARIABLE 2UL +#endif /* ODBCVER >= 0x0300 */ + +/* extended descriptor field */ +#if (ODBCVER >= 0x0300) +#define SQL_DESC_ARRAY_SIZE 20 +#define SQL_DESC_ARRAY_STATUS_PTR 21 +#define SQL_DESC_AUTO_UNIQUE_VALUE SQL_COLUMN_AUTO_INCREMENT +#define SQL_DESC_BASE_COLUMN_NAME 22 +#define SQL_DESC_BASE_TABLE_NAME 23 +#define SQL_DESC_BIND_OFFSET_PTR 24 +#define SQL_DESC_BIND_TYPE 25 +#define SQL_DESC_CASE_SENSITIVE SQL_COLUMN_CASE_SENSITIVE +#define SQL_DESC_CATALOG_NAME SQL_COLUMN_QUALIFIER_NAME +#define SQL_DESC_CONCISE_TYPE SQL_COLUMN_TYPE +#define SQL_DESC_DATETIME_INTERVAL_PRECISION 26 +#define SQL_DESC_DISPLAY_SIZE SQL_COLUMN_DISPLAY_SIZE +#define SQL_DESC_FIXED_PREC_SCALE SQL_COLUMN_MONEY +#define SQL_DESC_LABEL SQL_COLUMN_LABEL +#define SQL_DESC_LITERAL_PREFIX 27 +#define SQL_DESC_LITERAL_SUFFIX 28 +#define SQL_DESC_LOCAL_TYPE_NAME 29 +#define SQL_DESC_MAXIMUM_SCALE 30 +#define SQL_DESC_MINIMUM_SCALE 31 +#define SQL_DESC_NUM_PREC_RADIX 32 +#define SQL_DESC_PARAMETER_TYPE 33 +#define SQL_DESC_ROWS_PROCESSED_PTR 34 +#define SQL_DESC_SCHEMA_NAME SQL_COLUMN_OWNER_NAME +#define SQL_DESC_SEARCHABLE SQL_COLUMN_SEARCHABLE +#define SQL_DESC_TYPE_NAME SQL_COLUMN_TYPE_NAME +#define SQL_DESC_TABLE_NAME SQL_COLUMN_TABLE_NAME +#define SQL_DESC_UNSIGNED SQL_COLUMN_UNSIGNED +#define SQL_DESC_UPDATABLE SQL_COLUMN_UPDATABLE + +#endif + + +/* defines for diagnostics fields */ +#if (ODBCVER >= 0x0300) +#define SQL_DIAG_CURSOR_ROW_COUNT (-1249) +#define SQL_DIAG_ROW_NUMBER (-1248) +#define SQL_DIAG_COLUMN_NUMBER (-1247) +#endif /* ODBCVER >= 0x0300 */ + +/* dynamic function codes */ +#if (ODBCVER >= 0x0300) +#define SQL_DIAG_CALL 7 +#endif /* ODBCVER >= 0x0300 */ + +/* SQL extended datatypes */ +#define SQL_DATE 9 +#if (ODBCVER >= 0x0300) +#define SQL_INTERVAL 10 +#endif /* ODBCVER >= 0x0300 */ +#define SQL_TIME 10 +#define SQL_TIMESTAMP 11 +#define SQL_LONGVARCHAR (-1) +#define SQL_BINARY (-2) +#define SQL_VARBINARY (-3) +#define SQL_LONGVARBINARY (-4) +#define SQL_BIGINT (-5) +#define SQL_TINYINT (-6) +#define SQL_BIT (-7) + +#if (ODBCVER >= 0x0300) +/* interval code */ +#define SQL_CODE_YEAR 1 +#define SQL_CODE_MONTH 2 +#define SQL_CODE_DAY 3 +#define SQL_CODE_HOUR 4 +#define SQL_CODE_MINUTE 5 +#define SQL_CODE_SECOND 6 +#define SQL_CODE_YEAR_TO_MONTH 7 +#define SQL_CODE_DAY_TO_HOUR 8 +#define SQL_CODE_DAY_TO_MINUTE 9 +#define SQL_CODE_DAY_TO_SECOND 10 +#define SQL_CODE_HOUR_TO_MINUTE 11 +#define SQL_CODE_HOUR_TO_SECOND 12 +#define SQL_CODE_MINUTE_TO_SECOND 13 + +#define SQL_INTERVAL_YEAR (100 + SQL_CODE_YEAR) +#define SQL_INTERVAL_MONTH (100 + SQL_CODE_MONTH) +#define SQL_INTERVAL_DAY (100 + SQL_CODE_DAY) +#define SQL_INTERVAL_HOUR (100 + SQL_CODE_HOUR) +#define SQL_INTERVAL_MINUTE (100 + SQL_CODE_MINUTE) +#define SQL_INTERVAL_SECOND (100 + SQL_CODE_SECOND) +#define SQL_INTERVAL_YEAR_TO_MONTH (100 + SQL_CODE_YEAR_TO_MONTH) +#define SQL_INTERVAL_DAY_TO_HOUR (100 + SQL_CODE_DAY_TO_HOUR) +#define SQL_INTERVAL_DAY_TO_MINUTE (100 + SQL_CODE_DAY_TO_MINUTE) +#define SQL_INTERVAL_DAY_TO_SECOND (100 + SQL_CODE_DAY_TO_SECOND) +#define SQL_INTERVAL_HOUR_TO_MINUTE (100 + SQL_CODE_HOUR_TO_MINUTE) +#define SQL_INTERVAL_HOUR_TO_SECOND (100 + SQL_CODE_HOUR_TO_SECOND) +#define SQL_INTERVAL_MINUTE_TO_SECOND (100 + SQL_CODE_MINUTE_TO_SECOND) + +#else +#define SQL_INTERVAL_YEAR (-80) +#define SQL_INTERVAL_MONTH (-81) +#define SQL_INTERVAL_YEAR_TO_MONTH (-82) +#define SQL_INTERVAL_DAY (-83) +#define SQL_INTERVAL_HOUR (-84) +#define SQL_INTERVAL_MINUTE (-85) +#define SQL_INTERVAL_SECOND (-86) +#define SQL_INTERVAL_DAY_TO_HOUR (-87) +#define SQL_INTERVAL_DAY_TO_MINUTE (-88) +#define SQL_INTERVAL_DAY_TO_SECOND (-89) +#define SQL_INTERVAL_HOUR_TO_MINUTE (-90) +#define SQL_INTERVAL_HOUR_TO_SECOND (-91) +#define SQL_INTERVAL_MINUTE_TO_SECOND (-92) +#endif /* ODBCVER >= 0x0300 */ + + +#define SQL_UNICODE (-95) +#define SQL_UNICODE_VARCHAR (-96) +#define SQL_UNICODE_LONGVARCHAR (-97) +#define SQL_UNICODE_CHAR SQL_UNICODE + +#if (ODBCVER < 0x0300) +#define SQL_TYPE_DRIVER_START SQL_INTERVAL_YEAR +#define SQL_TYPE_DRIVER_END SQL_UNICODE_LONGVARCHAR +#endif /* ODBCVER < 0x0300 */ + +/* C datatype to SQL datatype mapping SQL types + ------------------- */ +#define SQL_C_CHAR SQL_CHAR /* CHAR, VARCHAR, DECIMAL, NUMERIC */ +#define SQL_C_LONG SQL_INTEGER /* INTEGER */ +#define SQL_C_SHORT SQL_SMALLINT /* SMALLINT */ +#define SQL_C_FLOAT SQL_REAL /* REAL */ +#define SQL_C_DOUBLE SQL_DOUBLE /* FLOAT, DOUBLE */ +#if (ODBCVER >= 0x0300) +#define SQL_C_NUMERIC SQL_NUMERIC +#endif /* ODBCVER >= 0x0300 */ +#define SQL_C_DEFAULT 99 + +#define SQL_SIGNED_OFFSET (-20) +#define SQL_UNSIGNED_OFFSET (-22) + +/* C datatype to SQL datatype mapping */ +#define SQL_C_DATE SQL_DATE +#define SQL_C_TIME SQL_TIME +#define SQL_C_TIMESTAMP SQL_TIMESTAMP +#if (ODBCVER >= 0x0300) +#define SQL_C_TYPE_DATE SQL_TYPE_DATE +#define SQL_C_TYPE_TIME SQL_TYPE_TIME +#define SQL_C_TYPE_TIMESTAMP SQL_TYPE_TIMESTAMP +#define SQL_C_INTERVAL_YEAR SQL_INTERVAL_YEAR +#define SQL_C_INTERVAL_MONTH SQL_INTERVAL_MONTH +#define SQL_C_INTERVAL_DAY SQL_INTERVAL_DAY +#define SQL_C_INTERVAL_HOUR SQL_INTERVAL_HOUR +#define SQL_C_INTERVAL_MINUTE SQL_INTERVAL_MINUTE +#define SQL_C_INTERVAL_SECOND SQL_INTERVAL_SECOND +#define SQL_C_INTERVAL_YEAR_TO_MONTH SQL_INTERVAL_YEAR_TO_MONTH +#define SQL_C_INTERVAL_DAY_TO_HOUR SQL_INTERVAL_DAY_TO_HOUR +#define SQL_C_INTERVAL_DAY_TO_MINUTE SQL_INTERVAL_DAY_TO_MINUTE +#define SQL_C_INTERVAL_DAY_TO_SECOND SQL_INTERVAL_DAY_TO_SECOND +#define SQL_C_INTERVAL_HOUR_TO_MINUTE SQL_INTERVAL_HOUR_TO_MINUTE +#define SQL_C_INTERVAL_HOUR_TO_SECOND SQL_INTERVAL_HOUR_TO_SECOND +#define SQL_C_INTERVAL_MINUTE_TO_SECOND SQL_INTERVAL_MINUTE_TO_SECOND +#endif /* ODBCVER >= 0x0300 */ +#define SQL_C_BINARY SQL_BINARY +#define SQL_C_BIT SQL_BIT +#if (ODBCVER >= 0x0300) +#define SQL_C_SBIGINT (SQL_BIGINT+SQL_SIGNED_OFFSET) /* SIGNED BIGINT */ +#define SQL_C_UBIGINT (SQL_BIGINT+SQL_UNSIGNED_OFFSET) /* UNSIGNED BIGINT */ +#endif /* ODBCVER >= 0x0300 */ +#define SQL_C_TINYINT SQL_TINYINT +#define SQL_C_SLONG (SQL_C_LONG+SQL_SIGNED_OFFSET) /* SIGNED INTEGER */ +#define SQL_C_SSHORT (SQL_C_SHORT+SQL_SIGNED_OFFSET) /* SIGNED SMALLINT */ +#define SQL_C_STINYINT (SQL_TINYINT+SQL_SIGNED_OFFSET) /* SIGNED TINYINT */ +#define SQL_C_ULONG (SQL_C_LONG+SQL_UNSIGNED_OFFSET) /* UNSIGNED INTEGER*/ +#define SQL_C_USHORT (SQL_C_SHORT+SQL_UNSIGNED_OFFSET) /* UNSIGNED SMALLINT*/ +#define SQL_C_UTINYINT (SQL_TINYINT+SQL_UNSIGNED_OFFSET) /* UNSIGNED TINYINT*/ +#define SQL_C_BOOKMARK SQL_C_ULONG /* BOOKMARK */ + +#define SQL_TYPE_NULL 0 +#if (ODBCVER < 0x0300) +#define SQL_TYPE_MIN SQL_BIT +#define SQL_TYPE_MAX SQL_VARCHAR +#endif + +#if (ODBCVER >= 0x0300) +#define SQL_C_VARBOOKMARK SQL_C_BINARY +#endif /* ODBCVER >= 0x0300 */ + +/* define for SQL_DIAG_ROW_NUMBER and SQL_DIAG_COLUMN_NUMBER */ +#if (ODBCVER >= 0x0300) +#define SQL_NO_ROW_NUMBER (-1) +#define SQL_NO_COLUMN_NUMBER (-1) +#define SQL_ROW_NUMBER_UNKNOWN (-2) +#define SQL_COLUMN_NUMBER_UNKNOWN (-2) +#endif + +/* SQLBindParameter extensions */ +#define SQL_DEFAULT_PARAM (-5) +#define SQL_IGNORE (-6) +#if (ODBCVER >= 0x0300) +#define SQL_COLUMN_IGNORE SQL_IGNORE +#endif /* ODBCVER >= 0x0300 */ +#define SQL_LEN_DATA_AT_EXEC_OFFSET (-100) +#define SQL_LEN_DATA_AT_EXEC(length) (-(length)+SQL_LEN_DATA_AT_EXEC_OFFSET) + +/* binary length for driver specific attributes */ +#define SQL_LEN_BINARY_ATTR_OFFSET (-100) +#define SQL_LEN_BINARY_ATTR(length) (-(length)+SQL_LEN_BINARY_ATTR_OFFSET) + +/* Defines used by Driver Manager when mapping SQLSetParam to SQLBindParameter +*/ +#define SQL_PARAM_TYPE_DEFAULT SQL_PARAM_INPUT_OUTPUT +#define SQL_SETPARAM_VALUE_MAX (-1L) + +/* SQLColAttributes defines */ +#define SQL_COLUMN_COUNT 0 +#define SQL_COLUMN_NAME 1 +#define SQL_COLUMN_TYPE 2 +#define SQL_COLUMN_LENGTH 3 +#define SQL_COLUMN_PRECISION 4 +#define SQL_COLUMN_SCALE 5 +#define SQL_COLUMN_DISPLAY_SIZE 6 +#define SQL_COLUMN_NULLABLE 7 +#define SQL_COLUMN_UNSIGNED 8 +#define SQL_COLUMN_MONEY 9 +#define SQL_COLUMN_UPDATABLE 10 +#define SQL_COLUMN_AUTO_INCREMENT 11 +#define SQL_COLUMN_CASE_SENSITIVE 12 +#define SQL_COLUMN_SEARCHABLE 13 +#define SQL_COLUMN_TYPE_NAME 14 +#define SQL_COLUMN_TABLE_NAME 15 +#define SQL_COLUMN_OWNER_NAME 16 +#define SQL_COLUMN_QUALIFIER_NAME 17 +#define SQL_COLUMN_LABEL 18 +#define SQL_COLATT_OPT_MAX SQL_COLUMN_LABEL +#if (ODBCVER < 0x0300) +#define SQL_COLUMN_DRIVER_START 1000 +#endif /* ODBCVER < 0x0300 */ + +#define SQL_COLATT_OPT_MIN SQL_COLUMN_COUNT + +/* SQLColAttributes subdefines for SQL_COLUMN_UPDATABLE */ +#define SQL_ATTR_READONLY 0 +#define SQL_ATTR_WRITE 1 +#define SQL_ATTR_READWRITE_UNKNOWN 2 + +/* SQLColAttributes subdefines for SQL_COLUMN_SEARCHABLE */ +/* These are also used by SQLGetInfo */ +#define SQL_UNSEARCHABLE 0 +#define SQL_LIKE_ONLY 1 +#define SQL_ALL_EXCEPT_LIKE 2 +#define SQL_SEARCHABLE 3 +#define SQL_PRED_SEARCHABLE SQL_SEARCHABLE + + +/* Special return values for SQLGetData */ +#define SQL_NO_TOTAL (-4) + +/********************************************/ +/* SQLGetFunctions: additional values for */ +/* fFunction to represent functions that */ +/* are not in the X/Open spec. */ +/********************************************/ + +#if (ODBCVER >= 0x0300) +#define SQL_API_SQLALLOCHANDLESTD 73 +#define SQL_API_SQLBULKOPERATIONS 24 +#endif /* ODBCVER >= 0x0300 */ +#define SQL_API_SQLBINDPARAMETER 72 +#define SQL_API_SQLBROWSECONNECT 55 +#define SQL_API_SQLCOLATTRIBUTES 6 +#define SQL_API_SQLCOLUMNPRIVILEGES 56 +#define SQL_API_SQLDESCRIBEPARAM 58 +#define SQL_API_SQLDRIVERCONNECT 41 +#define SQL_API_SQLDRIVERS 71 +#define SQL_API_SQLEXTENDEDFETCH 59 +#define SQL_API_SQLFOREIGNKEYS 60 +#define SQL_API_SQLMORERESULTS 61 +#define SQL_API_SQLNATIVESQL 62 +#define SQL_API_SQLNUMPARAMS 63 +#define SQL_API_SQLPARAMOPTIONS 64 +#define SQL_API_SQLPRIMARYKEYS 65 +#define SQL_API_SQLPROCEDURECOLUMNS 66 +#define SQL_API_SQLPROCEDURES 67 +#define SQL_API_SQLSETPOS 68 +#define SQL_API_SQLSETSCROLLOPTIONS 69 +#define SQL_API_SQLTABLEPRIVILEGES 70 + +/*-------------------------------------------*/ +/* SQL_EXT_API_LAST is not useful with ODBC */ +/* version 3.0 because some of the values */ +/* from X/Open are in the 10000 range. */ +/*-------------------------------------------*/ + +#if (ODBCVER < 0x0300) +#define SQL_EXT_API_LAST SQL_API_SQLBINDPARAMETER +#define SQL_NUM_FUNCTIONS 23 +#define SQL_EXT_API_START 40 +#define SQL_NUM_EXTENSIONS (SQL_EXT_API_LAST-SQL_EXT_API_START+1) +#endif + +/*--------------------------------------------*/ +/* SQL_API_ALL_FUNCTIONS returns an array */ +/* of 'booleans' representing whether a */ +/* function is implemented by the driver. */ +/* */ +/* CAUTION: Only functions defined in ODBC */ +/* version 2.0 and earlier are returned, the */ +/* new high-range function numbers defined by */ +/* X/Open break this scheme. See the new */ +/* method -- SQL_API_ODBC3_ALL_FUNCTIONS */ +/*--------------------------------------------*/ + +#define SQL_API_ALL_FUNCTIONS 0 /* See CAUTION above */ + +/*----------------------------------------------*/ +/* 2.X drivers export a dummy function with */ +/* ordinal number SQL_API_LOADBYORDINAL to speed*/ +/* loading under the windows operating system. */ +/* */ +/* CAUTION: Loading by ordinal is not supported */ +/* for 3.0 and above drivers. */ +/*----------------------------------------------*/ + +#define SQL_API_LOADBYORDINAL 199 /* See CAUTION above */ + +/*----------------------------------------------*/ +/* SQL_API_ODBC3_ALL_FUNCTIONS */ +/* This returns a bitmap, which allows us to */ +/* handle the higher-valued function numbers. */ +/* Use SQL_FUNC_EXISTS(bitmap,function_number) */ +/* to determine if the function exists. */ +/*----------------------------------------------*/ + + +#if (ODBCVER >= 0x0300) +#define SQL_API_ODBC3_ALL_FUNCTIONS 999 +#define SQL_API_ODBC3_ALL_FUNCTIONS_SIZE 250 /* array of 250 words */ + +#define SQL_FUNC_EXISTS(pfExists, uwAPI) \ + ((*(((UWORD*) (pfExists)) + ((uwAPI) >> 4)) \ + & (1 << ((uwAPI) & 0x000F)) \ + ) ? SQL_TRUE : SQL_FALSE \ + ) +#endif /* ODBCVER >= 0x0300 */ + + +/************************************************/ +/* Extended definitions for SQLGetInfo */ +/************************************************/ + +/*---------------------------------*/ +/* Values in ODBC 2.0 that are not */ +/* in the X/Open spec */ +/*---------------------------------*/ + +#define SQL_INFO_FIRST 0 +#define SQL_ACTIVE_CONNECTIONS 0 /* MAX_DRIVER_CONNECTIONS */ +#define SQL_ACTIVE_STATEMENTS 1 /* MAX_CONCURRENT_ACTIVITIES */ +#define SQL_DRIVER_HDBC 3 +#define SQL_DRIVER_HENV 4 +#define SQL_DRIVER_HSTMT 5 +#define SQL_DRIVER_NAME 6 +#define SQL_DRIVER_VER 7 +#define SQL_ODBC_API_CONFORMANCE 9 +#define SQL_ODBC_VER 10 +#define SQL_ROW_UPDATES 11 +#define SQL_ODBC_SAG_CLI_CONFORMANCE 12 +#define SQL_ODBC_SQL_CONFORMANCE 15 +#define SQL_PROCEDURES 21 +#define SQL_CONCAT_NULL_BEHAVIOR 22 +#define SQL_CURSOR_ROLLBACK_BEHAVIOR 24 +#define SQL_EXPRESSIONS_IN_ORDERBY 27 +#define SQL_MAX_OWNER_NAME_LEN 32 /* MAX_SCHEMA_NAME_LEN */ +#define SQL_MAX_PROCEDURE_NAME_LEN 33 +#define SQL_MAX_QUALIFIER_NAME_LEN 34 /* MAX_CATALOG_NAME_LEN */ +#define SQL_MULT_RESULT_SETS 36 +#define SQL_MULTIPLE_ACTIVE_TXN 37 +#define SQL_OUTER_JOINS 38 +#define SQL_OWNER_TERM 39 +#define SQL_PROCEDURE_TERM 40 +#define SQL_QUALIFIER_NAME_SEPARATOR 41 +#define SQL_QUALIFIER_TERM 42 +#define SQL_SCROLL_OPTIONS 44 +#define SQL_TABLE_TERM 45 +#define SQL_CONVERT_FUNCTIONS 48 +#define SQL_NUMERIC_FUNCTIONS 49 +#define SQL_STRING_FUNCTIONS 50 +#define SQL_SYSTEM_FUNCTIONS 51 +#define SQL_TIMEDATE_FUNCTIONS 52 +#define SQL_CONVERT_BIGINT 53 +#define SQL_CONVERT_BINARY 54 +#define SQL_CONVERT_BIT 55 +#define SQL_CONVERT_CHAR 56 +#define SQL_CONVERT_DATE 57 +#define SQL_CONVERT_DECIMAL 58 +#define SQL_CONVERT_DOUBLE 59 +#define SQL_CONVERT_FLOAT 60 +#define SQL_CONVERT_INTEGER 61 +#define SQL_CONVERT_LONGVARCHAR 62 +#define SQL_CONVERT_NUMERIC 63 +#define SQL_CONVERT_REAL 64 +#define SQL_CONVERT_SMALLINT 65 +#define SQL_CONVERT_TIME 66 +#define SQL_CONVERT_TIMESTAMP 67 +#define SQL_CONVERT_TINYINT 68 +#define SQL_CONVERT_VARBINARY 69 +#define SQL_CONVERT_VARCHAR 70 +#define SQL_CONVERT_LONGVARBINARY 71 +#define SQL_ODBC_SQL_OPT_IEF 73 /* SQL_INTEGRITY */ +#define SQL_CORRELATION_NAME 74 +#define SQL_NON_NULLABLE_COLUMNS 75 +#define SQL_DRIVER_HLIB 76 +#define SQL_DRIVER_ODBC_VER 77 +#define SQL_LOCK_TYPES 78 +#define SQL_POS_OPERATIONS 79 +#define SQL_POSITIONED_STATEMENTS 80 +#define SQL_BOOKMARK_PERSISTENCE 82 +#define SQL_STATIC_SENSITIVITY 83 +#define SQL_FILE_USAGE 84 +#define SQL_COLUMN_ALIAS 87 +#define SQL_GROUP_BY 88 +#define SQL_KEYWORDS 89 +#define SQL_OWNER_USAGE 91 +#define SQL_QUALIFIER_USAGE 92 +#define SQL_QUOTED_IDENTIFIER_CASE 93 +#define SQL_SUBQUERIES 95 +#define SQL_UNION 96 +#define SQL_MAX_ROW_SIZE_INCLUDES_LONG 103 +#define SQL_MAX_CHAR_LITERAL_LEN 108 +#define SQL_TIMEDATE_ADD_INTERVALS 109 +#define SQL_TIMEDATE_DIFF_INTERVALS 110 +#define SQL_NEED_LONG_DATA_LEN 111 +#define SQL_MAX_BINARY_LITERAL_LEN 112 +#define SQL_LIKE_ESCAPE_CLAUSE 113 +#define SQL_QUALIFIER_LOCATION 114 + +#if (ODBCVER >= 0x0201 && ODBCVER < 0x0300) +#define SQL_OJ_CAPABILITIES 65003 /* Temp value until ODBC 3.0 */ +#endif /* ODBCVER >= 0x0201 && ODBCVER < 0x0300 */ + +/*----------------------------------------------*/ +/* SQL_INFO_LAST and SQL_INFO_DRIVER_START are */ +/* not useful anymore, because X/Open has */ +/* values in the 10000 range. You */ +/* must contact X/Open directly to get a range */ +/* of numbers for driver-specific values. */ +/*----------------------------------------------*/ + +#if (ODBCVER < 0x0300) +#define SQL_INFO_LAST SQL_QUALIFIER_LOCATION +#define SQL_INFO_DRIVER_START 1000 +#endif /* ODBCVER < 0x0300 */ + +/*-----------------------------------------------*/ +/* ODBC 3.0 SQLGetInfo values that are not part */ +/* of the X/Open standard at this time. X/Open */ +/* standard values are in sql.h. */ +/*-----------------------------------------------*/ + +#if (ODBCVER >= 0x0300) +#define SQL_ACTIVE_ENVIRONMENTS 116 +#define SQL_ALTER_DOMAIN 117 + +#define SQL_SQL_CONFORMANCE 118 +#define SQL_DATETIME_LITERALS 119 + +#define SQL_ASYNC_MODE 10021 /* new X/Open spec */ +#define SQL_BATCH_ROW_COUNT 120 +#define SQL_BATCH_SUPPORT 121 +#define SQL_CATALOG_LOCATION SQL_QUALIFIER_LOCATION +#define SQL_CATALOG_NAME_SEPARATOR SQL_QUALIFIER_NAME_SEPARATOR +#define SQL_CATALOG_TERM SQL_QUALIFIER_TERM +#define SQL_CATALOG_USAGE SQL_QUALIFIER_USAGE +#define SQL_CONVERT_WCHAR 122 +#define SQL_CONVERT_INTERVAL_DAY_TIME 123 +#define SQL_CONVERT_INTERVAL_YEAR_MONTH 124 +#define SQL_CONVERT_WLONGVARCHAR 125 +#define SQL_CONVERT_WVARCHAR 126 +#define SQL_CREATE_ASSERTION 127 +#define SQL_CREATE_CHARACTER_SET 128 +#define SQL_CREATE_COLLATION 129 +#define SQL_CREATE_DOMAIN 130 +#define SQL_CREATE_SCHEMA 131 +#define SQL_CREATE_TABLE 132 +#define SQL_CREATE_TRANSLATION 133 +#define SQL_CREATE_VIEW 134 +#define SQL_DRIVER_HDESC 135 +#define SQL_DROP_ASSERTION 136 +#define SQL_DROP_CHARACTER_SET 137 +#define SQL_DROP_COLLATION 138 +#define SQL_DROP_DOMAIN 139 +#define SQL_DROP_SCHEMA 140 +#define SQL_DROP_TABLE 141 +#define SQL_DROP_TRANSLATION 142 +#define SQL_DROP_VIEW 143 +#define SQL_DYNAMIC_CURSOR_ATTRIBUTES1 144 +#define SQL_DYNAMIC_CURSOR_ATTRIBUTES2 145 +#define SQL_FORWARD_ONLY_CURSOR_ATTRIBUTES1 146 +#define SQL_FORWARD_ONLY_CURSOR_ATTRIBUTES2 147 +#define SQL_INDEX_KEYWORDS 148 +#define SQL_INFO_SCHEMA_VIEWS 149 +#define SQL_KEYSET_CURSOR_ATTRIBUTES1 150 +#define SQL_KEYSET_CURSOR_ATTRIBUTES2 151 +#define SQL_MAX_ASYNC_CONCURRENT_STATEMENTS 10022 /* new X/Open spec */ +#define SQL_ODBC_INTERFACE_CONFORMANCE 152 +#define SQL_PARAM_ARRAY_ROW_COUNTS 153 +#define SQL_PARAM_ARRAY_SELECTS 154 +#define SQL_SCHEMA_TERM SQL_OWNER_TERM +#define SQL_SCHEMA_USAGE SQL_OWNER_USAGE +#define SQL_SQL92_DATETIME_FUNCTIONS 155 +#define SQL_SQL92_FOREIGN_KEY_DELETE_RULE 156 +#define SQL_SQL92_FOREIGN_KEY_UPDATE_RULE 157 +#define SQL_SQL92_GRANT 158 +#define SQL_SQL92_NUMERIC_VALUE_FUNCTIONS 159 +#define SQL_SQL92_PREDICATES 160 +#define SQL_SQL92_RELATIONAL_JOIN_OPERATORS 161 +#define SQL_SQL92_REVOKE 162 +#define SQL_SQL92_ROW_VALUE_CONSTRUCTOR 163 +#define SQL_SQL92_STRING_FUNCTIONS 164 +#define SQL_SQL92_VALUE_EXPRESSIONS 165 +#define SQL_STANDARD_CLI_CONFORMANCE 166 +#define SQL_STATIC_CURSOR_ATTRIBUTES1 167 +#define SQL_STATIC_CURSOR_ATTRIBUTES2 168 + +#define SQL_AGGREGATE_FUNCTIONS 169 +#define SQL_DDL_INDEX 170 +#define SQL_DM_VER 171 +#define SQL_INSERT_STATEMENT 172 +#define SQL_UNION_STATEMENT SQL_UNION +#endif /* ODBCVER >= 0x0300 */ + +/* SQL_ALTER_TABLE bitmasks */ +#if (ODBCVER >= 0x0300) +/* the following 5 bitmasks are defined in sql.h +*#define SQL_AT_ADD_COLUMN 0x00000001L +*#define SQL_AT_DROP_COLUMN 0x00000002L +*#define SQL_AT_ADD_CONSTRAINT 0x00000008L +*/ +#define SQL_AT_ADD_COLUMN_SINGLE 0x00000020L +#define SQL_AT_ADD_COLUMN_DEFAULT 0x00000040L +#define SQL_AT_ADD_COLUMN_COLLATION 0x00000080L +#define SQL_AT_SET_COLUMN_DEFAULT 0x00000100L +#define SQL_AT_DROP_COLUMN_DEFAULT 0x00000200L +#define SQL_AT_DROP_COLUMN_CASCADE 0x00000400L +#define SQL_AT_DROP_COLUMN_RESTRICT 0x00000800L +#define SQL_AT_ADD_TABLE_CONSTRAINT 0x00001000L +#define SQL_AT_DROP_TABLE_CONSTRAINT_CASCADE 0x00002000L +#define SQL_AT_DROP_TABLE_CONSTRAINT_RESTRICT 0x00004000L +#define SQL_AT_CONSTRAINT_NAME_DEFINITION 0x00008000L +#define SQL_AT_CONSTRAINT_INITIALLY_DEFERRED 0x00010000L +#define SQL_AT_CONSTRAINT_INITIALLY_IMMEDIATE 0x00020000L +#define SQL_AT_CONSTRAINT_DEFERRABLE 0x00040000L +#define SQL_AT_CONSTRAINT_NON_DEFERRABLE 0x00080000L +#endif /* ODBCVER >= 0x0300 */ + +/* SQL_CONVERT_* return value bitmasks */ + +#define SQL_CVT_CHAR 0x00000001L +#define SQL_CVT_NUMERIC 0x00000002L +#define SQL_CVT_DECIMAL 0x00000004L +#define SQL_CVT_INTEGER 0x00000008L +#define SQL_CVT_SMALLINT 0x00000010L +#define SQL_CVT_FLOAT 0x00000020L +#define SQL_CVT_REAL 0x00000040L +#define SQL_CVT_DOUBLE 0x00000080L +#define SQL_CVT_VARCHAR 0x00000100L +#define SQL_CVT_LONGVARCHAR 0x00000200L +#define SQL_CVT_BINARY 0x00000400L +#define SQL_CVT_VARBINARY 0x00000800L +#define SQL_CVT_BIT 0x00001000L +#define SQL_CVT_TINYINT 0x00002000L +#define SQL_CVT_BIGINT 0x00004000L +#define SQL_CVT_DATE 0x00008000L +#define SQL_CVT_TIME 0x00010000L +#define SQL_CVT_TIMESTAMP 0x00020000L +#define SQL_CVT_LONGVARBINARY 0x00040000L +#if (ODBCVER >= 0x0300) +#define SQL_CVT_INTERVAL_YEAR_MONTH 0x00080000L +#define SQL_CVT_INTERVAL_DAY_TIME 0x00100000L +#define SQL_CVT_WCHAR 0x00200000L +#define SQL_CVT_WLONGVARCHAR 0x00400000L +#define SQL_CVT_WVARCHAR 0x00800000L + +#endif /* ODBCVER >= 0x0300 */ + + +/* SQL_CONVERT_FUNCTIONS functions */ +#define SQL_FN_CVT_CONVERT 0x00000001L +#if (ODBCVER >= 0x0300) +#define SQL_FN_CVT_CAST 0x00000002L +#endif /* ODBCVER >= 0x0300 */ + + +/* SQL_STRING_FUNCTIONS functions */ + +#define SQL_FN_STR_CONCAT 0x00000001L +#define SQL_FN_STR_INSERT 0x00000002L +#define SQL_FN_STR_LEFT 0x00000004L +#define SQL_FN_STR_LTRIM 0x00000008L +#define SQL_FN_STR_LENGTH 0x00000010L +#define SQL_FN_STR_LOCATE 0x00000020L +#define SQL_FN_STR_LCASE 0x00000040L +#define SQL_FN_STR_REPEAT 0x00000080L +#define SQL_FN_STR_REPLACE 0x00000100L +#define SQL_FN_STR_RIGHT 0x00000200L +#define SQL_FN_STR_RTRIM 0x00000400L +#define SQL_FN_STR_SUBSTRING 0x00000800L +#define SQL_FN_STR_UCASE 0x00001000L +#define SQL_FN_STR_ASCII 0x00002000L +#define SQL_FN_STR_CHAR 0x00004000L +#define SQL_FN_STR_DIFFERENCE 0x00008000L +#define SQL_FN_STR_LOCATE_2 0x00010000L +#define SQL_FN_STR_SOUNDEX 0x00020000L +#define SQL_FN_STR_SPACE 0x00040000L +#if (ODBCVER >= 0x0300) +#define SQL_FN_STR_BIT_LENGTH 0x00080000L +#define SQL_FN_STR_CHAR_LENGTH 0x00100000L +#define SQL_FN_STR_CHARACTER_LENGTH 0x00200000L +#define SQL_FN_STR_OCTET_LENGTH 0x00400000L +#define SQL_FN_STR_POSITION 0x00800000L +#endif /* ODBCVER >= 0x0300 */ + +/* SQL_SQL92_STRING_FUNCTIONS */ +#if (ODBCVER >= 0x0300) +#define SQL_SSF_CONVERT 0x00000001L +#define SQL_SSF_LOWER 0x00000002L +#define SQL_SSF_UPPER 0x00000004L +#define SQL_SSF_SUBSTRING 0x00000008L +#define SQL_SSF_TRANSLATE 0x00000010L +#define SQL_SSF_TRIM_BOTH 0x00000020L +#define SQL_SSF_TRIM_LEADING 0x00000040L +#define SQL_SSF_TRIM_TRAILING 0x00000080L +#endif /* ODBCVER >= 0x0300 */ + +/* SQL_NUMERIC_FUNCTIONS functions */ + +#define SQL_FN_NUM_ABS 0x00000001L +#define SQL_FN_NUM_ACOS 0x00000002L +#define SQL_FN_NUM_ASIN 0x00000004L +#define SQL_FN_NUM_ATAN 0x00000008L +#define SQL_FN_NUM_ATAN2 0x00000010L +#define SQL_FN_NUM_CEILING 0x00000020L +#define SQL_FN_NUM_COS 0x00000040L +#define SQL_FN_NUM_COT 0x00000080L +#define SQL_FN_NUM_EXP 0x00000100L +#define SQL_FN_NUM_FLOOR 0x00000200L +#define SQL_FN_NUM_LOG 0x00000400L +#define SQL_FN_NUM_MOD 0x00000800L +#define SQL_FN_NUM_SIGN 0x00001000L +#define SQL_FN_NUM_SIN 0x00002000L +#define SQL_FN_NUM_SQRT 0x00004000L +#define SQL_FN_NUM_TAN 0x00008000L +#define SQL_FN_NUM_PI 0x00010000L +#define SQL_FN_NUM_RAND 0x00020000L +#define SQL_FN_NUM_DEGREES 0x00040000L +#define SQL_FN_NUM_LOG10 0x00080000L +#define SQL_FN_NUM_POWER 0x00100000L +#define SQL_FN_NUM_RADIANS 0x00200000L +#define SQL_FN_NUM_ROUND 0x00400000L +#define SQL_FN_NUM_TRUNCATE 0x00800000L + +/* SQL_SQL92_NUMERIC_VALUE_FUNCTIONS */ +#if (ODBCVER >= 0x0300) +#define SQL_SNVF_BIT_LENGTH 0x00000001L +#define SQL_SNVF_CHAR_LENGTH 0x00000002L +#define SQL_SNVF_CHARACTER_LENGTH 0x00000004L +#define SQL_SNVF_EXTRACT 0x00000008L +#define SQL_SNVF_OCTET_LENGTH 0x00000010L +#define SQL_SNVF_POSITION 0x00000020L +#endif /* ODBCVER >= 0x0300 */ + +/* SQL_TIMEDATE_FUNCTIONS functions */ + +#define SQL_FN_TD_NOW 0x00000001L +#define SQL_FN_TD_CURDATE 0x00000002L +#define SQL_FN_TD_DAYOFMONTH 0x00000004L +#define SQL_FN_TD_DAYOFWEEK 0x00000008L +#define SQL_FN_TD_DAYOFYEAR 0x00000010L +#define SQL_FN_TD_MONTH 0x00000020L +#define SQL_FN_TD_QUARTER 0x00000040L +#define SQL_FN_TD_WEEK 0x00000080L +#define SQL_FN_TD_YEAR 0x00000100L +#define SQL_FN_TD_CURTIME 0x00000200L +#define SQL_FN_TD_HOUR 0x00000400L +#define SQL_FN_TD_MINUTE 0x00000800L +#define SQL_FN_TD_SECOND 0x00001000L +#define SQL_FN_TD_TIMESTAMPADD 0x00002000L +#define SQL_FN_TD_TIMESTAMPDIFF 0x00004000L +#define SQL_FN_TD_DAYNAME 0x00008000L +#define SQL_FN_TD_MONTHNAME 0x00010000L +#if (ODBCVER >= 0x0300) +#define SQL_FN_TD_CURRENT_DATE 0x00020000L +#define SQL_FN_TD_CURRENT_TIME 0x00040000L +#define SQL_FN_TD_CURRENT_TIMESTAMP 0x00080000L +#define SQL_FN_TD_EXTRACT 0x00100000L +#endif /* ODBCVER >= 0x0300 */ + +/* SQL_SQL92_DATETIME_FUNCTIONS */ +#if (ODBCVER >= 0x0300) +#define SQL_SDF_CURRENT_DATE 0x00000001L +#define SQL_SDF_CURRENT_TIME 0x00000002L +#define SQL_SDF_CURRENT_TIMESTAMP 0x00000004L +#endif /* ODBCVER >= 0x0300 */ + +/* SQL_SYSTEM_FUNCTIONS functions */ + +#define SQL_FN_SYS_USERNAME 0x00000001L +#define SQL_FN_SYS_DBNAME 0x00000002L +#define SQL_FN_SYS_IFNULL 0x00000004L + +/* SQL_TIMEDATE_ADD_INTERVALS and SQL_TIMEDATE_DIFF_INTERVALS functions */ + +#define SQL_FN_TSI_FRAC_SECOND 0x00000001L +#define SQL_FN_TSI_SECOND 0x00000002L +#define SQL_FN_TSI_MINUTE 0x00000004L +#define SQL_FN_TSI_HOUR 0x00000008L +#define SQL_FN_TSI_DAY 0x00000010L +#define SQL_FN_TSI_WEEK 0x00000020L +#define SQL_FN_TSI_MONTH 0x00000040L +#define SQL_FN_TSI_QUARTER 0x00000080L +#define SQL_FN_TSI_YEAR 0x00000100L + +/* bitmasks for SQL_DYNAMIC_CURSOR_ATTRIBUTES1, + * SQL_FORWARD_ONLY_CURSOR_ATTRIBUTES1, + * SQL_KEYSET_CURSOR_ATTRIBUTES1, and SQL_STATIC_CURSOR_ATTRIBUTES1 + */ +#if (ODBCVER >= 0x0300) +/* supported SQLFetchScroll FetchOrientation's */ +#define SQL_CA1_NEXT 0x00000001L +#define SQL_CA1_ABSOLUTE 0x00000002L +#define SQL_CA1_RELATIVE 0x00000004L +#define SQL_CA1_BOOKMARK 0x00000008L + +/* supported SQLSetPos LockType's */ +#define SQL_CA1_LOCK_NO_CHANGE 0x00000040L +#define SQL_CA1_LOCK_EXCLUSIVE 0x00000080L +#define SQL_CA1_LOCK_UNLOCK 0x00000100L + +/* supported SQLSetPos Operations */ +#define SQL_CA1_POS_POSITION 0x00000200L +#define SQL_CA1_POS_UPDATE 0x00000400L +#define SQL_CA1_POS_DELETE 0x00000800L +#define SQL_CA1_POS_REFRESH 0x00001000L + +/* positioned updates and deletes */ +#define SQL_CA1_POSITIONED_UPDATE 0x00002000L +#define SQL_CA1_POSITIONED_DELETE 0x00004000L +#define SQL_CA1_SELECT_FOR_UPDATE 0x00008000L + +/* supported SQLBulkOperations operations */ +#define SQL_CA1_BULK_ADD 0x00010000L +#define SQL_CA1_BULK_UPDATE_BY_BOOKMARK 0x00020000L +#define SQL_CA1_BULK_DELETE_BY_BOOKMARK 0x00040000L +#define SQL_CA1_BULK_FETCH_BY_BOOKMARK 0x00080000L +#endif /* ODBCVER >= 0x0300 */ + +/* bitmasks for SQL_DYNAMIC_CURSOR_ATTRIBUTES2, + * SQL_FORWARD_ONLY_CURSOR_ATTRIBUTES2, + * SQL_KEYSET_CURSOR_ATTRIBUTES2, and SQL_STATIC_CURSOR_ATTRIBUTES2 + */ +#if (ODBCVER >= 0x0300) +/* supported values for SQL_ATTR_SCROLL_CONCURRENCY */ +#define SQL_CA2_READ_ONLY_CONCURRENCY 0x00000001L +#define SQL_CA2_LOCK_CONCURRENCY 0x00000002L +#define SQL_CA2_OPT_ROWVER_CONCURRENCY 0x00000004L +#define SQL_CA2_OPT_VALUES_CONCURRENCY 0x00000008L + +/* sensitivity of the cursor to its own inserts, deletes, and updates */ +#define SQL_CA2_SENSITIVITY_ADDITIONS 0x00000010L +#define SQL_CA2_SENSITIVITY_DELETIONS 0x00000020L +#define SQL_CA2_SENSITIVITY_UPDATES 0x00000040L + +/* semantics of SQL_ATTR_MAX_ROWS */ +#define SQL_CA2_MAX_ROWS_SELECT 0x00000080L +#define SQL_CA2_MAX_ROWS_INSERT 0x00000100L +#define SQL_CA2_MAX_ROWS_DELETE 0x00000200L +#define SQL_CA2_MAX_ROWS_UPDATE 0x00000400L +#define SQL_CA2_MAX_ROWS_CATALOG 0x00000800L +#define SQL_CA2_MAX_ROWS_AFFECTS_ALL (SQL_CA2_MAX_ROWS_SELECT | \ + SQL_CA2_MAX_ROWS_INSERT | SQL_CA2_MAX_ROWS_DELETE | \ + SQL_CA2_MAX_ROWS_UPDATE | SQL_CA2_MAX_ROWS_CATALOG) + +/* semantics of SQL_DIAG_CURSOR_ROW_COUNT */ +#define SQL_CA2_CRC_EXACT 0x00001000L +#define SQL_CA2_CRC_APPROXIMATE 0x00002000L + +/* the kinds of positioned statements that can be simulated */ +#define SQL_CA2_SIMULATE_NON_UNIQUE 0x00004000L +#define SQL_CA2_SIMULATE_TRY_UNIQUE 0x00008000L +#define SQL_CA2_SIMULATE_UNIQUE 0x00010000L +#endif /* ODBCVER >= 0x0300 */ + +/* SQL_ODBC_API_CONFORMANCE values */ + +#define SQL_OAC_NONE 0x0000 +#define SQL_OAC_LEVEL1 0x0001 +#define SQL_OAC_LEVEL2 0x0002 + +/* SQL_ODBC_SAG_CLI_CONFORMANCE values */ + +#define SQL_OSCC_NOT_COMPLIANT 0x0000 +#define SQL_OSCC_COMPLIANT 0x0001 + +/* SQL_ODBC_SQL_CONFORMANCE values */ + +#define SQL_OSC_MINIMUM 0x0000 +#define SQL_OSC_CORE 0x0001 +#define SQL_OSC_EXTENDED 0x0002 + + +/* SQL_CONCAT_NULL_BEHAVIOR values */ + +#define SQL_CB_NULL 0x0000 +#define SQL_CB_NON_NULL 0x0001 + +/* SQL_SCROLL_OPTIONS masks */ + +#define SQL_SO_FORWARD_ONLY 0x00000001L +#define SQL_SO_KEYSET_DRIVEN 0x00000002L +#define SQL_SO_DYNAMIC 0x00000004L +#define SQL_SO_MIXED 0x00000008L +#define SQL_SO_STATIC 0x00000010L + +/* SQL_FETCH_DIRECTION masks */ + +/* SQL_FETCH_RESUME is no longer supported */ +#if (ODBCVER < 0x0300) +#define SQL_FD_FETCH_RESUME 0x00000040L +#endif + +#define SQL_FD_FETCH_BOOKMARK 0x00000080L + +/* SQL_TXN_ISOLATION_OPTION masks */ +/* SQL_TXN_VERSIONING is no longer supported */ +#if (ODBCVER < 0x0300) +#define SQL_TXN_VERSIONING 0x00000010L +#endif + +/* SQL_CORRELATION_NAME values */ + +#define SQL_CN_NONE 0x0000 +#define SQL_CN_DIFFERENT 0x0001 +#define SQL_CN_ANY 0x0002 + +/* SQL_NON_NULLABLE_COLUMNS values */ + +#define SQL_NNC_NULL 0x0000 +#define SQL_NNC_NON_NULL 0x0001 + +/* SQL_NULL_COLLATION values */ + +#define SQL_NC_START 0x0002 +#define SQL_NC_END 0x0004 + +/* SQL_FILE_USAGE values */ + +#define SQL_FILE_NOT_SUPPORTED 0x0000 +#define SQL_FILE_TABLE 0x0001 +#define SQL_FILE_QUALIFIER 0x0002 +#define SQL_FILE_CATALOG SQL_FILE_QUALIFIER // ODBC 3.0 + + +/* SQL_GETDATA_EXTENSIONS values */ + +#define SQL_GD_BLOCK 0x00000004L +#define SQL_GD_BOUND 0x00000008L + +/* SQL_POSITIONED_STATEMENTS masks */ + +#define SQL_PS_POSITIONED_DELETE 0x00000001L +#define SQL_PS_POSITIONED_UPDATE 0x00000002L +#define SQL_PS_SELECT_FOR_UPDATE 0x00000004L + +/* SQL_GROUP_BY values */ + +#define SQL_GB_NOT_SUPPORTED 0x0000 +#define SQL_GB_GROUP_BY_EQUALS_SELECT 0x0001 +#define SQL_GB_GROUP_BY_CONTAINS_SELECT 0x0002 +#define SQL_GB_NO_RELATION 0x0003 +#if (ODBCVER >= 0x0300) +#define SQL_GB_COLLATE 0x0004 + +#endif /* ODBCVER >= 0x0300 */ + +/* SQL_OWNER_USAGE masks */ + +#define SQL_OU_DML_STATEMENTS 0x00000001L +#define SQL_OU_PROCEDURE_INVOCATION 0x00000002L +#define SQL_OU_TABLE_DEFINITION 0x00000004L +#define SQL_OU_INDEX_DEFINITION 0x00000008L +#define SQL_OU_PRIVILEGE_DEFINITION 0x00000010L + +/* SQL_SCHEMA_USAGE masks */ +#if (ODBCVER >= 0x0300) +#define SQL_SU_DML_STATEMENTS SQL_OU_DML_STATEMENTS +#define SQL_SU_PROCEDURE_INVOCATION SQL_OU_PROCEDURE_INVOCATION +#define SQL_SU_TABLE_DEFINITION SQL_OU_TABLE_DEFINITION +#define SQL_SU_INDEX_DEFINITION SQL_OU_INDEX_DEFINITION +#define SQL_SU_PRIVILEGE_DEFINITION SQL_OU_PRIVILEGE_DEFINITION +#endif /* ODBCVER >= 0x0300 */ + +/* SQL_QUALIFIER_USAGE masks */ + +#define SQL_QU_DML_STATEMENTS 0x00000001L +#define SQL_QU_PROCEDURE_INVOCATION 0x00000002L +#define SQL_QU_TABLE_DEFINITION 0x00000004L +#define SQL_QU_INDEX_DEFINITION 0x00000008L +#define SQL_QU_PRIVILEGE_DEFINITION 0x00000010L + +#if (ODBCVER >= 0x0300) +/* SQL_CATALOG_USAGE masks */ +#define SQL_CU_DML_STATEMENTS SQL_QU_DML_STATEMENTS +#define SQL_CU_PROCEDURE_INVOCATION SQL_QU_PROCEDURE_INVOCATION +#define SQL_CU_TABLE_DEFINITION SQL_QU_TABLE_DEFINITION +#define SQL_CU_INDEX_DEFINITION SQL_QU_INDEX_DEFINITION +#define SQL_CU_PRIVILEGE_DEFINITION SQL_QU_PRIVILEGE_DEFINITION +#endif /* ODBCVER >= 0x0300 */ + +/* SQL_SUBQUERIES masks */ + +#define SQL_SQ_COMPARISON 0x00000001L +#define SQL_SQ_EXISTS 0x00000002L +#define SQL_SQ_IN 0x00000004L +#define SQL_SQ_QUANTIFIED 0x00000008L +#define SQL_SQ_CORRELATED_SUBQUERIES 0x00000010L + +/* SQL_UNION masks */ + +#define SQL_U_UNION 0x00000001L +#define SQL_U_UNION_ALL 0x00000002L + +/* SQL_BOOKMARK_PERSISTENCE values */ + +#define SQL_BP_CLOSE 0x00000001L +#define SQL_BP_DELETE 0x00000002L +#define SQL_BP_DROP 0x00000004L +#define SQL_BP_TRANSACTION 0x00000008L +#define SQL_BP_UPDATE 0x00000010L +#define SQL_BP_OTHER_HSTMT 0x00000020L +#define SQL_BP_SCROLL 0x00000040L + +/* SQL_STATIC_SENSITIVITY values */ + +#define SQL_SS_ADDITIONS 0x00000001L +#define SQL_SS_DELETIONS 0x00000002L +#define SQL_SS_UPDATES 0x00000004L + +/* SQL_VIEW values */ +#define SQL_CV_CREATE_VIEW 0x00000001L +#define SQL_CV_CHECK_OPTION 0x00000002L +#define SQL_CV_CASCADED 0x00000004L +#define SQL_CV_LOCAL 0x00000008L + +/* SQL_LOCK_TYPES masks */ + +#define SQL_LCK_NO_CHANGE 0x00000001L +#define SQL_LCK_EXCLUSIVE 0x00000002L +#define SQL_LCK_UNLOCK 0x00000004L + +/* SQL_POS_OPERATIONS masks */ + +#define SQL_POS_POSITION 0x00000001L +#define SQL_POS_REFRESH 0x00000002L +#define SQL_POS_UPDATE 0x00000004L +#define SQL_POS_DELETE 0x00000008L +#define SQL_POS_ADD 0x00000010L + +/* SQL_QUALIFIER_LOCATION values */ + +#define SQL_QL_START 0x0001 +#define SQL_QL_END 0x0002 + +/* Here start return values for ODBC 3.0 SQLGetInfo */ + +#if (ODBCVER >= 0x0300) +/* SQL_AGGREGATE_FUNCTIONS bitmasks */ +#define SQL_AF_AVG 0x00000001L +#define SQL_AF_COUNT 0x00000002L +#define SQL_AF_MAX 0x00000004L +#define SQL_AF_MIN 0x00000008L +#define SQL_AF_SUM 0x00000010L +#define SQL_AF_DISTINCT 0x00000020L +#define SQL_AF_ALL (SQL_AF_AVG | SQL_AF_COUNT | SQL_AF_MAX | SQL_AF_MIN | SQL_AF_SUM | SQL_AF_DISTINCT) + +/* SQL_SQL_CONFORMANCE bit masks */ +#define SQL_SC_SQL92_ENTRY 0x00000001L +#define SQL_SC_FIPS127_2_TRANSITIONAL 0x00000002L +#define SQL_SC_SQL92_INTERMEDIATE 0x00000004L +#define SQL_SC_SQL92_FULL 0x00000008L + +/* SQL_DATETIME_LITERALS masks */ +#define SQL_DL_SQL92_DATE 0x00000001L +#define SQL_DL_SQL92_TIME 0x00000002L +#define SQL_DL_SQL92_TIMESTAMP 0x00000004L +#define SQL_DL_SQL92_INTERVAL_YEAR 0x00000008L +#define SQL_DL_SQL92_INTERVAL_MONTH 0x00000010L +#define SQL_DL_SQL92_INTERVAL_DAY 0x00000020L +#define SQL_DL_SQL92_INTERVAL_HOUR 0x00000040L +#define SQL_DL_SQL92_INTERVAL_MINUTE 0x00000080L +#define SQL_DL_SQL92_INTERVAL_SECOND 0x00000100L +#define SQL_DL_SQL92_INTERVAL_YEAR_TO_MONTH 0x00000200L +#define SQL_DL_SQL92_INTERVAL_DAY_TO_HOUR 0x00000400L +#define SQL_DL_SQL92_INTERVAL_DAY_TO_MINUTE 0x00000800L +#define SQL_DL_SQL92_INTERVAL_DAY_TO_SECOND 0x00001000L +#define SQL_DL_SQL92_INTERVAL_HOUR_TO_MINUTE 0x00002000L +#define SQL_DL_SQL92_INTERVAL_HOUR_TO_SECOND 0x00004000L +#define SQL_DL_SQL92_INTERVAL_MINUTE_TO_SECOND 0x00008000L + +/* SQL_CATALOG_LOCATION values */ +#define SQL_CL_START SQL_QL_START +#define SQL_CL_END SQL_QL_END + +/* values for SQL_BATCH_ROW_COUNT */ +#define SQL_BRC_PROCEDURES 0x0000001 +#define SQL_BRC_EXPLICIT 0x0000002 +#define SQL_BRC_ROLLED_UP 0x0000004 + +/* bitmasks for SQL_BATCH_SUPPORT */ +#define SQL_BS_SELECT_EXPLICIT 0x00000001L +#define SQL_BS_ROW_COUNT_EXPLICIT 0x00000002L +#define SQL_BS_SELECT_PROC 0x00000004L +#define SQL_BS_ROW_COUNT_PROC 0x00000008L + +/* Values for SQL_PARAM_ARRAY_ROW_COUNTS getinfo */ +#define SQL_PARC_BATCH 1 +#define SQL_PARC_NO_BATCH 2 + +/* values for SQL_PARAM_ARRAY_SELECT_BATCH*/ +#define SQL_PAS_BATCH 1 +#define SQL_PAS_NO_BATCH 2 +#define SQL_PAS_NO_SELECT 3 + +/* Bitmasks for SQL_INDEX_KEYWORDS */ +#define SQL_IK_NONE 0x00000000L +#define SQL_IK_ASC 0x00000001L +#define SQL_IK_DESC 0x00000002L +#define SQL_IK_ALL (SQL_IK_ASC | SQL_IK_DESC) + +/* Bitmasks for SQL_INFO_SCHEMA_VIEWS */ + +#define SQL_ISV_ASSERTIONS 0x00000001L +#define SQL_ISV_CHARACTER_SETS 0x00000002L +#define SQL_ISV_CHECK_CONSTRAINTS 0x00000004L +#define SQL_ISV_COLLATIONS 0x00000008L +#define SQL_ISV_COLUMN_DOMAIN_USAGE 0x00000010L +#define SQL_ISV_COLUMN_PRIVILEGES 0x00000020L +#define SQL_ISV_COLUMNS 0x00000040L +#define SQL_ISV_CONSTRAINT_COLUMN_USAGE 0x00000080L +#define SQL_ISV_CONSTRAINT_TABLE_USAGE 0x00000100L +#define SQL_ISV_DOMAIN_CONSTRAINTS 0x00000200L +#define SQL_ISV_DOMAINS 0x00000400L +#define SQL_ISV_KEY_COLUMN_USAGE 0x00000800L +#define SQL_ISV_REFERENTIAL_CONSTRAINTS 0x00001000L +#define SQL_ISV_SCHEMATA 0x00002000L +#define SQL_ISV_SQL_LANGUAGES 0x00004000L +#define SQL_ISV_TABLE_CONSTRAINTS 0x00008000L +#define SQL_ISV_TABLE_PRIVILEGES 0x00010000L +#define SQL_ISV_TABLES 0x00020000L +#define SQL_ISV_TRANSLATIONS 0x00040000L +#define SQL_ISV_USAGE_PRIVILEGES 0x00080000L +#define SQL_ISV_VIEW_COLUMN_USAGE 0x00100000L +#define SQL_ISV_VIEW_TABLE_USAGE 0x00200000L +#define SQL_ISV_VIEWS 0x00400000L + +/* Bitmasks for SQL_ASYNC_MODE */ + +#define SQL_AM_NONE 0 +#define SQL_AM_CONNECTION 1 +#define SQL_AM_STATEMENT 2 + +/* Bitmasks for SQL_ALTER_DOMAIN */ +#define SQL_AD_CONSTRAINT_NAME_DEFINITION 0x00000001L +#define SQL_AD_ADD_DOMAIN_CONSTRAINT 0x00000002L +#define SQL_AD_DROP_DOMAIN_CONSTRAINT 0x00000004L +#define SQL_AD_ADD_DOMAIN_DEFAULT 0x00000008L +#define SQL_AD_DROP_DOMAIN_DEFAULT 0x00000010L +#define SQL_AD_ADD_CONSTRAINT_INITIALLY_DEFERRED 0x00000020L +#define SQL_AD_ADD_CONSTRAINT_INITIALLY_IMMEDIATE 0x00000040L +#define SQL_AD_ADD_CONSTRAINT_DEFERRABLE 0x00000080L +#define SQL_AD_ADD_CONSTRAINT_NON_DEFERRABLE 0x00000100L + + +/* SQL_CREATE_SCHEMA bitmasks */ +#define SQL_CS_CREATE_SCHEMA 0x00000001L +#define SQL_CS_AUTHORIZATION 0x00000002L +#define SQL_CS_DEFAULT_CHARACTER_SET 0x00000004L + +/* SQL_CREATE_TRANSLATION bitmasks */ +#define SQL_CTR_CREATE_TRANSLATION 0x00000001L + +/* SQL_CREATE_ASSERTION bitmasks */ +#define SQL_CA_CREATE_ASSERTION 0x00000001L +#define SQL_CA_CONSTRAINT_INITIALLY_DEFERRED 0x00000010L +#define SQL_CA_CONSTRAINT_INITIALLY_IMMEDIATE 0x00000020L +#define SQL_CA_CONSTRAINT_DEFERRABLE 0x00000040L +#define SQL_CA_CONSTRAINT_NON_DEFERRABLE 0x00000080L + +/* SQL_CREATE_CHARACTER_SET bitmasks */ +#define SQL_CCS_CREATE_CHARACTER_SET 0x00000001L +#define SQL_CCS_COLLATE_CLAUSE 0x00000002L +#define SQL_CCS_LIMITED_COLLATION 0x00000004L + +/* SQL_CREATE_COLLATION bitmasks */ +#define SQL_CCOL_CREATE_COLLATION 0x00000001L + +/* SQL_CREATE_DOMAIN bitmasks */ +#define SQL_CDO_CREATE_DOMAIN 0x00000001L +#define SQL_CDO_DEFAULT 0x00000002L +#define SQL_CDO_CONSTRAINT 0x00000004L +#define SQL_CDO_COLLATION 0x00000008L +#define SQL_CDO_CONSTRAINT_NAME_DEFINITION 0x00000010L +#define SQL_CDO_CONSTRAINT_INITIALLY_DEFERRED 0x00000020L +#define SQL_CDO_CONSTRAINT_INITIALLY_IMMEDIATE 0x00000040L +#define SQL_CDO_CONSTRAINT_DEFERRABLE 0x00000080L +#define SQL_CDO_CONSTRAINT_NON_DEFERRABLE 0x00000100L + +/* SQL_CREATE_TABLE bitmasks */ +#define SQL_CT_CREATE_TABLE 0x00000001L +#define SQL_CT_COMMIT_PRESERVE 0x00000002L +#define SQL_CT_COMMIT_DELETE 0x00000004L +#define SQL_CT_GLOBAL_TEMPORARY 0x00000008L +#define SQL_CT_LOCAL_TEMPORARY 0x00000010L +#define SQL_CT_CONSTRAINT_INITIALLY_DEFERRED 0x00000020L +#define SQL_CT_CONSTRAINT_INITIALLY_IMMEDIATE 0x00000040L +#define SQL_CT_CONSTRAINT_DEFERRABLE 0x00000080L +#define SQL_CT_CONSTRAINT_NON_DEFERRABLE 0x00000100L +#define SQL_CT_COLUMN_CONSTRAINT 0x00000200L +#define SQL_CT_COLUMN_DEFAULT 0x00000400L +#define SQL_CT_COLUMN_COLLATION 0x00000800L +#define SQL_CT_TABLE_CONSTRAINT 0x00001000L +#define SQL_CT_CONSTRAINT_NAME_DEFINITION 0x00002000L + +/* SQL_DDL_INDEX bitmasks */ +#define SQL_DI_CREATE_INDEX 0x00000001L +#define SQL_DI_DROP_INDEX 0x00000002L + +/* SQL_DROP_COLLATION bitmasks */ +#define SQL_DC_DROP_COLLATION 0x00000001L + +/* SQL_DROP_DOMAIN bitmasks */ +#define SQL_DD_DROP_DOMAIN 0x00000001L +#define SQL_DD_RESTRICT 0x00000002L +#define SQL_DD_CASCADE 0x00000004L + +/* SQL_DROP_SCHEMA bitmasks */ +#define SQL_DS_DROP_SCHEMA 0x00000001L +#define SQL_DS_RESTRICT 0x00000002L +#define SQL_DS_CASCADE 0x00000004L + +/* SQL_DROP_CHARACTER_SET bitmasks */ +#define SQL_DCS_DROP_CHARACTER_SET 0x00000001L + +/* SQL_DROP_ASSERTION bitmasks */ +#define SQL_DA_DROP_ASSERTION 0x00000001L + +/* SQL_DROP_TABLE bitmasks */ +#define SQL_DT_DROP_TABLE 0x00000001L +#define SQL_DT_RESTRICT 0x00000002L +#define SQL_DT_CASCADE 0x00000004L + +/* SQL_DROP_TRANSLATION bitmasks */ +#define SQL_DTR_DROP_TRANSLATION 0x00000001L + +/* SQL_DROP_VIEW bitmasks */ +#define SQL_DV_DROP_VIEW 0x00000001L +#define SQL_DV_RESTRICT 0x00000002L +#define SQL_DV_CASCADE 0x00000004L + +/* SQL_INSERT_STATEMENT bitmasks */ +#define SQL_IS_INSERT_LITERALS 0x00000001L +#define SQL_IS_INSERT_SEARCHED 0x00000002L +#define SQL_IS_SELECT_INTO 0x00000004L + +/* SQL_ODBC_INTERFACE_CONFORMANCE values */ +#define SQL_OIC_CORE 1UL +#define SQL_OIC_LEVEL1 2UL +#define SQL_OIC_LEVEL2 3UL + +/* SQL_SQL92_FOREIGN_KEY_DELETE_RULE bitmasks */ +#define SQL_SFKD_CASCADE 0x00000001L +#define SQL_SFKD_NO_ACTION 0x00000002L +#define SQL_SFKD_SET_DEFAULT 0x00000004L +#define SQL_SFKD_SET_NULL 0x00000008L + +/* SQL_SQL92_FOREIGN_KEY_UPDATE_RULE bitmasks */ +#define SQL_SFKU_CASCADE 0x00000001L +#define SQL_SFKU_NO_ACTION 0x00000002L +#define SQL_SFKU_SET_DEFAULT 0x00000004L +#define SQL_SFKU_SET_NULL 0x00000008L + +/* SQL_SQL92_GRANT bitmasks */ +#define SQL_SG_USAGE_ON_DOMAIN 0x00000001L +#define SQL_SG_USAGE_ON_CHARACTER_SET 0x00000002L +#define SQL_SG_USAGE_ON_COLLATION 0x00000004L +#define SQL_SG_USAGE_ON_TRANSLATION 0x00000008L +#define SQL_SG_WITH_GRANT_OPTION 0x00000010L +#define SQL_SG_DELETE_TABLE 0x00000020L +#define SQL_SG_INSERT_TABLE 0x00000040L +#define SQL_SG_INSERT_COLUMN 0x00000080L +#define SQL_SG_REFERENCES_TABLE 0x00000100L +#define SQL_SG_REFERENCES_COLUMN 0x00000200L +#define SQL_SG_SELECT_TABLE 0x00000400L +#define SQL_SG_UPDATE_TABLE 0x00000800L +#define SQL_SG_UPDATE_COLUMN 0x00001000L + +/* SQL_SQL92_PREDICATES bitmasks */ +#define SQL_SP_EXISTS 0x00000001L +#define SQL_SP_ISNOTNULL 0x00000002L +#define SQL_SP_ISNULL 0x00000004L +#define SQL_SP_MATCH_FULL 0x00000008L +#define SQL_SP_MATCH_PARTIAL 0x00000010L +#define SQL_SP_MATCH_UNIQUE_FULL 0x00000020L +#define SQL_SP_MATCH_UNIQUE_PARTIAL 0x00000040L +#define SQL_SP_OVERLAPS 0x00000080L +#define SQL_SP_UNIQUE 0x00000100L +#define SQL_SP_LIKE 0x00000200L +#define SQL_SP_IN 0x00000400L +#define SQL_SP_BETWEEN 0x00000800L +#define SQL_SP_COMPARISON 0x00001000L +#define SQL_SP_QUANTIFIED_COMPARISON 0x00002000L + +/* SQL_SQL92_RELATIONAL_JOIN_OPERATORS bitmasks */ +#define SQL_SRJO_CORRESPONDING_CLAUSE 0x00000001L +#define SQL_SRJO_CROSS_JOIN 0x00000002L +#define SQL_SRJO_EXCEPT_JOIN 0x00000004L +#define SQL_SRJO_FULL_OUTER_JOIN 0x00000008L +#define SQL_SRJO_INNER_JOIN 0x00000010L +#define SQL_SRJO_INTERSECT_JOIN 0x00000020L +#define SQL_SRJO_LEFT_OUTER_JOIN 0x00000040L +#define SQL_SRJO_NATURAL_JOIN 0x00000080L +#define SQL_SRJO_RIGHT_OUTER_JOIN 0x00000100L +#define SQL_SRJO_UNION_JOIN 0x00000200L + +/* SQL_SQL92_REVOKE bitmasks */ +#define SQL_SR_USAGE_ON_DOMAIN 0x00000001L +#define SQL_SR_USAGE_ON_CHARACTER_SET 0x00000002L +#define SQL_SR_USAGE_ON_COLLATION 0x00000004L +#define SQL_SR_USAGE_ON_TRANSLATION 0x00000008L +#define SQL_SR_GRANT_OPTION_FOR 0x00000010L +#define SQL_SR_CASCADE 0x00000020L +#define SQL_SR_RESTRICT 0x00000040L +#define SQL_SR_DELETE_TABLE 0x00000080L +#define SQL_SR_INSERT_TABLE 0x00000100L +#define SQL_SR_INSERT_COLUMN 0x00000200L +#define SQL_SR_REFERENCES_TABLE 0x00000400L +#define SQL_SR_REFERENCES_COLUMN 0x00000800L +#define SQL_SR_SELECT_TABLE 0x00001000L +#define SQL_SR_UPDATE_TABLE 0x00002000L +#define SQL_SR_UPDATE_COLUMN 0x00004000L + +/* SQL_SQL92_ROW_VALUE_CONSTRUCTOR bitmasks */ +#define SQL_SRVC_VALUE_EXPRESSION 0x00000001L +#define SQL_SRVC_NULL 0x00000002L +#define SQL_SRVC_DEFAULT 0x00000004L +#define SQL_SRVC_ROW_SUBQUERY 0x00000008L + +/* SQL_SQL92_VALUE_EXPRESSIONS bitmasks */ +#define SQL_SVE_CASE 0x00000001L +#define SQL_SVE_CAST 0x00000002L +#define SQL_SVE_COALESCE 0x00000004L +#define SQL_SVE_NULLIF 0x00000008L + +/* SQL_STANDARD_CLI_CONFORMANCE bitmasks */ +#define SQL_SCC_XOPEN_CLI_VERSION1 0x00000001L +#define SQL_SCC_ISO92_CLI 0x00000002L + +/* SQL_UNION_STATEMENT bitmasks */ +#define SQL_US_UNION SQL_U_UNION +#define SQL_US_UNION_ALL SQL_U_UNION_ALL + +#endif /* ODBCVER >= 0x0300 */ + +/* additional SQLDataSources fetch directions */ +#if (ODBCVER >= 0x0300) +#define SQL_FETCH_FIRST_USER 31 +#define SQL_FETCH_FIRST_SYSTEM 32 +#endif /* ODBCVER >= 0x0300 */ + + +/* Defines for SQLSetPos */ +#define SQL_ENTIRE_ROWSET 0 + +/* Operations in SQLSetPos */ +#define SQL_POSITION 0 /* 1.0 FALSE */ +#define SQL_REFRESH 1 /* 1.0 TRUE */ +#define SQL_UPDATE 2 +#define SQL_DELETE 3 + +/* Operations in SQLBulkOperations */ +#define SQL_ADD 4 +#define SQL_SETPOS_MAX_OPTION_VALUE SQL_ADD +#if (ODBCVER >= 0x0300) +#define SQL_UPDATE_BY_BOOKMARK 5 +#define SQL_DELETE_BY_BOOKMARK 6 +#define SQL_FETCH_BY_BOOKMARK 7 + +#endif /* ODBCVER >= 0x0300 */ + +/* Lock options in SQLSetPos */ +#define SQL_LOCK_NO_CHANGE 0 /* 1.0 FALSE */ +#define SQL_LOCK_EXCLUSIVE 1 /* 1.0 TRUE */ +#define SQL_LOCK_UNLOCK 2 + +#define SQL_SETPOS_MAX_LOCK_VALUE SQL_LOCK_UNLOCK + +/* Macros for SQLSetPos */ +#define SQL_POSITION_TO(hstmt,irow) SQLSetPos(hstmt,irow,SQL_POSITION,SQL_LOCK_NO_CHANGE) +#define SQL_LOCK_RECORD(hstmt,irow,fLock) SQLSetPos(hstmt,irow,SQL_POSITION,fLock) +#define SQL_REFRESH_RECORD(hstmt,irow,fLock) SQLSetPos(hstmt,irow,SQL_REFRESH,fLock) +#define SQL_UPDATE_RECORD(hstmt,irow) SQLSetPos(hstmt,irow,SQL_UPDATE,SQL_LOCK_NO_CHANGE) +#define SQL_DELETE_RECORD(hstmt,irow) SQLSetPos(hstmt,irow,SQL_DELETE,SQL_LOCK_NO_CHANGE) +#define SQL_ADD_RECORD(hstmt,irow) SQLSetPos(hstmt,irow,SQL_ADD,SQL_LOCK_NO_CHANGE) + +/* Column types and scopes in SQLSpecialColumns. */ +#define SQL_BEST_ROWID 1 +#define SQL_ROWVER 2 + +/* Defines for SQLSpecialColumns (returned in the result set) + SQL_PC_UNKNOWN and SQL_PC_PSEUDO are defined in sql.h */ +#define SQL_PC_NOT_PSEUDO 1 + +/* Defines for SQLStatistics */ +#define SQL_QUICK 0 +#define SQL_ENSURE 1 + +/* Defines for SQLStatistics (returned in the result set) + SQL_INDEX_CLUSTERED, SQL_INDEX_HASHED, and SQL_INDEX_OTHER are + defined in sql.h */ +#define SQL_TABLE_STAT 0 + + +/* Defines for SQLTables */ +#if (ODBCVER >= 0x0300) +#define SQL_ALL_CATALOGS "%" +#define SQL_ALL_SCHEMAS "%" +#define SQL_ALL_TABLE_TYPES "%" +#endif /* ODBCVER >= 0x0300 */ + +/* Options for SQLDriverConnect */ +#define SQL_DRIVER_NOPROMPT 0 +#define SQL_DRIVER_COMPLETE 1 +#define SQL_DRIVER_PROMPT 2 +#define SQL_DRIVER_COMPLETE_REQUIRED 3 + +#ifndef RC_INVOKED + +SQLRETURN SQL_API SQLDriverConnect( + SQLHDBC hdbc, + SQLHWND hwnd, + SQLCHAR *szConnStrIn, + SQLSMALLINT cbConnStrIn, + SQLCHAR *szConnStrOut, + SQLSMALLINT cbConnStrOutMax, + SQLSMALLINT *pcbConnStrOut, + SQLUSMALLINT fDriverCompletion); + +#endif /* RC_INVOKED */ + +/* Level 2 Functions */ + +/* SQLExtendedFetch "fFetchType" values */ +#define SQL_FETCH_BOOKMARK 8 + +/* SQLExtendedFetch "rgfRowStatus" element values */ +#define SQL_ROW_SUCCESS 0 +#define SQL_ROW_DELETED 1 +#define SQL_ROW_UPDATED 2 +#define SQL_ROW_NOROW 3 +#define SQL_ROW_ADDED 4 +#define SQL_ROW_ERROR 5 +#if (ODBCVER >= 0x0300) +#define SQL_ROW_SUCCESS_WITH_INFO 6 +#define SQL_ROW_PROCEED 0 +#define SQL_ROW_IGNORE 1 +#endif + +/* value for SQL_DESC_ARRAY_STATUS_PTR */ +#if (ODBCVER >= 0x0300) +#define SQL_PARAM_SUCCESS 0 +#define SQL_PARAM_SUCCESS_WITH_INFO 6 +#define SQL_PARAM_ERROR 5 +#define SQL_PARAM_UNUSED 7 +#define SQL_PARAM_DIAG_UNAVAILABLE 1 + +#define SQL_PARAM_PROCEED 0 +#define SQL_PARAM_IGNORE 1 +#endif /* ODBCVER >= 0x0300 */ + +/* Defines for SQLForeignKeys (returned in result set) */ +#define SQL_CASCADE 0 +#define SQL_RESTRICT 1 +#define SQL_SET_NULL 2 +#if (ODBCVER >= 0x0250) +#define SQL_NO_ACTION 3 +#define SQL_SET_DEFAULT 4 +#endif /* ODBCVER >= 0x0250 */ +#if (ODBCVER >= 0x0300) +#define SQL_INITIALLY_DEFERRED 5 +#define SQL_INITIALLY_IMMEDIATE 6 +#endif /* ODBCVER >= 0x0300 */ + +/* Defines for SQLBindParameter and + SQLProcedureColumns (returned in the result set) */ +#define SQL_PARAM_TYPE_UNKNOWN 0 +#define SQL_PARAM_INPUT 1 +#define SQL_PARAM_INPUT_OUTPUT 2 +#define SQL_RESULT_COL 3 +#define SQL_PARAM_OUTPUT 4 +#define SQL_RETURN_VALUE 5 + +/* Defines for SQLProcedures (returned in the result set) */ +#define SQL_PT_UNKNOWN 0 +#define SQL_PT_PROCEDURE 1 +#define SQL_PT_FUNCTION 2 + +#ifndef RC_INVOKED + +/* This define is too large for RC */ +#define SQL_ODBC_KEYWORDS \ +"ABSOLUTE,ACTION,ADA,ADD,ALL,ALLOCATE,ALTER,AND,ANY,ARE,AS,"\ +"ASC,ASSERTION,AT,AUTHORIZATION,AVG,"\ +"BEGIN,BETWEEN,BIT,BIT_LENGTH,BOTH,BY,CASCADE,CASCADED,CASE,CAST,CATALOG,"\ +"CHAR,CHAR_LENGTH,CHARACTER,CHARACTER_LENGTH,CHECK,CLOSE,COALESCE,"\ +"COLLATE,COLLATION,COLUMN,COMMIT,CONNECT,CONNECTION,CONSTRAINT,"\ +"CONSTRAINTS,CONTINUE,CONVERT,CORRESPONDING,COUNT,CREATE,CROSS,CURRENT,"\ +"CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,"\ +"DATE,DAY,DEALLOCATE,DEC,DECIMAL,DECLARE,DEFAULT,DEFERRABLE,"\ +"DEFERRED,DELETE,DESC,DESCRIBE,DESCRIPTOR,DIAGNOSTICS,DISCONNECT,"\ +"DISTINCT,DOMAIN,DOUBLE,DROP,"\ +"ELSE,END,END-EXEC,ESCAPE,EXCEPT,EXCEPTION,EXEC,EXECUTE,"\ +"EXISTS,EXTERNAL,EXTRACT,"\ +"FALSE,FETCH,FIRST,FLOAT,FOR,FOREIGN,FORTRAN,FOUND,FROM,FULL,"\ +"GET,GLOBAL,GO,GOTO,GRANT,GROUP,HAVING,HOUR,"\ +"IDENTITY,IMMEDIATE,IN,INCLUDE,INDEX,INDICATOR,INITIALLY,INNER,"\ +"INPUT,INSENSITIVE,INSERT,INT,INTEGER,INTERSECT,INTERVAL,INTO,IS,ISOLATION,"\ +"JOIN,KEY,LANGUAGE,LAST,LEADING,LEFT,LEVEL,LIKE,LOCAL,LOWER,"\ +"MATCH,MAX,MIN,MINUTE,MODULE,MONTH,"\ +"NAMES,NATIONAL,NATURAL,NCHAR,NEXT,NO,NONE,NOT,NULL,NULLIF,NUMERIC,"\ +"OCTET_LENGTH,OF,ON,ONLY,OPEN,OPTION,OR,ORDER,OUTER,OUTPUT,OVERLAPS,"\ +"PAD,PARTIAL,PASCAL,PLI,POSITION,PRECISION,PREPARE,PRESERVE,"\ +"PRIMARY,PRIOR,PRIVILEGES,PROCEDURE,PUBLIC,"\ +"READ,REAL,REFERENCES,RELATIVE,RESTRICT,REVOKE,RIGHT,ROLLBACK,ROWS"\ +"SCHEMA,SCROLL,SECOND,SECTION,SELECT,SESSION,SESSION_USER,SET,SIZE,"\ +"SMALLINT,SOME,SPACE,SQL,SQLCA,SQLCODE,SQLERROR,SQLSTATE,SQLWARNING,"\ +"SUBSTRING,SUM,SYSTEM_USER,"\ +"TABLE,TEMPORARY,THEN,TIME,TIMESTAMP,TIMEZONE_HOUR,TIMEZONE_MINUTE,"\ +"TO,TRAILING,TRANSACTION,TRANSLATE,TRANSLATION,TRIM,TRUE,"\ +"UNION,UNIQUE,UNKNOWN,UPDATE,UPPER,USAGE,USER,USING,"\ +"VALUE,VALUES,VARCHAR,VARYING,VIEW,WHEN,WHENEVER,WHERE,WITH,WORK,WRITE,"\ +"YEAR,ZONE" + +SQLRETURN SQL_API SQLBrowseConnect( + SQLHDBC hdbc, + SQLCHAR *szConnStrIn, + SQLSMALLINT cbConnStrIn, + SQLCHAR *szConnStrOut, + SQLSMALLINT cbConnStrOutMax, + SQLSMALLINT *pcbConnStrOut); + +#if (ODBCVER >= 0x0300) +SQLRETURN SQL_API SQLBulkOperations( + SQLHSTMT StatementHandle, + SQLSMALLINT Operation); +#endif /* ODBCVER >= 0x0300 */ + +SQLRETURN SQL_API SQLColAttributes( + SQLHSTMT hstmt, + SQLUSMALLINT icol, + SQLUSMALLINT fDescType, + SQLPOINTER rgbDesc, + SQLSMALLINT cbDescMax, + SQLSMALLINT *pcbDesc, + SQLINTEGER *pfDesc); + +SQLRETURN SQL_API SQLColumnPrivileges( + SQLHSTMT hstmt, + SQLCHAR *szCatalogName, + SQLSMALLINT cbCatalogName, + SQLCHAR *szSchemaName, + SQLSMALLINT cbSchemaName, + SQLCHAR *szTableName, + SQLSMALLINT cbTableName, + SQLCHAR *szColumnName, + SQLSMALLINT cbColumnName); + +SQLRETURN SQL_API SQLDescribeParam( + SQLHSTMT hstmt, + SQLUSMALLINT ipar, + SQLSMALLINT *pfSqlType, + SQLUINTEGER *pcbParamDef, + SQLSMALLINT *pibScale, + SQLSMALLINT *pfNullable); + +SQLRETURN SQL_API SQLExtendedFetch( + SQLHSTMT hstmt, + SQLUSMALLINT fFetchType, + SQLINTEGER irow, + SQLUINTEGER *pcrow, + SQLUSMALLINT *rgfRowStatus); + +SQLRETURN SQL_API SQLForeignKeys( + SQLHSTMT hstmt, + SQLCHAR *szPkCatalogName, + SQLSMALLINT cbPkCatalogName, + SQLCHAR *szPkSchemaName, + SQLSMALLINT cbPkSchemaName, + SQLCHAR *szPkTableName, + SQLSMALLINT cbPkTableName, + SQLCHAR *szFkCatalogName, + SQLSMALLINT cbFkCatalogName, + SQLCHAR *szFkSchemaName, + SQLSMALLINT cbFkSchemaName, + SQLCHAR *szFkTableName, + SQLSMALLINT cbFkTableName); + +SQLRETURN SQL_API SQLMoreResults( + SQLHSTMT hstmt); + +SQLRETURN SQL_API SQLNativeSql( + SQLHDBC hdbc, + SQLCHAR *szSqlStrIn, + SQLINTEGER cbSqlStrIn, + SQLCHAR *szSqlStr, + SQLINTEGER cbSqlStrMax, + SQLINTEGER *pcbSqlStr); + +SQLRETURN SQL_API SQLNumParams( + SQLHSTMT hstmt, + SQLSMALLINT *pcpar); + +SQLRETURN SQL_API SQLParamOptions( + SQLHSTMT hstmt, + SQLUINTEGER crow, + SQLUINTEGER *pirow); + +SQLRETURN SQL_API SQLPrimaryKeys( + SQLHSTMT hstmt, + SQLCHAR *szCatalogName, + SQLSMALLINT cbCatalogName, + SQLCHAR *szSchemaName, + SQLSMALLINT cbSchemaName, + SQLCHAR *szTableName, + SQLSMALLINT cbTableName); + +SQLRETURN SQL_API SQLProcedureColumns( + SQLHSTMT hstmt, + SQLCHAR *szCatalogName, + SQLSMALLINT cbCatalogName, + SQLCHAR *szSchemaName, + SQLSMALLINT cbSchemaName, + SQLCHAR *szProcName, + SQLSMALLINT cbProcName, + SQLCHAR *szColumnName, + SQLSMALLINT cbColumnName); + +SQLRETURN SQL_API SQLProcedures( + SQLHSTMT hstmt, + SQLCHAR *szCatalogName, + SQLSMALLINT cbCatalogName, + SQLCHAR *szSchemaName, + SQLSMALLINT cbSchemaName, + SQLCHAR *szProcName, + SQLSMALLINT cbProcName); + +SQLRETURN SQL_API SQLSetPos( + SQLHSTMT hstmt, + SQLUSMALLINT irow, + SQLUSMALLINT fOption, + SQLUSMALLINT fLock); + +SQLRETURN SQL_API SQLTablePrivileges( + SQLHSTMT hstmt, + SQLCHAR *szCatalogName, + SQLSMALLINT cbCatalogName, + SQLCHAR *szSchemaName, + SQLSMALLINT cbSchemaName, + SQLCHAR *szTableName, + SQLSMALLINT cbTableName); + +SQLRETURN SQL_API SQLDrivers( + SQLHENV henv, + SQLUSMALLINT fDirection, + SQLCHAR *szDriverDesc, + SQLSMALLINT cbDriverDescMax, + SQLSMALLINT *pcbDriverDesc, + SQLCHAR *szDriverAttributes, + SQLSMALLINT cbDrvrAttrMax, + SQLSMALLINT *pcbDrvrAttr); + +SQLRETURN SQL_API SQLBindParameter( + SQLHSTMT hstmt, + SQLUSMALLINT ipar, + SQLSMALLINT fParamType, + SQLSMALLINT fCType, + SQLSMALLINT fSqlType, + SQLUINTEGER cbColDef, + SQLSMALLINT ibScale, + SQLPOINTER rgbValue, + SQLINTEGER cbValueMax, + SQLINTEGER *pcbValue); + + +#endif /* RC_INVOKED */ + +/*---------------------------------------------------------*/ +/* SQLAllocHandleStd is implemented to make SQLAllocHandle */ +/* compatible with X/Open standard. an application should */ +/* not call SQLAllocHandleStd directly */ +/*---------------------------------------------------------*/ +#ifdef ODBC_STD +#define SQLAllocHandle SQLAllocHandleStd +#define SQLAllocEnv(phenv) SQLAllocHandleStd(SQL_HANDLE_ENV, SQL_NULL_HANDLE, phenv) + +/* Internal type subcodes */ +#define SQL_YEAR SQL_CODE_YEAR +#define SQL_MONTH SQL_CODE_MONTH +#define SQL_DAY SQL_CODE_DAY +#define SQL_HOUR SQL_CODE_HOUR +#define SQL_MINUTE SQL_CODE_MINUTE +#define SQL_SECOND SQL_CODE_SECOND +#define SQL_YEAR_TO_MONTH SQL_CODE_YEAR_TO_MONTH +#define SQL_DAY_TO_HOUR SQL_CODE_DAY_TO_HOUR +#define SQL_DAY_TO_MINUTE SQL_CODE_DAY_TO_MINUTE +#define SQL_DAY_TO_SECOND SQL_CODE_DAY_TO_SECOND +#define SQL_HOUR_TO_MINUTE SQL_CODE_HOUR_TO_MINUTE +#define SQL_HOUR_TO_SECOND SQL_CODE_HOUR_TO_SECOND +#define SQL_MINUTE_TO_SECOND SQL_CODE_MINUTE_TO_SECOND +#endif /* ODBC_STD */ + +#if (ODBCVER >= 0x0300) +#ifndef RC_INVOKED +SQLRETURN SQL_API SQLAllocHandleStd( + SQLSMALLINT fHandleType, + SQLHANDLE hInput, + SQLHANDLE *phOutput); +#endif /* RC_INVOKED */ +#endif + +/* Deprecated defines from prior versions of ODBC */ +#define SQL_DATABASE_NAME 16 /* Use SQLGetConnectOption/SQL_CURRENT_QUALIFIER */ +#define SQL_FD_FETCH_PREV SQL_FD_FETCH_PRIOR +#define SQL_FETCH_PREV SQL_FETCH_PRIOR +#define SQL_CONCUR_TIMESTAMP SQL_CONCUR_ROWVER +#define SQL_SCCO_OPT_TIMESTAMP SQL_SCCO_OPT_ROWVER +#define SQL_CC_DELETE SQL_CB_DELETE +#define SQL_CR_DELETE SQL_CB_DELETE +#define SQL_CC_CLOSE SQL_CB_CLOSE +#define SQL_CR_CLOSE SQL_CB_CLOSE +#define SQL_CC_PRESERVE SQL_CB_PRESERVE +#define SQL_CR_PRESERVE SQL_CB_PRESERVE +/* SQL_FETCH_RESUME is not supported by 2.0+ drivers */ +#if (ODBCVER < 0x0300) +#define SQL_FETCH_RESUME 7 +#endif +#define SQL_SCROLL_FORWARD_ONLY 0L /*-SQL_CURSOR_FORWARD_ONLY */ +#define SQL_SCROLL_KEYSET_DRIVEN (-1L) /*-SQL_CURSOR_KEYSET_DRIVEN */ +#define SQL_SCROLL_DYNAMIC (-2L) /*-SQL_CURSOR_DYNAMIC */ +#define SQL_SCROLL_STATIC (-3L) /*-SQL_CURSOR_STATIC */ + +/* Deprecated functions from prior versions of ODBC */ +#ifndef RC_INVOKED + +SQLRETURN SQL_API SQLSetScrollOptions( /* Use SQLSetStmtOptions */ + SQLHSTMT hstmt, + SQLUSMALLINT fConcurrency, + SQLINTEGER crowKeyset, + SQLUSMALLINT crowRowset); + +#endif /* RC_INVOKED */ + + +#ifdef __cplusplus +} /* End of extern "C" { */ +#endif /* __cplusplus */ + +#if defined(WIN32) +#include "sqlucode.h" +#endif + +// Tracing section + +#define TRACE_VERSION 1000 // Version of trace API + +RETCODE SQL_API TraceOpenLogFile(LPWSTR,LPWSTR,DWORD);// open a trace log file +RETCODE SQL_API TraceCloseLogFile(); // Request to close a trace log +VOID SQL_API TraceReturn(RETCODE,RETCODE); // Processes trace after FN is called +DWORD SQL_API TraceVersion(); // Returns trace API version + +#endif /* __SQLEXT */ diff --git a/Online/sqldb/src/sqlite.cpp b/Online/sqldb/src/sqlite.cpp new file mode 100644 index 000000000..488e00afd --- /dev/null +++ b/Online/sqldb/src/sqlite.cpp @@ -0,0 +1,344 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Package : RPC +// +// Author : Markus Frank +//========================================================================== + +/// Framework include files +#include <sqldb/sqlite.h> + +/// SQLite3 header +#include <sqlite3.h> + +/// C/C++ headers +#include <stdexcept> + +#define SQLDB_SQLITE_INLINE + +/// SQLDB namespace declaration +namespace sqldb { + + namespace sqlite_imp { + /// + inline bool sqlite_ok(int status) { + return status == SQLITE_OK; + } + /// + inline field_type gen_field_type(int typ) { + switch(typ) { + case SQLITE_INTEGER: return TYPE_LONGLONG; + case SQLITE_FLOAT: return TYPE_DOUBLE; + case SQLITE_BLOB: return TYPE_STRING; + case SQLITE3_TEXT: return TYPE_BLOB; + case SQLITE_NULL: return TYPE_NULL; + default: break; + } + throw std::runtime_error("Invalid data type encountered!"); + } + /// + int do_exec(sqlite3* h, const char* sql, char** err=0) { + if ( h ) { + char* e = nullptr; + if ( !sqlite_ok(::sqlite3_exec(h, sql, nullptr, nullptr, &e)) ) { + if ( err != nullptr ) *err = e; + return ::sqlite3_extended_errcode(h); + } + return sqldb::OK; + } + return sqldb::INVALID_HANDLE; + } + + /// Technology abstraction layer + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + struct column_bind { + public: + sqlite::statement_t* stmt { nullptr }; + size_t column { 0 }; + public: + int bind_null(column_data& d) { + d.length = 0; + d.item.pointer = nullptr; + return sqldb::OK; + } + /// Generic parameter binding + int bind_str(column_data& d, const char* ptr, size_t len) { + if ( ptr ) { + pointers_t p(ptr); + d.bind_pointer(p.puint8, len); + ::sqlite3_bind_text(this->stmt, this->column+1, p.pstring, len, SQLITE_STATIC); + return sqldb::OK; + } + return bind_null(d); + } + /// Generic parameter binding + int bind_blob(column_data& d, const unsigned char* ptr, size_t len) { + if ( ptr ) { + d.bind_pointer(ptr, len); + ::sqlite3_bind_blob(this->stmt, this->column+1, d.item.pointer, len, SQLITE_STATIC); + return sqldb::OK; + } + return bind_null(d); + } + /// Type specific parameter bind + template <typename T> int bind_int(column_data& d, T& to, const T* from) { + to = *from; + d.length = sizeof(int); + ::sqlite3_bind_int(this->stmt, this->column+1, int(to)); + return sqldb::OK; + } + /// Type specific parameter bind + int bind_int64(column_data& d, int64_t& to, const int64_t* from) { + to = *from; + d.length = sizeof(int64_t); + ::sqlite3_bind_int64(this->stmt, this->column+1, to); + return sqldb::OK; + } + /// Type specific parameter bind + template <typename T> int bind_flt(column_data& d, T& to, const T* from) { + to = *from; + d.length = sizeof(double); + ::sqlite3_bind_double(this->stmt, this->column+1, double(to)); + return sqldb::OK; + } + }; + + /// Technology abstraction layer + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + struct sqlite_statement_backend : public statement::backend { + public: + sqlite::statement_t* _handle { nullptr }; + + /// Initializing constructor + sqlite_statement_backend() = default; + /// Access to the internal statement handle. Throws exception if the handle is invalid + sqlite::statement_t* handle() const { + if ( this->_handle ) return this->_handle; + throw std::runtime_error("Failed to access statement handle. [Invalid handle]"); + } + /// Access error message on failure + virtual std::string errmsg() override { + return ::sqlite3_errmsg(::sqlite3_db_handle(this->_handle)); + } + /// Access the number of rows affected by the last statement + long rows_affected() override { + return ::sqlite3_changes(::sqlite3_db_handle(this->_handle)); + } + /// Character string of the prepared statement + const char* sql() override { + return ::sqlite3_expanded_sql(this->_handle); + } + /// + int bind(size_t col, field_type typ, const void* data, size_t len) override { + pointers_t ptr(data); + column_data& d = this->params[col]; + column_bind b {this->_handle, col}; + + switch(d.type=typ) { + case TYPE_TINY: return b.bind_int(d, d.item.int8, ptr.pint8); + case TYPE_SHORT: return b.bind_int(d, d.item.int16, ptr.pint16); + case TYPE_LONG: return b.bind_int(d, d.item.int32, ptr.pint32); + case TYPE_LONGLONG: return b.bind_int64(d, d.item.int64, ptr.pint64); + case TYPE_FLOAT: return b.bind_flt(d, d.item.real32, ptr.preal32); + case TYPE_DOUBLE: return b.bind_flt(d, d.item.real64, ptr.preal64); + case TYPE_STRING: return b.bind_str(d, ptr.pstring, len); + case TYPE_BLOB: return b.bind_blob(d, ptr.puint8, len); + case TYPE_TIME: return b.bind_int (d, d.item.int64, ptr.pint64); + default: return invalid_statement("Invalid SQLite parameter data type"); + } + return sqldb::OK; + } + /// Fetch next result row. Stop if result is NON-zero + int fetch_one() { + if ( auto* h = this->_handle ) { + int ret = ::sqlite3_step(h); + if( ret == SQLITE_ROW ) { + for( size_t col=0; col < this->fields.size(); col++ ) { + auto& d = this->fields[col]; + int typ = ::sqlite3_column_type(h,col); + d.type = gen_field_type(typ); + switch(typ) { + case SQLITE_INTEGER: + d.item.int64 = ::sqlite3_column_int64(h, col); + d.length = sizeof(int64_t); + break; + case SQLITE_FLOAT: + d.item.real64 = ::sqlite3_column_double(h, col); + d.length = sizeof(double); + break; + case SQLITE_NULL: + case SQLITE_BLOB: + case SQLITE3_TEXT: + d.item.pointer = ::sqlite3_column_blob(h, col); + d.length = ::sqlite3_column_bytes(h, col); + break; + default: + break; + } + } + return sqldb::OK; + } + else if ( ret == SQLITE_DONE ) { + return sqldb::DONE; + } + return sqldb::ERROR; + } + return invalid_statement(); + } + /// Prepare a new statement and allocate resources. + int prepare(sqlite::dbase_t* db, const std::string& sql_str) { + sqlite::statement_t* stmt = nullptr; + if ( auto* h = this->_handle ) { + ::sqlite3_reset(h); + ::sqlite3_finalize(h); + this->_handle = nullptr; + } + int ret = ::sqlite3_prepare_v2(db, sql_str.c_str(), sql_str.length()+1, &stmt, nullptr); + if ( sqlite_ok(ret) ) { + size_t num_param = ::sqlite3_bind_parameter_count(stmt); + size_t num_field = ::sqlite3_column_count(stmt); + this->params.resize(num_param, column_data()); + this->fields.resize(num_field, column_data()); + this->_handle = stmt; + this->stmt = this; + return sqldb::OK; + } + return sqldb::ERROR; + } + int action(action_type type) override { + switch(type) { + case RESET: /// Reset all data to re-execute the statement with new parameters + if( auto* h = this->_handle ) { + if ( !sqlite_ok(::sqlite3_clear_bindings(h)) ) + return sqldb::ERROR; + else if ( !sqlite_ok(::sqlite3_reset(h)) ) + return sqldb::ERROR; + } + return sqldb::OK; + case FINALIZE: /// Finalize and run-down the statement. Release all resources. + if( auto* h = this->_handle ) { + ::sqlite3_reset(h); + ::sqlite3_finalize(h); + this->_handle = nullptr; + } + return sqldb::OK; + case EXEC: /// Execute prepared statement + if( auto* h = this->_handle ) { + const char* str = ::sqlite3_sql(this->_handle); + // On select we call fetch_one after execute. Do nothing + if ( 0 == ::strncasecmp(str, "SELECT", 6) ) + return sqldb::OK; + // For all other statements, call 'step' to execute the statement + int ret = ::sqlite3_step(h); + if( ret == SQLITE_ROW ) + return sqldb::OK; + else if ( ret == SQLITE_DONE ) + return sqldb::DONE; + } + return sqldb::ERROR; + case FETCH: /// Fetch next result row. Stop if result is NON-zero + return this->fetch_one(); + case ERRNO: /// Access error number of last failed statement + return ::sqlite3_extended_errcode(::sqlite3_db_handle(this->_handle)); + default: + throw std::runtime_error("Invalid action request received!"); + } + } + }; + + /// Technology abstraction layer for the database + /** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + struct sqlite_dbase_backend : public database::backend { + sqlite::dbase_t* _handle {nullptr}; + + /// Access to the internal database handle. Throws exception if the handle is invalid + sqlite::dbase_t* handle() const { + if ( _handle ) return _handle; + throw std::runtime_error("Invalid MySQL database handle"); + } + /// Access error message on failure + virtual std::string errmsg() override { + return ::sqlite3_errmsg(_handle); + } + /// Access the number of rows affected by the last statement + virtual long rows_affected() override { + return ::sqlite3_changes(this->_handle); + } + /// Execute prepared statement + virtual int execute(std::string& error, const std::string& sql_str) override { + char* err = nullptr; + int ret = do_exec(this->handle(), sql_str.c_str(), &err); + if ( ret != sqldb::OK && err != nullptr ) error = err; + return ret; + } + /// Prepare a new statement and allocate resources. + int prepare(statement& bs, const std::string& sql_str) override { + auto* stm_imp = new sqlite_statement_backend(); + bs.intern.reset(stm_imp); + return stm_imp->prepare(this->_handle, sql_str); + } + /// Open database + int open(const std::string& connect_string) { + this->_handle = nullptr; + if ( !sqlite_ok(::sqlite3_open(connect_string.c_str(), &this->_handle)) ) + return sqldb::ERROR; + this->name = connect_string; + return sqldb::OK; + } + /// Perform multiple actions + virtual int action(action_type type) override { + switch(type) { + case BEGIN: /// Start transaction + return do_exec(this->handle(), "BEGIN TRANSACTION;"); + case COMMIT: /// Commit transaction + return do_exec(this->handle(), "COMMIT TRANSACTION;"); + case ROLLBACK: /// Rollback transaction + return do_exec(this->handle(), "ROLLBACK TRANSACTION;"); + case CLOSE: /// Close database access + if ( this->_handle ) + ::sqlite3_close(this->_handle); + this->_handle = nullptr; + return sqldb::OK; + case ERRNO: /// Access error number of last failed statement + return ::sqlite3_errcode(_handle); + default: + throw std::runtime_error("Invalid actoin request received!"); + } + } + }; + } + + /// Open the database connection using all information in passed string + template <> std::pair<std::shared_ptr<database::backend>, int> + database::backend::open<sqlite>(const std::string& connect_string) { + typedef sqlite_imp::sqlite_dbase_backend _imp_t; + auto db = std::make_shared<_imp_t>(); + int ret = db->open(connect_string); + if ( ret == sqldb::OK ) + return make_pair(std::move(db), sqldb::OK); + return make_pair(std::make_shared<_imp_t>(), ret); + } +} // End namespace sqldb diff --git a/Online/sqldb/src/sqltypes.h b/Online/sqldb/src/sqltypes.h new file mode 100755 index 000000000..ac9c96713 --- /dev/null +++ b/Online/sqldb/src/sqltypes.h @@ -0,0 +1,266 @@ +/********************************************************************* +** SQLTYPES.H - This file defines the types used in ODBC +** +** (C) Copyright 1995-1996 By Microsoft Corp. +** +** Created 04/10/95 for 2.50 specification +** Updated 12/11/95 for 3.00 specification +*********************************************************************/ + +#ifndef __SQLTYPES +#define __SQLTYPES + +#include <wchar.h> + +/* if ODBCVER is not defined, assume version 3.00 */ +#ifndef ODBCVER +#define ODBCVER 0x0300 +#endif /* ODBCVER */ + +#ifdef __cplusplus +extern "C" { /* Assume C declarations for C++ */ +#endif /* __cplusplus */ + +/* environment specific definitions */ +#ifndef EXPORT +#define EXPORT +#endif + +#ifdef WIN32 +#define SQL_API __stdcall +#else +#define SQL_API +#endif + +#ifndef RC_INVOKED + +/* API declaration data types */ +typedef unsigned char SQLCHAR; +#if (ODBCVER >= 0x0300) +typedef signed char SQLSCHAR; +typedef unsigned char SQLDATE; +typedef unsigned char SQLDECIMAL; +typedef double SQLDOUBLE; +typedef double SQLFLOAT; +#endif +typedef long SQLINTEGER; +typedef unsigned long SQLUINTEGER; +#if (ODBCVER >= 0x0300) +typedef unsigned char SQLNUMERIC; +#endif +typedef void * SQLPOINTER; +#if (ODBCVER >= 0x0300) +typedef float SQLREAL; +#endif +typedef short SQLSMALLINT; +typedef unsigned short SQLUSMALLINT; +#if (ODBCVER >= 0x0300) +typedef unsigned char SQLTIME; +typedef unsigned char SQLTIMESTAMP; +typedef unsigned char SQLVARCHAR; +#endif + +/* function return type */ +typedef SQLSMALLINT SQLRETURN; + +/* generic data structures */ +#if (ODBCVER >= 0x0300) +#if defined(WIN32) +typedef void* SQLHANDLE; +#else +typedef SQLINTEGER SQLHANDLE; +#endif /* defined(WIN32) */ +typedef SQLHANDLE SQLHENV; +typedef SQLHANDLE SQLHDBC; +typedef SQLHANDLE SQLHSTMT; +typedef SQLHANDLE SQLHDESC; +#else +#if defined(WIN32) +typedef void* SQLHENV; +typedef void* SQLHDBC; +typedef void* SQLHSTMT; +#else +typedef SQLINTEGER SQLHENV; +typedef SQLINTEGER SQLHDBC; +typedef SQLINTEGER SQLHSTMT; +#endif /* defined(WIN32) */ +#endif /* ODBCVER >= 0x0300 */ + +/* SQL portable types for C */ +typedef unsigned char UCHAR; +typedef signed char SCHAR; +typedef SCHAR SQLSCHAR; +typedef long int SDWORD; +typedef short int SWORD; +typedef unsigned long int UDWORD; +typedef unsigned short int UWORD; +typedef UDWORD SQLUINTEGER; + +typedef signed long SLONG; +typedef signed short SSHORT; +typedef unsigned long ULONG; +typedef unsigned short USHORT; +typedef double SDOUBLE; +typedef double LDOUBLE; +typedef float SFLOAT; + +typedef void* PTR; + +typedef void* HENV; +typedef void* HDBC; +typedef void* HSTMT; + +typedef signed short RETCODE; + +#if defined(WIN32) || defined(OS2) +typedef HWND SQLHWND; +#elif defined (UNIX) +typedef Widget SQLHWND; +#else +/* placehold for future O/S GUI window handle definition */ +typedef SQLPOINTER SQLHWND; +#endif + +#ifndef __SQLDATE +#define __SQLDATE +/* transfer types for DATE, TIME, TIMESTAMP */ +typedef struct tagDATE_STRUCT +{ + SQLSMALLINT year; + SQLUSMALLINT month; + SQLUSMALLINT day; +} DATE_STRUCT; + +#if (ODBCVER >= 0x0300) +typedef DATE_STRUCT SQL_DATE_STRUCT; +#endif /* ODBCVER >= 0x0300 */ + +typedef struct tagTIME_STRUCT +{ + SQLUSMALLINT hour; + SQLUSMALLINT minute; + SQLUSMALLINT second; +} TIME_STRUCT; + +#if (ODBCVER >= 0x0300) +typedef TIME_STRUCT SQL_TIME_STRUCT; +#endif /* ODBCVER >= 0x0300 */ + +typedef struct tagTIMESTAMP_STRUCT +{ + SQLSMALLINT year; + SQLUSMALLINT month; + SQLUSMALLINT day; + SQLUSMALLINT hour; + SQLUSMALLINT minute; + SQLUSMALLINT second; + SQLUINTEGER fraction; +} TIMESTAMP_STRUCT; + +#if (ODBCVER >= 0x0300) +typedef TIMESTAMP_STRUCT SQL_TIMESTAMP_STRUCT; +#endif /* ODBCVER >= 0x0300 */ + + +/* + * enumerations for DATETIME_INTERVAL_SUBCODE values for interval data types + * these values are from SQL-92 + */ + +#if (ODBCVER >= 0x0300) +typedef enum +{ + SQL_IS_YEAR = 1, + SQL_IS_MONTH = 2, + SQL_IS_DAY = 3, + SQL_IS_HOUR = 4, + SQL_IS_MINUTE = 5, + SQL_IS_SECOND = 6, + SQL_IS_YEAR_TO_MONTH = 7, + SQL_IS_DAY_TO_HOUR = 8, + SQL_IS_DAY_TO_MINUTE = 9, + SQL_IS_DAY_TO_SECOND = 10, + SQL_IS_HOUR_TO_MINUTE = 11, + SQL_IS_HOUR_TO_SECOND = 12, + SQL_IS_MINUTE_TO_SECOND = 13 +} SQLINTERVAL; + +#endif /* ODBCVER >= 0x0300 */ + +#if (ODBCVER >= 0x0300) +typedef struct tagSQL_YEAR_MONTH +{ + SQLUINTEGER year; + SQLUINTEGER month; +} SQL_YEAR_MONTH_STRUCT; + +typedef struct tagSQL_DAY_SECOND +{ + SQLUINTEGER day; + SQLUINTEGER hour; + SQLUINTEGER minute; + SQLUINTEGER second; + SQLUINTEGER fraction; +} SQL_DAY_SECOND_STRUCT; + +typedef struct tagSQL_INTERVAL_STRUCT +{ + SQLINTERVAL interval_type; + SQLSMALLINT interval_sign; + union { + SQL_YEAR_MONTH_STRUCT year_month; + SQL_DAY_SECOND_STRUCT day_second; + } intval; + +} SQL_INTERVAL_STRUCT; + +#endif /* ODBCVER >= 0x0300 */ + +#endif /* __SQLDATE */ + +/* the ODBC C types for SQL_C_SBIGINT and SQL_C_UBIGINT */ +#if (ODBCVER >= 0x0300) +#if (_MSC_VER >= 900) +#define ODBCINT64 __int64 +#endif + +/* If using other compilers, define ODBCINT64 to the + approriate 64 bit integer type */ +#ifdef ODBCINT64 +typedef ODBCINT64 SQLBIGINT; +typedef unsigned ODBCINT64 SQLUBIGINT; +#endif +#endif /* ODBCVER >= 0x0300 */ + +/* internal representation of numeric data type */ +#if (ODBCVER >= 0x0300) +#define SQL_MAX_NUMERIC_LEN 16 +typedef struct tagSQL_NUMERIC_STRUCT +{ + SQLCHAR precision; + SQLSCHAR scale; + SQLCHAR sign; /* 1 if positive, 0 if negative */ + SQLCHAR val[SQL_MAX_NUMERIC_LEN]; +} SQL_NUMERIC_STRUCT; +#endif /* ODBCVER >= 0x0300 */ + +typedef unsigned long int BOOKMARK; + +typedef wchar_t SQLWCHAR; + +#ifdef UNICODE +typedef SQLWCHAR SQLTCHAR; +#else +typedef SQLCHAR SQLTCHAR; +#endif /* UNICODE */ + + + +#endif /* RC_INVOKED */ + + +#ifdef __cplusplus +} /* End of extern "C" { */ +#endif /* __cplusplus */ + +#endif /* #ifndef __SQLTYPES */ diff --git a/Online/sqldb/test/sqldb_test.cpp b/Online/sqldb/test/sqldb_test.cpp new file mode 100644 index 000000000..0138f36a7 --- /dev/null +++ b/Online/sqldb/test/sqldb_test.cpp @@ -0,0 +1,66 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Package : RPC +// +// Author : Markus Frank +//========================================================================== + + +/// Framework include files +#include <sqldb/sqldb.h> + +/// C/C++ include files +#include <iostream> + +using namespace std; + +class mysql; +class sqlite; +template <typename T> void test_db() { + cout << "Hello World!" << endl; + + sqldb::database db; + if ( db.open<T>("") != sqldb::OK ) { + } + sqldb::statement stmt = sqldb::statement::create(db, ""); + if ( !stmt.is_valid() ) { + } + stmt.bind(1, 'a'); + stmt.bind(2, short(1)); + stmt.bind(3, int(10)); + stmt.bind(4, long(100)); + stmt.bind(5, float(1000e0)); + stmt.bind(6, double(1000e0)); + stmt.bind(7, "Hello World!"); + + if ( stmt.execute() == sqldb::OK ) { + while( stmt.fetch_one() == sqldb::OK ) { + int8_t tiny = stmt.get<int8_t>(1); + int16_t small = stmt.get<int16_t>(2); + int32_t word = stmt.get<int32_t>(3); + int64_t huge = stmt.get<int64_t>(4); + float flt = stmt.get<float>(5); + double dbl = stmt.get<double>(6); + } + stmt.reset(); + stmt.finalize(); + } +} + +extern "C" int sqldb_run_tests(int, char**) { +#ifdef HAVE_MYSQL + test_db<mysql>(); +#endif +#ifdef HAVE_SQLITE + test_db<sqlite>(); +#endif + return 0; +} -- GitLab From afcba296029d1fbbad3036d8fa15e7f80676b056 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Fri, 30 Apr 2021 18:09:49 +0200 Subject: [PATCH 19/26] Save Controller and Storage changes. Merge Flavios commit --- TestBeam/job/DummyTask.py | 250 +++++++++++++++++++++++++++ TestBeam/job/Monitor.py | 42 +++++ TestBeam/options/DataflowArch_TB.xml | 34 ++++ TestBeam/options/MBM.opts | 11 ++ TestBeam/options/OnlineEnv.opts | 11 ++ TestBeam/options/OnlineEnvBase.py | 10 ++ TestBeam/options/Producer.opts | 38 ++++ 7 files changed, 396 insertions(+) create mode 100755 TestBeam/job/DummyTask.py create mode 100644 TestBeam/job/Monitor.py create mode 100644 TestBeam/options/DataflowArch_TB.xml create mode 100644 TestBeam/options/MBM.opts create mode 100644 TestBeam/options/OnlineEnv.opts create mode 100644 TestBeam/options/OnlineEnvBase.py create mode 100644 TestBeam/options/Producer.opts diff --git a/TestBeam/job/DummyTask.py b/TestBeam/job/DummyTask.py new file mode 100755 index 000000000..c7db34b13 --- /dev/null +++ b/TestBeam/job/DummyTask.py @@ -0,0 +1,250 @@ +""" + Package module + + \author M.Frank + \version 1.0 + +""" +from __future__ import print_function +from builtins import str +from builtins import range +from builtins import object +import os +import sys +import time +# import _fifo_log +import copy +import pydim +import threading + + +VERBOSE = 1 +DEBUG = 2 +INFO = 3 +WARNING = 4 +ERROR = 5 +ALWAYS = 6 + +global minPrintLevel +minPrintLevel = INFO + +utgid = '' +if 'UTGID' in os.environ: + utgid = os.environ['UTGID']+': ' + +# ------------------------------------------------------------------------------ +def log(level,msg): + """ + + \author M.Frank + \version 1.0 + \date 30/06/2002 + """ + import sys + try: + f = sys.stdout + if level < minPrintLevel: + return + elif level == DEBUG: + print('DEBUG %-24s %s'%(utgid, msg, ), file=f) + elif level == INFO: + print('INFO %-24s %s'%(utgid, msg, ), file=f) + elif level == WARNING: + print('WARNING %-24s %s'%(utgid, msg, ), file=f) + elif level == ERROR: + print('ERROR %-24s %s'%(utgid, msg, ), file=f) + elif level == ALWAYS: + print('SUCCESS %-24s %s'%(utgid, msg, ), file=f) + elif level > ALWAYS: + print('INFO %-24s %s'%(utgid, msg, ), file=f) + f.flush() + except Exception as X: + print('Exception failure: '+str(X)) + +#---------------------------------------------------------------------------------------- +class Monitor(object): + def __init__(self, partitionid): + self.done = int(time.time()) + self.last = int(time.time()) + self.pid = os.getpid() + self.partID = partitionid + +#---------------------------------------------------------------------------------------- +class FSM(object): + """ + No need to take locks. Everything is protected by the global DIM lock. + """ + ST_OFFLINE = 'OFFLINE' + ST_UNKNOWN = 'UNKNOWN' + ST_NOT_READY = 'NOT_READY' + ST_READY = 'READY' + ST_RUNNING = 'RUNNING' + ST_STOPPED = 'READY' + ST_PAUSED = 'PAUSED' + ST_ERROR = 'ERROR' + Meta = {ST_UNKNOWN: 'U', ST_NOT_READY: 'N', ST_READY: 'r', ST_RUNNING: 'R', ST_STOPPED: 'r', ST_PAUSED: 'r', ST_ERROR: 'E'} + CMD_CONFIGURE = 'configure' + CMD_START = 'start' + CMD_STOP = 'stop' + CMD_RESET = 'reset' + CMD_CONTINUE = 'continue' + CMD_PAUSE = 'pause' + CMD_LOAD = 'load' + CMD_UNLOAD = 'unload' + CMD_START_TRIGGER = 'start_trigger' + CMD_STOP_TRIGGER = 'stop_trigger' + + CMD_GET_STATE = '!state' + CMD_FORCE_RESET = 'RESET' + + #-------------------------------------------------------------------------------------- + def __init__(self, name, partitionid=0, auto=False): + self.monitor = Monitor(partitionid) + self.__name = name + self.__auto = auto + self.__keep = True + self.__state = FSM.ST_NOT_READY + self.previousState = FSM.ST_UNKNOWN + self.__cmdID = pydim.dis_add_cmnd(name,'C',self.callback,1) + self.__svcID = pydim.dis_add_service(name+'/status', 'C', self.service, 1) + self.__fsmID = pydim.dis_add_service(name+'/fsm_status','L:2;I:1;C:4;I:1',self.fsm_service, 4) + log(INFO,'DIM Command starting.... ') + pydim.dis_update_service(self.__svcID, (self.__state,)) + pydim.dis_update_service(self.__fsmID, self.fsm_service(self)) + log(INFO,'DIM Command starting....Done ') + + #-------------------------------------------------------------------------------------- + def state(self): + return self.__state + + #-------------------------------------------------------------------------------------- + def keepAlive(self): + return self.__keep + + #-------------------------------------------------------------------------------------- + def start(self): + log(INFO,"Start serving DIM....") + pydim.dis_start_serving(self.__name) + ## self.handleCommand(FSM.CMD_LOAD) + if self.__auto: + self.handleCommand(FSM.CMD_CONFIGURE) + self.handleCommand(FSM.CMD_START) + + #-------------------------------------------------------------------------------------- + def service(self,tag): + return self.__state + + #-------------------------------------------------------------------------------------- + def inState(self, states): + if type(states) is list: + for s in states: + if self.__state == s: return True + elif type(states) is tuple: + for s in states: + if self.__state == s: return True + elif type(states) is set: + for s in states: + if self.__state == s: return True + elif self.__state == states: + return True + return False + + #-------------------------------------------------------------------------------------- + def fsm_service(self,tag): + meta = FSM.Meta[self.__state] + self.monitor.metastate = meta+meta+'S'+' ' + self.monitor.last = self.monitor.done + self.monitor.done = int(time.time()) + result = (int(self.monitor.last), + int(self.monitor.done), + int(self.monitor.pid), + self.monitor.metastate, + int(self.monitor.partID),) + log(INFO,'+++ Publish FSM state: %-16s metastate: %s'%(str(self.__state),str(result),)) + return result + + def _setState(self, new_state): + self.previousState = self.__state + self.__state = new_state + + #-------------------------------------------------------------------------------------- + def handleCommand(self,cmd): + self.monitor.last = int(time.time()) + log(INFO,'+++ Handle DIM Command: '+str(cmd)) + sys.stdout.flush() + cb = 'handle'+cmd[0].upper()+cmd[1:] + if hasattr(self,cb): + if not getattr(self,cb)(): + self.state = FSM.ST_ERROR + if hasattr(self,'on'+FSM.ST_ERROR): + getattr(self,'on'+FSM.ST_ERROR)() + cmd = '' + + if cmd == FSM.CMD_CONFIGURE: + self._setState(FSM.ST_READY) + elif cmd == FSM.CMD_START: + self._setState(FSM.ST_RUNNING) + elif cmd == FSM.CMD_STOP: + self._setState(FSM.ST_STOPPED) + elif cmd == FSM.CMD_RESET: + self._setState(FSM.ST_NOT_READY) + elif cmd == FSM.CMD_CONTINUE: + self._setState(FSM.ST_RUNNING) + elif cmd == FSM.CMD_PAUSE: + self._setState(FSM.ST_PAUSED) + elif cmd == FSM.CMD_LOAD: + self._setState(FSM.ST_NOT_READY) + elif cmd == FSM.CMD_UNLOAD: + self._setState(FSM.ST_UNKNOWN) + self.__keep = False + elif cmd == FSM.CMD_FORCE_RESET: + self._setState(FSM.ST_UNKNOWN) + self.__keep = False + elif cmd == FSM.CMD_GET_STATE: + pass + elif cmd == FSM.CMD_STOP_TRIGGER: + return + elif cmd == FSM.CMD_START_TRIGGER: + return + else: + self._setState(FSM.ST_ERROR) + + if hasattr(self,'on'+self.__state): + getattr(self,'on'+self.__state)() + + pydim.dis_update_service(self.__svcID,(self.__state,)) + if self.__fsmID != 0: pydim.dis_update_service(self.__fsmID) + + + #-------------------------------------------------------------------------------------- + def callback(self, *args): + log(DEBUG,'++ FSM status callback. Args are %s'%str(args)) + r = args[0][0] + r = r[:r.find('\0')] + self.handleCommand(r) + +def runVoidTask(name=None,auto=False): + global minPrintLevel + utgid = '' + partition = 0x103 + if name is not None: utgid = name + for i in range(len(sys.argv)): + a = sys.argv[i] + if a[:1] == '-a': auto = True + if a[:2] == '-u': utgid = sys.argv[i+1] + if a[:4] == '-par': partition = int(sys.argv[i+1]) + if a[:4] == '-pri': minPrintLevel = int(sys.argv[i+1]) + if not len(utgid) and 'UTGID' in os.environ: + utgid = os.environ['UTGID'] + log(INFO,"FSM task: utgid: %s auto: %s print level: %d"%(utgid,str(auto), minPrintLevel,)) + fsm = FSM(name=utgid, partitionid=partition, auto=auto) + pydim.dis_start_serving(utgid) + ## fsm.handleCommand(FSM.CMD_LOAD) + fsm.start() + while fsm.keepAlive(): + time.sleep(1) + +if __name__ == "__main__": + import pdb + #pdb.set_trace() + runVoidTask() diff --git a/TestBeam/job/Monitor.py b/TestBeam/job/Monitor.py new file mode 100644 index 000000000..860ae6801 --- /dev/null +++ b/TestBeam/job/Monitor.py @@ -0,0 +1,42 @@ +""" + Minimal Gaudi task in the online environment + + + @author M.Frank +""" +from __future__ import print_function +__version__ = "1.0" +__author__ = "Markus Frank <Markus.Frank@cern.ch>" + +import os +import sys +import Configurables +import Gaudi.Configuration as Gaudi + +import GaudiOnline +import OnlineEnvBase as OnlineEnv + +from Configurables import Online__FlowManager as FlowManager +application = GaudiOnline.Passthrough(outputLevel=OnlineEnv.OutputLevel, + partitionName=OnlineEnv.PartitionName, + partitionID=OnlineEnv.PartitionID, + classType=GaudiOnline.Class1) +application.setup_fifolog() +application.setup_mbm_access('Input', True) +writer = application.setup_mbm_output('EventOutput') +writer.MBM_maxConsumerWait = 10 +application.setup_hive(FlowManager("EventLoop"), 40) +application.setup_algorithms(writer, 0.05) +application.setup_monitoring() +application.monSvc.DimUpdateInterval = 1 +application.config.numEventThreads = 15 +application.config.MBM_numConnections = 8 +application.config.MBM_numEventThreads = 5 +application.config.MBM_requests = [ + 'EvType=3;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0', + 'EvType=2;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0', + 'EvType=1;TriggerMask=0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF,0xFFFFFFFF;VetoMask=0,0,0,0;MaskType=ANY;UserType=ONE;Frequency=PERC;Perc=100.0' +] +# +print('Setup complete....') + diff --git a/TestBeam/options/DataflowArch_TB.xml b/TestBeam/options/DataflowArch_TB.xml new file mode 100644 index 000000000..46c0eff0c --- /dev/null +++ b/TestBeam/options/DataflowArch_TB.xml @@ -0,0 +1,34 @@ +<tasks_inventory> + + <task name="MBM" user="online" group="onliners"> + <command>/home/online/TestBeam/job/runTask.sh</command> + <argument name="-type" value="${NAME}"/> + <argument name="-runinfo" value="${RUNINFO}"/> + <argument name="-partition" value="${PARTITION}"/> + <fmcparam name="utgid" value="${PARTITION}_${NODE}_${NAME}_${INSTANCE}"/> + <fmcparam name="wd" value="/"/> + <timeout action="Any" value="20"/> + </task> + + <task name="Producer" user="online" group="onliners"> + <command>/home/online/TestBeam/job/runTask.sh</command> + <argument name="-type" value="${NAME}"/> + <argument name="-runinfo" value="${RUNINFO}"/> + <argument name="-partition" value="${PARTITION}"/> + <fmcparam name="utgid" value="${PARTITION}_${NODE}_${NAME}_${INSTANCE}"/> + <fmcparam name="wd" value="/"/> + <timeout action="Any" value="30"/> + </task> + + <task name="Monitor" user="online" group="onliners" instances="NUMBER_OF_INSTANCES"> + <command>/home/online/TestBeam/job/runTask.sh</command> + <argument name="-type" value="${NAME}"/> + <argument name="-runinfo" value="${RUNINFO}"/> + <argument name="-partition" value="${PARTITION}"/> + <fmcparam name="utgid" value="${PARTITION}_${NODE}_${NAME}_${INSTANCE}"/> + <fmcparam name="wd" value="/"/> + <timeout action="Any" value="30"/> + <timeout action="load" value="30"/> + </task> + +</tasks_inventory> diff --git a/TestBeam/options/MBM.opts b/TestBeam/options/MBM.opts new file mode 100644 index 000000000..c12e282a9 --- /dev/null +++ b/TestBeam/options/MBM.opts @@ -0,0 +1,11 @@ +#pragma print off +#include "OnlineEnv.opts" +#include "$FARMCONFIGROOT/options/Logging.opts" +OnlineEnv.MBM_setup = "-s=1000000 -e=50 -u=30 -b=18 -t=1 -y -i=Input -f -c -s=10000 -e=50 -u=30 -b=12 -t=1 -y -i=Output -f -c"; +// +Manager.Setup = {"Dataflow_MBMServer/MEPManager"}; +Manager.Services = {"Dataflow_UI/UI"}; +// +MEPManager.PartitionBuffers = true; +MEPManager.PartitionName = @OnlineEnv.PartitionName; +MEPManager.InitFlags = @OnlineEnv.MBM_setup; diff --git a/TestBeam/options/OnlineEnv.opts b/TestBeam/options/OnlineEnv.opts new file mode 100644 index 000000000..0c1dedc72 --- /dev/null +++ b/TestBeam/options/OnlineEnv.opts @@ -0,0 +1,11 @@ +// +// ---------------- General partition parameters: +OnlineEnv.PartitionID = 65535; +OnlineEnv.PartitionIDName = "FFFF"; +OnlineEnv.PartitionName = "TEST"; +OnlineEnv.Activity = "Upgrade"; +OnlineEnv.TAE = 0; +OnlineEnv.OutputLevel = 3; +OnlineEnv.AcceptRate = 1.0; +OnlineEnv.OnlineVersion = "None"; +OnlineEnv.passThroughDelay = 0; diff --git a/TestBeam/options/OnlineEnvBase.py b/TestBeam/options/OnlineEnvBase.py new file mode 100644 index 000000000..ea02b39fd --- /dev/null +++ b/TestBeam/options/OnlineEnvBase.py @@ -0,0 +1,10 @@ +# ---------------- General partition parameters: +PartitionID = 65535 +PartitionIDName = "FFFF" +PartitionName = "TEST" +Activity = "Upgrade" +OnlineVersion = "v0" +MooreVersion = "v0" +MooreOnlineVersion = "v0" +MooreStartupMode = 0 +OutputLevel = 3 diff --git a/TestBeam/options/Producer.opts b/TestBeam/options/Producer.opts new file mode 100644 index 000000000..1abdf9de7 --- /dev/null +++ b/TestBeam/options/Producer.opts @@ -0,0 +1,38 @@ +#pragma print off +#include "OnlineEnv.opts" +#include "$FARMCONFIGROOT/options/Logging.opts" +#include "$FARMCONFIGROOT/options/Monitoring.opts" +// +Manager.Services = {"Dataflow_MBMClient/MEPManager", + "Dataflow_BurstReader/Reader", + "Dataflow_RunableWrapper/Wrap", + "Dataflow_UI/UI" + }; +Manager.Runable = "Wrap"; +Wrap.Callable = "Reader"; +Task.HavePause = true; +// +Reader.Buffer = "Input"; +Reader.BrokenHosts = ""; +Reader.Directories = {"/home/frankm/data"}; +Reader.FilePrefix = "fakeMDF_"; +Reader.AllowedRuns = {"*"}; +Reader.MuDelay = 0; +Reader.DeleteFiles = false; +Reader.SaveRest = false; +Reader.PauseSleep = 2; // Optional wait time until 'Output' event queue is empty +Reader.InitialSleep = 0; +Reader.MaxPauseWait = 1; +Reader.GoService = ""; +Reader.Rescan = 1; +Reader.RequireConsumers = 0; +Reader.MMapFiles = 0; +Reader.ReuseFile = 1; +Reader.PackingFactor = 1000; +Reader.AllocationSizekB = 10000; +// +MEPManager.PartitionBuffers = true; +MEPManager.PartitionName = @OnlineEnv.PartitionName; +MEPManager.PartitionID = @OnlineEnv.PartitionID; +MEPManager.Buffers = {"Input"}; + -- GitLab From f7685a42161a49541a7ad8ecdcc66a54869b8f21 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Tue, 4 May 2021 12:43:25 +0200 Subject: [PATCH 20/26] Merge Flavios latest MR --- Online/EventBuilding/options/EB_BU_algo.opts | 6 +- Online/EventBuilding/options/EB_RU_algo.opts | 16 +- .../options/EB_Transport_properties.opts | 4 +- Online/EventBuilding/options/MFPGen.opts | 11 + .../options/MFP_generator_params.opts | 4 +- Online/sqldb/CMakeLists.txt | 40 +- Online/sqldb/sqldb/mysql.h | 6 +- Online/sqldb/sqldb/postgresql.h | 36 ++ Online/sqldb/sqldb/sqldb.h | 4 +- Online/sqldb/src/mysql.cpp | 17 +- Online/sqldb/src/oracle.cpp | 6 +- Online/sqldb/src/postgresql.cpp | 488 ++++++++++++++++++ Online/sqldb/src/sqlite.cpp | 6 +- Online/sqldb/test/sqldb_test.cpp | 12 + cmake/FindMySQL.cmake | 33 ++ cmake/FindPostgreSQL.cmake | 32 ++ 16 files changed, 684 insertions(+), 37 deletions(-) create mode 100755 Online/EventBuilding/options/MFPGen.opts create mode 100644 Online/sqldb/sqldb/postgresql.h create mode 100644 Online/sqldb/src/postgresql.cpp create mode 100644 cmake/FindMySQL.cmake create mode 100644 cmake/FindPostgreSQL.cmake diff --git a/Online/EventBuilding/options/EB_BU_algo.opts b/Online/EventBuilding/options/EB_BU_algo.opts index 9a57a1f49..308ea7c2c 100755 --- a/Online/EventBuilding/options/EB_BU_algo.opts +++ b/Online/EventBuilding/options/EB_BU_algo.opts @@ -1,7 +1,7 @@ -BU.MBM_name = { 'Input', 'Input', 'Input', 'Input' }; +BU.MBM_name = { 'Input', 'Input' }; BU.buffer_size = {}; -BU.buffer_type = { 2, 2, 2, 2 }; +BU.buffer_type = { 2, 2 }; BU.out_file_prefix = ""; BU.shmem_prefix = "BU_"; BU.stop_timeout = 60; -BU.write_to_file = { FALSE, FALSE, FALSE, FALSE }; +BU.write_to_file = { FALSE, FALSE }; diff --git a/Online/EventBuilding/options/EB_RU_algo.opts b/Online/EventBuilding/options/EB_RU_algo.opts index 1fd2b0b01..0d55e62df 100644 --- a/Online/EventBuilding/options/EB_RU_algo.opts +++ b/Online/EventBuilding/options/EB_RU_algo.opts @@ -1,7 +1,9 @@ -BU.MBM_name = { 'Input', 'Input', 'Input', 'Input' }; -BU.buffer_size = {}; -BU.buffer_type = { 2, 2, 2, 2 }; -BU.out_file_prefix = ""; -BU.shmem_prefix = ""; -BU.stop_timeout = 60; -BU.write_to_file = { FALSE, FALSE, FALSE, FALSE }; +RU.MDF_filename = "/home/fpisani/mc_data/biger_file.mdf"; +RU.PCIe40_ids = {}; +RU.PCIe40_names = { 'tdtel051_1', 'tdtel052_0', 'tdtel052_1', 'tdtel053_0', 'tdtel053_1' }; +RU.buffer_sizes = {}; +RU.buffer_type = { 1 }; +RU.n_MFPs = 100; +RU.n_fragment = 3000; +RU.shmem_prefix = "RU_"; +RU.stop_timeout = 60; diff --git a/Online/EventBuilding/options/EB_Transport_properties.opts b/Online/EventBuilding/options/EB_Transport_properties.opts index a0a3fd72c..876ec9a89 100755 --- a/Online/EventBuilding/options/EB_Transport_properties.opts +++ b/Online/EventBuilding/options/EB_Transport_properties.opts @@ -2,8 +2,8 @@ EB_transport.BU_ranks = {}; EB_transport.MPI_errors_return = FALSE; EB_transport.RU_ranks = {}; EB_transport.RUs_per_nic = {}; -EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.J22931.json"; +EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.LhX721.json"; EB_transport.n_par_mess = 0; -EB_transport.n_sources_per_ru = { 3, 2, 2, 2 }; +EB_transport.n_sources_per_ru = { 3 }; EB_transport.shift_pattern = {}; EB_transport.src_ids = {}; diff --git a/Online/EventBuilding/options/MFPGen.opts b/Online/EventBuilding/options/MFPGen.opts new file mode 100755 index 000000000..5dd523382 --- /dev/null +++ b/Online/EventBuilding/options/MFPGen.opts @@ -0,0 +1,11 @@ +#pragma print on +#include "$INFO_OPTIONS" +#include "$EVENTBUILDINGROOT/options/EB_transport.opts" +#include "$FARMCONFIG_OPTIONS/Logging.opts" + +Manager.Services = {"MFP_generator/MFP_generator", + "Dataflow_RunableWrapper/Wrap" + }; +Manager.Runable = "Wrap"; +Wrap.Callable = "MFP_generator"; +#include "$EVENTBUILDINGROOT/options/MFP_generator_params.opts" diff --git a/Online/EventBuilding/options/MFP_generator_params.opts b/Online/EventBuilding/options/MFP_generator_params.opts index e5f19ea36..fe3d45721 100644 --- a/Online/EventBuilding/options/MFP_generator_params.opts +++ b/Online/EventBuilding/options/MFP_generator_params.opts @@ -2,7 +2,7 @@ MFP_generator.OutputLevel = 3; MFP_generator.bank_size = 100; MFP_generator.event_rate = 300000000; MFP_generator.n_banks = 30000; +MFP_generator.numa_layout = { 4, 2 }; MFP_generator.random_sizes = FALSE; MFP_generator.reconnect = FALSE; -MFP_generator.shmem_prefix = "RU"; -MFP_generator.numa_layout = {}; +MFP_generator.shmem_prefix = "RU_"; diff --git a/Online/sqldb/CMakeLists.txt b/Online/sqldb/CMakeLists.txt index b71ca29c3..c68b6a4ab 100644 --- a/Online/sqldb/CMakeLists.txt +++ b/Online/sqldb/CMakeLists.txt @@ -9,16 +9,17 @@ # Author : M.Frank # #=============================================================================== -################################################################################ +# # Package: sqldb # gaudi_subdir(sqldb v0r1) -################################################################################ +# --------------------------------------------------------------------------------------- # find_package(Boost REQUIRED) find_package(SQLite3 REQUIRED) +find_package(Oracle) +find_package(PostgreSQL) find_package(MySQL) -find_package(Oracle REQUIRED) # include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) # @@ -26,18 +27,39 @@ gaudi_add_library(sqldb_sqlite src/sqlite.cpp PUBLIC_HEADERS sqldb INCLUDE_DIRS Boost SQLite3 LINK_LIBRARIES SQLite3) +target_compile_definitions(sqldb_sqlite PRIVATE -DSQLDB_HAVE_SQLITE=1) +# +# --------------------------------------------------------------------------------------- +# Oracle interface +# --------------------------------------------------------------------------------------- +if ( ORACLE_FOUND ) + gaudi_add_library(sqldb_oracle src/oracle.cpp + NO_PUBLIC_HEADERS + INCLUDE_DIRS Boost Oracle + LINK_LIBRARIES Oracle) + target_compile_definitions(sqldb_oracle PRIVATE -DSQLDB_HAVE_ORACLE=1) +endif() +# +# --------------------------------------------------------------------------------------- +# PostgreSQL interface +# --------------------------------------------------------------------------------------- +if ( PostgreSQL_FOUND ) + gaudi_add_library(sqldb_postgresql src/postgresql.cpp + NO_PUBLIC_HEADERS + INCLUDE_DIRS Boost PostgreSQL ${PostgreSQL_INCLUDE_DIR} + LINK_LIBRARIES PostgreSQL) + target_compile_definitions(sqldb_postgresql PRIVATE -DSQLDB_HAVE_POSTGRESQL=1) +endif() # +# --------------------------------------------------------------------------------------- +# MySQL interface +# --------------------------------------------------------------------------------------- if ( MySQL_FOUND ) gaudi_add_library(sqldb_mysql src/mysql.cpp NO_PUBLIC_HEADERS INCLUDE_DIRS Boost MySQL LINK_LIBRARIES MySQL) -endif() -if ( ORACLE_FOUND ) - gaudi_add_library(sqldb_oracle src/oracle.cpp - NO_PUBLIC_HEADERS - INCLUDE_DIRS Boost Oracle - LINK_LIBRARIES Oracle) + target_compile_definitions(sqldb_mysql PRIVATE -DSQLDB_HAVE_MYSQL=1) endif() # --------------------------------------------------------------------------------------- # Testing diff --git a/Online/sqldb/sqldb/mysql.h b/Online/sqldb/sqldb/mysql.h index 0abc46b4d..64cecb117 100644 --- a/Online/sqldb/sqldb/mysql.h +++ b/Online/sqldb/sqldb/mysql.h @@ -12,8 +12,8 @@ // // Author : Markus Frank //========================================================================== -#ifndef ONLINEBASE_CPP_MYSQL_INTERFACE_H -#define ONLINEBASE_CPP_MYSQL_INTERFACE_H +#ifndef SQLDB_MYSQL_INTERFACE_H +#define SQLDB_MYSQL_INTERFACE_H /// Framework include files #include <sqldb/sqldb.h> @@ -31,4 +31,4 @@ class mysql : public sqldb::traits { struct statement_t; }; -#endif // ONLINEBASE_CPP_MYSQL_INTERFACE_H +#endif // SQLDB_MYSQL_INTERFACE_H diff --git a/Online/sqldb/sqldb/postgresql.h b/Online/sqldb/sqldb/postgresql.h new file mode 100644 index 000000000..a7b20dfc3 --- /dev/null +++ b/Online/sqldb/sqldb/postgresql.h @@ -0,0 +1,36 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Package : RPC +// +// Author : Markus Frank +//========================================================================== +#ifndef SQLDB_POSTGRESQL_INTERFACE_H +#define SQLDB_POSTGRESQL_INTERFACE_H + +/// Framework include files +#include <sqldb/sqldb.h> + +typedef struct pg_conn PGconn; + +/// Technology abstraction layer +/** + * + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ +class postgresql : public sqldb::traits { + public: + typedef PGconn dbase_t; + struct statement_t; +}; + +#endif // SQLDB_POSTGRESQL_INTERFACE_H diff --git a/Online/sqldb/sqldb/sqldb.h b/Online/sqldb/sqldb/sqldb.h index f291f0c52..97d1aaf79 100644 --- a/Online/sqldb/sqldb/sqldb.h +++ b/Online/sqldb/sqldb/sqldb.h @@ -223,8 +223,8 @@ namespace sqldb { class time time; } item; int type { TYPE_NULL }; - bool null { false }; - bool error { false }; + char null { 0 }; + char error { 0 }; /// Constructor column_data() { diff --git a/Online/sqldb/src/mysql.cpp b/Online/sqldb/src/mysql.cpp index 697745b27..5e458a2a8 100644 --- a/Online/sqldb/src/mysql.cpp +++ b/Online/sqldb/src/mysql.cpp @@ -17,7 +17,7 @@ #include <sqldb/mysql.h> /// MySQL header -#include <mysql/mysql.h> +#include <mysql.h> struct mysql::dbase_t : public MYSQL { }; struct mysql::statement_t : public MYSQL_STMT { }; @@ -77,7 +77,7 @@ namespace sqldb { public: /// Bind parameter to null value int bind_store_null(column_data* d) { - d->null = true; + d->null = 1; this->is_null = &d->null; this->buffer = nullptr; this->length = nullptr; @@ -277,8 +277,7 @@ namespace sqldb { virtual int bind(size_t col, field_type typ, const void* data, size_t len) override { pointers_t ptr(data); auto [d, b] = this->param(col); - - ::memset(&b, 0, sizeof(column_bind)); + ::memset(b, 0, sizeof(column_bind)); d->type = typ; b->buffer_type = mysql_field_type(typ); switch(typ) { @@ -292,7 +291,7 @@ namespace sqldb { case TYPE_STRING: return b->bind_store(d, ptr.puint8, len); case TYPE_BLOB: return b->bind_store(d, ptr.puint8, len); /// For now store time stamps as long integers (time_t) - case TYPE_TIME: return b->bind_store(d, d->item.time.stamp, ptr.puint64); + case TYPE_TIME: return b->bind_store(d, d->item.time.stamp, ptr.pint64); default: return invalid_statement("Invalid MySQL parameter data type"); } return sqldb::OK; @@ -384,7 +383,7 @@ namespace sqldb { return sqldb::INVALID_HANDLE; } /// Prepare a new statement and allocate resources. - int prepare(statement& bs, const std::string& sql_str) { + int prepare(statement& bs, const std::string& sql_str) override { auto* stm_imp = new statement_backend(); bs.intern.reset(stm_imp); return stm_imp->prepare(this->handle(), sql_str); @@ -410,7 +409,11 @@ namespace sqldb { dbase_backend() = default; }; } +} +/// SQLDB namespace declaration +namespace sqldb { + /// Open the database connection using all information in passed string template <> std::pair<std::shared_ptr<database::backend>, int> @@ -419,6 +422,6 @@ namespace sqldb { int ret = db->open(connect_string); if ( ret == sqldb::OK ) return make_pair(db, sqldb::OK); - return make_pair(std::make_shared<mysql_imp::dbase_backend>(), ret); + return make_pair(std::shared_ptr<database::backend>(), ret); } } // End namespace sqldb diff --git a/Online/sqldb/src/oracle.cpp b/Online/sqldb/src/oracle.cpp index eb9ba90b8..d8b2d73ec 100644 --- a/Online/sqldb/src/oracle.cpp +++ b/Online/sqldb/src/oracle.cpp @@ -632,6 +632,10 @@ namespace sqldb { return sqldb::ERROR; } } +} + +/// SQLDB namespace declaration +namespace sqldb { /// Open the database connection using all information in passed string template <> std::pair<std::shared_ptr<database::backend>, int> @@ -640,6 +644,6 @@ namespace sqldb { int ret = db->open(connect_string); if ( ret == sqldb::OK ) return make_pair(db, sqldb::OK); - return make_pair(std::make_shared<oracle_imp::dbase_backend>(), ret); + return make_pair(std::shared_ptr<database::backend>(), ret); } } // End namespace sqldb diff --git a/Online/sqldb/src/postgresql.cpp b/Online/sqldb/src/postgresql.cpp new file mode 100644 index 000000000..cad061843 --- /dev/null +++ b/Online/sqldb/src/postgresql.cpp @@ -0,0 +1,488 @@ +//========================================================================== +// LHCb Online software suite +//-------------------------------------------------------------------------- +// Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN) +// All rights reserved. +// +// For the licensing terms see OnlineSys/LICENSE. +// +//-------------------------------------------------------------------------- +// +// Package : RPC +// +// Author : Markus Frank +//========================================================================== + +/// Framework include files +#include <sqldb/postgresql.h> + +#if defined(SQLDB_HAVE_POSTGRESQL) + +/// MySQL header +#include <libpq-fe.h> + +//struct postgresql::statement_t : public POSTGRESQL_STMT { }; + +/// SQLDB namespace declaration +namespace sqldb { + + namespace postgresql_imp { + /// + inline int postgresql_ok(const PGresult* status) { + if ( status ) { + ExecStatusType sc = ::PQresultStatus(status); + switch( sc ) { + case PGRES_COMMAND_OK: + case PGRES_TUPLES_OK: + case PGRES_COPY_OUT: + case PGRES_COPY_IN: + case PGRES_EMPTY_QUERY: + return sqldb::OK; + case PGRES_NONFATAL_ERROR: + return sqldb::OK; + case PGRES_BAD_RESPONSE: + case PGRES_FATAL_ERROR: + default: + return sqldb::ERROR; + } + } + return sqldb::ERROR; + } +#if 0 + /// + inline field_type gen_field_type(enum_field_types typ) { + switch(typ) { + case POSTGRESQL_TYPE_TINY: return TYPE_TINY; + case POSTGRESQL_TYPE_SHORT: return TYPE_SHORT; + case POSTGRESQL_TYPE_LONG: return TYPE_LONG; + case POSTGRESQL_TYPE_LONGLONG: return TYPE_LONGLONG; + case POSTGRESQL_TYPE_FLOAT: return TYPE_FLOAT; + case POSTGRESQL_TYPE_DOUBLE: return TYPE_DOUBLE; + case POSTGRESQL_TYPE_STRING: return TYPE_STRING; + case POSTGRESQL_TYPE_BLOB: return TYPE_BLOB; + case POSTGRESQL_TYPE_TIME: return TYPE_TIME; + case POSTGRESQL_TYPE_DATE: return TYPE_TIME; + case POSTGRESQL_TYPE_DATETIME: return TYPE_TIME; + default: + break; + } + throw std::runtime_error("Invalid data type encountered!"); + } + /// + inline enum_field_types postgresql_field_type(field_type typ) { + switch(typ) { + case TYPE_TINY: return POSTGRESQL_TYPE_TINY; + case TYPE_SHORT: return POSTGRESQL_TYPE_SHORT; + case TYPE_LONG: return POSTGRESQL_TYPE_LONG; + case TYPE_LONGLONG: return POSTGRESQL_TYPE_LONGLONG; + case TYPE_FLOAT: return POSTGRESQL_TYPE_FLOAT; + case TYPE_DOUBLE: return POSTGRESQL_TYPE_DOUBLE; + case TYPE_STRING: return POSTGRESQL_TYPE_STRING; + case TYPE_BLOB: return POSTGRESQL_TYPE_BLOB; + case TYPE_TIME: return POSTGRESQL_TYPE_DATETIME; + default: + break; + } + throw std::runtime_error("Invalid data type encountered!"); + } + + /// Technology abstraction layer + /** + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + struct column_bind : public POSTGRESQL_BIND { + public: + /// Bind parameter to null value + int bind_store_null(column_data* d) { + d->null = true; + this->is_null = &d->null; + this->buffer = nullptr; + this->length = nullptr; + this->error = nullptr; + return sqldb::OK; + } + /// Generic parameter binding + int bind_store(column_data* d, const unsigned char* ptr, size_t len) { + if ( ptr ) { + d->bind_pointer(ptr, len); + this->buffer = d->item.string; + this->length = &d->length; + this->is_null = nullptr; + this->error = nullptr; + return sqldb::OK; + } + return this->bind_store_null(d); + } + /// Type specific parameter bind + template <typename T> int bind_store(column_data* d, T& to, const T* from) { + if ( from ) { + to = *from; + d->length = sizeof(T); + this->buffer = &to; + this->length = nullptr; + this->is_null = nullptr; + this->error = nullptr; + return sqldb::OK; + } + return this->bind_store_null(d); + } + /// Bind output field + int bind_fetch(column_data* d, void* to, size_t len) { + d->null = false; + d->error = false; + d->length = 0; + this->buffer = to; + this->length = &d->length; + this->is_null = &d->null; + this->error = &d->error; + this->buffer_length = len; + return sqldb::OK; + } + }; +#endif + /// Technology abstraction layer for a Postgresql prepared statement + /** + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + struct statement_backend : public statement::backend { + public: + postgresql::dbase_t* _handle { nullptr }; + + /// Access to the internal statement handle. Throws exception if the handle is invalid + postgresql::dbase_t* handle() const { + if ( this->_handle ) return this->_handle; + throw std::runtime_error("Failed to access statement handle. [Invalid handle]"); + } +#if 0 + typedef std::tuple<column_data*, column_bind*, POSTGRESQL_FIELD*> field_t; + std::vector<column_bind> _pbind, _fbind; + std::string _sql_string; + POSTGRESQL_STMT* _imp { nullptr }; + + std::pair<column_data*, column_bind*> param(int col) { + return std::make_pair(¶ms[col], &_pbind[col]); + } + field_t field(int col) { + return field_t(&this->fields[col], &_fbind[col], &_imp->fields[col]); + } +#endif + /// Access error message on failure + virtual std::string errmsg() override { + return ::PQerrorMessage(this->handle()); + } + /// Character string of the prepared statement + virtual const char* sql() override { + return "";//this->_sql_string.c_str(); + } + /// Access the number of rows affected by the last statement + virtual long rows_affected() override { + return 0;//::postgresql_stmt_num_rows(this->handle()); + } + + /// Prepare a new statement and allocate resources. + int prepare(postgresql::dbase_t* db, const std::string& sql_str) { +#if 0 + + POSTGRESQL_STMT* stmt = nullptr; + if ( _imp ) ::postgresql_stmt_reset(_imp); + if ( _imp ) ::postgresql_stmt_close(_imp); + stmt = ::postgresql_stmt_init(db); + if ( stmt ) { + int ret = ::postgresql_stmt_prepare(stmt, sql_str.c_str(), sql_str.length()); + if ( ret == sqldb::OK ) { + int num_param = ::postgresql_stmt_param_count(stmt); + int num_field = ::postgresql_stmt_field_count(stmt); + this->_pbind.resize(num_param, column_bind()); + this->params.resize(num_param, column_data()); + this->_fbind.resize(num_field, column_bind()); + this->fields.resize(num_field, column_data()); + this->_sql_string = sql_str; + this->_imp = stmt; + this->stmt = this; + return sqldb::OK; + } + return sqldb::ERROR; + } +#endif + return invalid_statement(); + } + + /// Execute prepared statement + virtual int execute() { +#if 0 + if( auto* h = this->_imp ) { + if( this->_pbind.size() ) { + if( ::postgresql_stmt_bind_param(h, &this->_pbind.front()) ) { + return invalid_statement(::postgresql_stmt_error(h)); + } + } + if( sqldb::OK == ::postgresql_stmt_execute(h) ) { + if( sqldb::OK != ::postgresql_stmt_store_result(h) ) { + if( auto* r = ::postgresql_stmt_result_metadata(h) ) { + ::postgresql_free_result(r); + return invalid_statement(::postgresql_stmt_error(h)); + } + } + return sqldb::OK; + } + return invalid_statement(::postgresql_stmt_error(h)); + } +#endif + return invalid_statement(); + } + + /// Fetch next result row. Stop if result is NON-zero + virtual int fetch_one() { +#if 0 + if ( this->_imp ) { + size_t nfld = this->_fbind.size(); + if ( nfld > 0 ) { + ::memset(&this->_fbind.front(), 0, nfld*sizeof(column_bind)); + for( size_t i=0; i < nfld; i++ ) { + this->bind_fetch(i); + if ( !postgresql_ok(::postgresql_stmt_bind_result(this->_imp, &this->_fbind.front())) ) + return invalid_statement(::postgresql_stmt_error(this->_imp)); + } + } + int ret = ::postgresql_stmt_fetch(this->_imp); + if( ret == POSTGRESQL_NO_DATA ) { + return sqldb::NO_DATA; + } + else if( ret == POSTGRESQL_DATA_TRUNCATED ) { + for( size_t i=0; i < nfld; i++ ) { + auto& d = this->fields[i]; + if( d.error && !d.null && d.length >= sizeof(d.buf) ) { + auto& b = this->_fbind[i]; + d.vbuf.resize(d.length); + d.item.blob = &d.vbuf.front(); + b.buffer = d.item.blob; + b.buffer_length = d.length; + if( !postgresql_ok(::postgresql_stmt_fetch_column(this->_imp,&b,i,0)) ) { + return invalid_statement(::postgresql_stmt_error(this->_imp)); + } + } + } + } + + return sqldb::OK; + } +#endif + return invalid_statement(); + } + + /// Bind output field identified by column number + int bind_fetch(size_t col) { +#if 0 + auto* p = this; + auto [d, b, f] = p->field(col); + + ::memset(b, 0, sizeof(column_bind)); + b->buffer_type = f->type; + d->type = gen_field_type(f->type); + switch(d->type) { + case TYPE_TINY: return b->bind_fetch(d, &d->item.int8, sizeof(d->item.int8)); + case TYPE_SHORT: return b->bind_fetch(d, &d->item.int16, sizeof(d->item.int16)); + case TYPE_LONG: return b->bind_fetch(d, &d->item.int32, sizeof(d->item.int32)); + case TYPE_LONGLONG: return b->bind_fetch(d, &d->item.int64, sizeof(d->item.int64)); + case TYPE_FLOAT: return b->bind_fetch(d, &d->item.real32, sizeof(d->item.real32)); + case TYPE_DOUBLE: return b->bind_fetch(d, &d->item.real64, sizeof(d->item.real64)); + case TYPE_STRING: return b->bind_fetch(d, d->buf, sizeof(d->buf)); + case TYPE_BLOB: return b->bind_fetch(d, d->buf, sizeof(d->buf)); + case TYPE_TIME: return b->bind_fetch(d, &d->item.time.stamp, sizeof(d->item.int64)); + default: return invalid_statement("Invalid Postgresql Field data type"); + } +#endif + return sqldb::OK; + } + + /// Bind a single column to the data passed + virtual int bind(size_t col, field_type typ, const void* data, size_t len) override { +#if 0 + pointers_t ptr(data); + auto [d, b] = this->param(col); + + ::memset(&b, 0, sizeof(column_bind)); + d->type = typ; + b->buffer_type = postgresql_field_type(typ); + switch(typ) { + case TYPE_NULL: return b->bind_store_null(d); + case TYPE_TINY: return b->bind_store(d, d->item.uint8, ptr.puint8); + case TYPE_SHORT: return b->bind_store(d, d->item.uint16, ptr.puint16); + case TYPE_LONG: return b->bind_store(d, d->item.uint32, ptr.puint32); + case TYPE_LONGLONG: return b->bind_store(d, d->item.uint64, ptr.puint64); + case TYPE_FLOAT: return b->bind_store(d, d->item.real32, ptr.preal32); + case TYPE_DOUBLE: return b->bind_store(d, d->item.real64, ptr.preal64); + case TYPE_STRING: return b->bind_store(d, ptr.puint8, len); + case TYPE_BLOB: return b->bind_store(d, ptr.puint8, len); + /// For now store time stamps as long integers (time_t) + case TYPE_TIME: return b->bind_store(d, d->item.time.stamp, ptr.puint64); + default: return invalid_statement("Invalid Postgresql parameter data type"); + } +#endif + return sqldb::OK; + } + + /// Action routine for various functions + int action(action_type type) override { + switch(type) { +#if 0 + case EXEC: /// Execute prepared statement + return this->execute(); + case FETCH: /// Fetch next result row. Stop if result is NON-zero + return this->fetch_one(); + case RESET: /// Reset all data to re-execute the statement with new parameters + if( auto* h = this->handle() ) { + if( !postgresql_ok(::postgresql_stmt_reset(h)) ) + return sqldb::ERROR; + } + return sqldb::OK; + case FINALIZE: /// Finalize and run-down the statement. Release all resources. + if ( this->_imp ) { + ::postgresql_stmt_reset(this->_imp); + ::postgresql_stmt_close(this->_imp); + this->_imp = nullptr; + } + return sqldb::OK; + case ERRNO: /// Access error number of last failed statement + return ::postgresql_stmt_errno(this->handle()); +#endif + default: + throw std::runtime_error("Invalid action request received!"); + } + } + }; + + /// Technology abstraction layer for a Postgresql database + /** + * \author M.Frank + * \version 1.0 + * \date 02.05.2017 + */ + struct dbase_backend : public database::backend { + postgresql::dbase_t* _handle { nullptr }; + ::PGresult* _result { nullptr }; + + /// Access to the internal database handle. Throws exception if the handle is invalid + postgresql::dbase_t* handle() const { + if ( _handle ) return _handle; + throw std::runtime_error("Invalid Postgresql database handle"); + } + /// Access error message on failure + virtual std::string errmsg() override { + if ( this->_result ) { + return ::PQresultErrorMessage(this->_result); + } + return ::PQerrorMessage(this->handle()); + } + /// Access the number of rows affected by the last statement + virtual long rows_affected() override { + if ( this->_result != nullptr ) { + return ::PQntuples(this->_result); + } + return -1; + } + void clear_result() { + if ( this->_result != nullptr ) { + ::PQclear(this->_result); + this->_result = nullptr; + } + } + /// Perform multiple actions + virtual int action(action_type type) override { + switch(type) { + case CLOSE: + clear_result(); + if ( this->_handle ) { + ::PQfinish(this->_handle); + this->_handle = nullptr; + } + return sqldb::OK; + case BEGIN: /// Start transaction + clear_result(); + this->_result = ::PQexec(this->handle(), "BEGIN"); + return postgresql_ok(this->_result); + case COMMIT: /// Commit transaction + clear_result(); + this->_result = ::PQexec(this->handle(), "COMMIT"); + return postgresql_ok(this->_result); + case ROLLBACK: /// Rollback transaction + clear_result(); + this->_result = ::PQexec(this->handle(), "ROLLBACK"); + return postgresql_ok(this->_result); + case ERRNO: /// Access error number of last failed statement + if ( this->_result != nullptr ) { + return ::PQresultStatus(this->_result); + } + return ::PQstatus(this->handle()); + default: + throw std::runtime_error("Invalid action request received!"); + } + } + /// Execute prepared statement + virtual int execute(std::string& err, const std::string& sql) override { + err = ""; + if( this->_handle ) { + clear_result(); + this->_result = ::PQexec(this->_handle, sql.c_str()); + if ( postgresql_ok(this->_result) == sqldb::OK ) { + return sqldb::OK; + } + if ( this->_result ) { + ExecStatusType status = ::PQresultStatus(this->_result); + err = ::PQresStatus(status); + return sqldb::ERROR; + } + err = "Fatal PostgreSQL error"; + return sqldb::ERROR; + } + err = "Invalid database connection"; + return sqldb::INVALID_HANDLE; + } + /// Prepare a new statement and allocate resources. + virtual int prepare(statement& bs, const std::string& sql_str) override { + auto* stm_imp = new statement_backend(); + bs.intern.reset(stm_imp); + return stm_imp->prepare(this->handle(), sql_str); + } + /// Open database + int open(const std::string& connect_string) { + auto args = connection_args().parse(connect_string); + this->_handle = ::PQsetdbLogin(args["host"].c_str(), + args["port"].c_str(), + args["options"].c_str(), + nullptr, // tty + args["database"].c_str(), + args["user"].c_str(), + args["password"].c_str()); + if ( this->_handle == nullptr ) { + return sqldb::ERROR; + } + this->name = args["database"]; + return sqldb::OK; + } + + /// Initializing constructor + dbase_backend() = default; + }; + } +} + +/// SQLDB namespace declaration +namespace sqldb { + /// Open the database connection using all information in passed string + template <> + std::pair<std::shared_ptr<database::backend>, int> + database::backend::open<postgresql>(const std::string& connect_string) { + auto db = std::make_shared<postgresql_imp::dbase_backend>(); + int ret = db->open(connect_string); + if ( ret == sqldb::OK ) + return make_pair(db, sqldb::OK); + return make_pair(std::shared_ptr<database::backend>(), ret); + } +} // End namespace sqldb + +#endif diff --git a/Online/sqldb/src/sqlite.cpp b/Online/sqldb/src/sqlite.cpp index 488e00afd..1f249f811 100644 --- a/Online/sqldb/src/sqlite.cpp +++ b/Online/sqldb/src/sqlite.cpp @@ -330,6 +330,10 @@ namespace sqldb { } }; } +} + +/// SQLDB namespace declaration +namespace sqldb { /// Open the database connection using all information in passed string template <> std::pair<std::shared_ptr<database::backend>, int> @@ -339,6 +343,6 @@ namespace sqldb { int ret = db->open(connect_string); if ( ret == sqldb::OK ) return make_pair(std::move(db), sqldb::OK); - return make_pair(std::make_shared<_imp_t>(), ret); + return make_pair(std::shared_ptr<database::backend>(), ret); } } // End namespace sqldb diff --git a/Online/sqldb/test/sqldb_test.cpp b/Online/sqldb/test/sqldb_test.cpp index 0138f36a7..108e88fd5 100644 --- a/Online/sqldb/test/sqldb_test.cpp +++ b/Online/sqldb/test/sqldb_test.cpp @@ -49,6 +49,18 @@ template <typename T> void test_db() { int64_t huge = stmt.get<int64_t>(4); float flt = stmt.get<float>(5); double dbl = stmt.get<double>(6); + if ( tiny == 0 ) { + } + if ( small == 0 ) { + } + if ( word == 0 ) { + } + if ( huge == 0 ) { + } + if ( flt == 0e0 ) { + } + if ( dbl == 0e0 ) { + } } stmt.reset(); stmt.finalize(); diff --git a/cmake/FindMySQL.cmake b/cmake/FindMySQL.cmake new file mode 100644 index 000000000..0bbdcf05c --- /dev/null +++ b/cmake/FindMySQL.cmake @@ -0,0 +1,33 @@ +# - Locate MySQL library +# Defines: +# +# MYSQL_FOUND +# MYSQL_INCLUDE_DIR +# MYSQL_INCLUDE_DIRS (not cached) +# MYSQL_LIBRARY +# MYSQL_LIBRARIES (not cached) +# MYSQL_LIBRARY_DIRS (not cached) +# MYSQL_EXECUTABLE +# +SET(MYSQL_DIR /cvmfs/sft.cern.ch/lcg/releases/LCG_97/mysql/10.4.12/x86_64-centos7-gcc9-dbg) +# +FIND_PATH(MYSQL_INCLUDE_DIR mysql.h + HINTS ${MYSQL_DIR}/include/mysql /usr/include/mysql + ) +# +FIND_LIBRARY(MYSQL_LIBRARY NAMES mysqlclient_r libmysqlclient.lib) +# +# handle the QUIETLY and REQUIRED arguments and set MYSQL_FOUND to TRUE if +# all listed variables are TRUE +INCLUDE(FindPackageHandleStandardArgs) +# +FIND_PACKAGE_HANDLE_STANDARD_ARGS(MySQL DEFAULT_MSG MYSQL_INCLUDE_DIR MYSQL_LIBRARY) +# +MARK_AS_ADVANCED(MYSQL_FOUND MYSQL_INCLUDE_DIR MYSQL_LIBRARY) +# +SET(MYSQL_INCLUDE_DIRS ${MYSQL_INCLUDE_DIR} ${MYSQL_INCLUDE_DIR}/mysql) +# +GET_FILENAME_COMPONENT(MYSQL_LIBRARY_DIRS ${MYSQL_LIBRARY} PATH) +# +SET(MYSQL_LIBRARIES ${MYSQL_LIBRARY}) +# diff --git a/cmake/FindPostgreSQL.cmake b/cmake/FindPostgreSQL.cmake new file mode 100644 index 000000000..200c8c4eb --- /dev/null +++ b/cmake/FindPostgreSQL.cmake @@ -0,0 +1,32 @@ +# - Locate MySQL library +# Defines: +# +# PostgreSQL_FOUND +# PostgreSQL_INCLUDE_DIR +# PostgreSQL_INCLUDE_DIRS (not cached) +# PostgreSQL_LIBRARY +# PostgreSQL_LIBRARIES (not cached) +# PostgreSQL_LIBRARY_DIRS (not cached) +# PostgreSQL_EXECUTABLE +# +SET(PostgreSQL_DIR /cvmfs/sft.cern.ch/lcg/external/postgresql/9.5.1/x86_64-slc6-gcc49-opt) +# +find_path(PostgreSQL_INCLUDE_DIR libpq-fe.h + HINTS ${PostgreSQL_DIR}/include /usr/include + ) +# +FIND_LIBRARY(PostgreSQL_LIBRARY NAMES mysqlclient_r libmysqlclient.lib) +# +# handle the QUIETLY and REQUIRED arguments and set PostgreSQL_FOUND to TRUE if +# all listed variables are TRUE +INCLUDE(FindPackageHandleStandardArgs) +# +FIND_PACKAGE_HANDLE_STANDARD_ARGS(MySQL DEFAULT_MSG PostgreSQL_INCLUDE_DIR PostgreSQL_LIBRARY) +# +MARK_AS_ADVANCED(PostgreSQL_FOUND PostgreSQL_INCLUDE_DIR PostgreSQL_LIBRARY) +# +SET(PostgreSQL_INCLUDE_DIRS ${PostgreSQL_INCLUDE_DIR} ${PostgreSQL_INCLUDE_DIR}/mysql) +# +GET_FILENAME_COMPONENT(PostgreSQL_LIBRARY_DIRS ${PostgreSQL_LIBRARY} PATH) +# +SET(PostgreSQL_LIBRARIES ${PostgreSQL_LIBRARY}) -- GitLab From 388443edc429f0dc0b34e6a0d31897e36d1e4945 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Tue, 4 May 2021 19:03:44 +0200 Subject: [PATCH 21/26] Update pending stuff from EB and sqldb --- .../options/EB_Transport_properties.opts | 2 +- .../options/MFP_generator_params.opts | 2 +- Online/sqldb/CMakeLists.txt | 2 +- Online/sqldb/sqldb/postgresql.h | 7 +- Online/sqldb/sqldb/sqldb.h | 12 +- Online/sqldb/src/postgresql.cpp | 240 +++++++++++++----- cmake/FindPostgreSQL.cmake | 38 +-- 7 files changed, 211 insertions(+), 92 deletions(-) diff --git a/Online/EventBuilding/options/EB_Transport_properties.opts b/Online/EventBuilding/options/EB_Transport_properties.opts index 876ec9a89..100f72393 100755 --- a/Online/EventBuilding/options/EB_Transport_properties.opts +++ b/Online/EventBuilding/options/EB_Transport_properties.opts @@ -2,7 +2,7 @@ EB_transport.BU_ranks = {}; EB_transport.MPI_errors_return = FALSE; EB_transport.RU_ranks = {}; EB_transport.RUs_per_nic = {}; -EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.LhX721.json"; +EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.XM9928.json"; EB_transport.n_par_mess = 0; EB_transport.n_sources_per_ru = { 3 }; EB_transport.shift_pattern = {}; diff --git a/Online/EventBuilding/options/MFP_generator_params.opts b/Online/EventBuilding/options/MFP_generator_params.opts index fe3d45721..2ef58e6b5 100644 --- a/Online/EventBuilding/options/MFP_generator_params.opts +++ b/Online/EventBuilding/options/MFP_generator_params.opts @@ -5,4 +5,4 @@ MFP_generator.n_banks = 30000; MFP_generator.numa_layout = { 4, 2 }; MFP_generator.random_sizes = FALSE; MFP_generator.reconnect = FALSE; -MFP_generator.shmem_prefix = "RU_"; +MFP_generator.shmem_prefix = "RU"; diff --git a/Online/sqldb/CMakeLists.txt b/Online/sqldb/CMakeLists.txt index c68b6a4ab..0e437c9bf 100644 --- a/Online/sqldb/CMakeLists.txt +++ b/Online/sqldb/CMakeLists.txt @@ -46,7 +46,7 @@ endif() if ( PostgreSQL_FOUND ) gaudi_add_library(sqldb_postgresql src/postgresql.cpp NO_PUBLIC_HEADERS - INCLUDE_DIRS Boost PostgreSQL ${PostgreSQL_INCLUDE_DIR} + INCLUDE_DIRS Boost PostgreSQL LINK_LIBRARIES PostgreSQL) target_compile_definitions(sqldb_postgresql PRIVATE -DSQLDB_HAVE_POSTGRESQL=1) endif() diff --git a/Online/sqldb/sqldb/postgresql.h b/Online/sqldb/sqldb/postgresql.h index a7b20dfc3..6949d44fe 100644 --- a/Online/sqldb/sqldb/postgresql.h +++ b/Online/sqldb/sqldb/postgresql.h @@ -18,7 +18,8 @@ /// Framework include files #include <sqldb/sqldb.h> -typedef struct pg_conn PGconn; +typedef struct pg_conn PGconn; +typedef struct pg_result PQresult; /// Technology abstraction layer /** @@ -29,8 +30,8 @@ typedef struct pg_conn PGconn; */ class postgresql : public sqldb::traits { public: - typedef PGconn dbase_t; - struct statement_t; + typedef PGconn dbase_t; + typedef PQresult statement_t; }; #endif // SQLDB_POSTGRESQL_INTERFACE_H diff --git a/Online/sqldb/sqldb/sqldb.h b/Online/sqldb/sqldb/sqldb.h index 97d1aaf79..83aa2866c 100644 --- a/Online/sqldb/sqldb/sqldb.h +++ b/Online/sqldb/sqldb/sqldb.h @@ -42,11 +42,13 @@ namespace sqldb { public: }; - static constexpr int OK = 0; - static constexpr int ERROR = EINVAL; - static constexpr int DONE = 10000; - static constexpr int NO_DATA = 10002; - static constexpr int INVALID_HANDLE = 10004; + enum sqldb_status { + OK = 0, + ERROR = EINVAL, + DONE = 10000, + NO_DATA = 10002, + INVALID_HANDLE = 10004 + }; enum field_type { TYPE_NULL = 14, diff --git a/Online/sqldb/src/postgresql.cpp b/Online/sqldb/src/postgresql.cpp index cad061843..0c3d86761 100644 --- a/Online/sqldb/src/postgresql.cpp +++ b/Online/sqldb/src/postgresql.cpp @@ -20,13 +20,28 @@ /// MySQL header #include <libpq-fe.h> +#include <sql3types.h> -//struct postgresql::statement_t : public POSTGRESQL_STMT { }; +#define POSTGRESQL_TYPE_BLOB 123456 /// SQLDB namespace declaration namespace sqldb { namespace postgresql_imp { + + /// + std::string _escape(postgresql::dbase_t* hdl, const char* str, size_t length) { + std::vector<char> buf(2*length+1); + size_t len = ::PQescapeStringConn(hdl, &buf.front(), str, length, 0); + return std::string(&buf.front(), len); + } + /// + void clear_result(PQresult*& result) { + if ( result != nullptr ) { + ::PQclear(result); + result = nullptr; + } + } /// inline int postgresql_ok(const PGresult* status) { if ( status ) { @@ -48,44 +63,47 @@ namespace sqldb { } return sqldb::ERROR; } -#if 0 /// - inline field_type gen_field_type(enum_field_types typ) { + inline field_type gen_field_type(int typ) { switch(typ) { - case POSTGRESQL_TYPE_TINY: return TYPE_TINY; - case POSTGRESQL_TYPE_SHORT: return TYPE_SHORT; - case POSTGRESQL_TYPE_LONG: return TYPE_LONG; - case POSTGRESQL_TYPE_LONGLONG: return TYPE_LONGLONG; - case POSTGRESQL_TYPE_FLOAT: return TYPE_FLOAT; - case POSTGRESQL_TYPE_DOUBLE: return TYPE_DOUBLE; - case POSTGRESQL_TYPE_STRING: return TYPE_STRING; - case POSTGRESQL_TYPE_BLOB: return TYPE_BLOB; - case POSTGRESQL_TYPE_TIME: return TYPE_TIME; - case POSTGRESQL_TYPE_DATE: return TYPE_TIME; - case POSTGRESQL_TYPE_DATETIME: return TYPE_TIME; + case SQL3_SMALLINT: return TYPE_SHORT; + case SQL3_INTEGER: return TYPE_LONGLONG; + case SQL3_FLOAT: return TYPE_FLOAT; + case SQL3_DOUBLE_PRECISION: return TYPE_DOUBLE; + case SQL3_CHARACTER: return TYPE_STRING; + case POSTGRESQL_TYPE_BLOB: return TYPE_BLOB; + case SQL3_DATE_TIME_TIMESTAMP: return TYPE_TIME; +#if 0 + case SQL3_DDT_DATE: return TYPE_TIME; + case SQL3_DDT_TIME: return TYPE_TIME; + case SQL3_DDT_TIMESTAMP: return TYPE_TIME; + case SQL3_DDT_TIME_WITH_TIME_ZONE: return TYPE_TIME; + case SQL3_DDT_TIMESTAMP_WITH_TIME_ZONE: return TYPE_TIME; +#endif default: break; } throw std::runtime_error("Invalid data type encountered!"); } /// - inline enum_field_types postgresql_field_type(field_type typ) { + inline int postgresql_field_type(field_type typ) { switch(typ) { - case TYPE_TINY: return POSTGRESQL_TYPE_TINY; - case TYPE_SHORT: return POSTGRESQL_TYPE_SHORT; - case TYPE_LONG: return POSTGRESQL_TYPE_LONG; - case TYPE_LONGLONG: return POSTGRESQL_TYPE_LONGLONG; - case TYPE_FLOAT: return POSTGRESQL_TYPE_FLOAT; - case TYPE_DOUBLE: return POSTGRESQL_TYPE_DOUBLE; - case TYPE_STRING: return POSTGRESQL_TYPE_STRING; + case TYPE_TINY: return SQL3_CHARACTER; + case TYPE_SHORT: return SQL3_SMALLINT; + case TYPE_LONG: return SQL3_INTEGER; + case TYPE_LONGLONG: return SQL3_INTEGER; + case TYPE_FLOAT: return SQL3_FLOAT; + case TYPE_DOUBLE: return SQL3_DOUBLE_PRECISION; + case TYPE_STRING: return SQL3_CHARACTER; case TYPE_BLOB: return POSTGRESQL_TYPE_BLOB; - case TYPE_TIME: return POSTGRESQL_TYPE_DATETIME; + case TYPE_TIME: return SQL3_DATE_TIME_TIMESTAMP; default: break; } throw std::runtime_error("Invalid data type encountered!"); } +#if 0 /// Technology abstraction layer /** * \author M.Frank @@ -150,17 +168,26 @@ namespace sqldb { */ struct statement_backend : public statement::backend { public: - postgresql::dbase_t* _handle { nullptr }; + typedef enum { + null_param, + text_param, + binary_param + } param_type; + + postgresql::dbase_t* _handle { nullptr }; + postgresql::statement_t* _result { nullptr }; + std::string _sql; + std::string _id; + std::stringstream _fmt; + std::vector<std::string> _params; + std::vector<char const *> _params_p; + std::vector<size_t> _params_len; + std::vector<param_type> _params_typ; + int _current {0}; - /// Access to the internal statement handle. Throws exception if the handle is invalid - postgresql::dbase_t* handle() const { - if ( this->_handle ) return this->_handle; - throw std::runtime_error("Failed to access statement handle. [Invalid handle]"); - } #if 0 typedef std::tuple<column_data*, column_bind*, POSTGRESQL_FIELD*> field_t; std::vector<column_bind> _pbind, _fbind; - std::string _sql_string; POSTGRESQL_STMT* _imp { nullptr }; std::pair<column_data*, column_bind*> param(int col) { @@ -170,21 +197,111 @@ namespace sqldb { return field_t(&this->fields[col], &_fbind[col], &_imp->fields[col]); } #endif + template <typename T> T _to(const char* s) { + T result; + this->_fmt.clear(); + this->_fmt.str((s && *s) ? s : "-1"); + this->_fmt >> result; + return result; + } + /// Access to the internal statement handle. Throws exception if the handle is invalid + postgresql::dbase_t* handle() const { + if ( this->_handle ) return this->_handle; + throw std::runtime_error("Failed to access statement handle. [Invalid handle]"); + } /// Access error message on failure virtual std::string errmsg() override { return ::PQerrorMessage(this->handle()); } /// Character string of the prepared statement virtual const char* sql() override { - return "";//this->_sql_string.c_str(); + return this->_sql.c_str(); } /// Access the number of rows affected by the last statement virtual long rows_affected() override { - return 0;//::postgresql_stmt_num_rows(this->handle()); + if ( this->_result ) { + const char* s = ::PQcmdTuples(this->_result); + if ( s && *s ) + return _to<long>(s); + } + return -1; + } + /// + sqldb_status reset() { + if( this->_handle ) { + clear_result(this->_result); + std::vector<std::string> vals(_params); + std::vector<size_t> lens(_params.size(),0); + std::vector<char const *> pval(_params.size(),0); + std::vector<param_type> typs(_params.size(),null_param); + _params.swap(vals); + _params_p.swap(pval); + _params_len.swap(lens); + _params_typ.swap(typs); + return sqldb::OK; + } + return sqldb::ERROR; } /// Prepare a new statement and allocate resources. - int prepare(postgresql::dbase_t* db, const std::string& sql_str) { + sqldb_status prepare(postgresql::dbase_t* db, const std::string& sql_str, uint64_t id) { + this->_handle = db; + this->_current = 0; + this->_fmt.imbue(std::locale::classic()); + this->_fmt.str(std::string()); + this->_fmt.clear(); + this->_fmt << "sqldb_pq_stmt_" << id; + this->_id = this->_fmt.str(); + this->_fmt.str(std::string()); + this->_fmt.clear(); + this->_sql.clear(); + this->_sql.reserve(sql_str.size()); + int num_params = 0; + bool inside = false; + for(unsigned i=0;i<sql_str.size();i++) { + char c = sql_str[i]; + switch(c) { + case '\'': + inside = !inside; + break; + case '?': + if( !inside ) { + this->_sql += '$'; + ++num_params; + this->_fmt << num_params; + this->_sql += this->_fmt.str(); + this->_fmt.str(std::string()); + this->_fmt.clear(); + } + break; + default: + this->_sql += c; + break; + } + } + + _params.resize(num_params,std::string()); + _params_typ.resize(num_params, null_param); + _params_p.resize(num_params, nullptr); + _params_len.resize(num_params, 0); + this->reset(); + + this->_result = ::PQprepare(this->_handle,_id.c_str(),this->_sql.c_str(),0,0); + try { + if( !this->_result ) { + throw std::runtime_error("sqldb(postgres): Failed to create prepared statement object!"); + } + if ( ::PQresultStatus(this->_result) != PGRES_COMMAND_OK ) { + throw std::runtime_error("sqldb(postgres): statement preparation failed: "+errmsg()); + } + } + catch(...) { + clear_result(this->_result); + throw; + } + clear_result(this->_result); + return sqldb::OK; + } #if 0 POSTGRESQL_STMT* stmt = nullptr; @@ -200,7 +317,7 @@ namespace sqldb { this->params.resize(num_param, column_data()); this->_fbind.resize(num_field, column_bind()); this->fields.resize(num_field, column_data()); - this->_sql_string = sql_str; + this->_sql = sql_str; this->_imp = stmt; this->stmt = this; return sqldb::OK; @@ -208,8 +325,6 @@ namespace sqldb { return sqldb::ERROR; } #endif - return invalid_statement(); - } /// Execute prepared statement virtual int execute() { @@ -329,27 +444,29 @@ namespace sqldb { /// Action routine for various functions int action(action_type type) override { switch(type) { -#if 0 case EXEC: /// Execute prepared statement - return this->execute(); + return sqldb::OK; case FETCH: /// Fetch next result row. Stop if result is NON-zero - return this->fetch_one(); - case RESET: /// Reset all data to re-execute the statement with new parameters - if( auto* h = this->handle() ) { - if( !postgresql_ok(::postgresql_stmt_reset(h)) ) - return sqldb::ERROR; + if ( this->_result ) { + return ::PQgetisnull(this->_result, this->_current, 0) ? sqldb::OK : sqldb::ERROR; } - return sqldb::OK; + return sqldb::ERROR; + case RESET: /// Reset all data to re-execute the statement with new parameters + return this->reset(); case FINALIZE: /// Finalize and run-down the statement. Release all resources. - if ( this->_imp ) { - ::postgresql_stmt_reset(this->_imp); - ::postgresql_stmt_close(this->_imp); - this->_imp = nullptr; + if ( this->_handle != nullptr && !this->_id.empty() ) { + this->_fmt.str(""); + this->_fmt.clear(); + this->_fmt << "DEALLOCATE " << this->_id; + this->_result = ::PQexec(this->_handle, this->_fmt.str().c_str()); + clear_result(this->_result); } return sqldb::OK; case ERRNO: /// Access error number of last failed statement - return ::postgresql_stmt_errno(this->handle()); -#endif + if ( this->_result != nullptr ) { + return ::PQresultStatus(this->_result); + } + return ::PQstatus(this->handle()); default: throw std::runtime_error("Invalid action request received!"); } @@ -365,7 +482,8 @@ namespace sqldb { struct dbase_backend : public database::backend { postgresql::dbase_t* _handle { nullptr }; ::PGresult* _result { nullptr }; - + unsigned long long _prepared_id { 0 }; + /// Access to the internal database handle. Throws exception if the handle is invalid postgresql::dbase_t* handle() const { if ( _handle ) return _handle; @@ -385,32 +503,26 @@ namespace sqldb { } return -1; } - void clear_result() { - if ( this->_result != nullptr ) { - ::PQclear(this->_result); - this->_result = nullptr; - } - } /// Perform multiple actions virtual int action(action_type type) override { switch(type) { case CLOSE: - clear_result(); + clear_result(this->_result); if ( this->_handle ) { ::PQfinish(this->_handle); this->_handle = nullptr; } return sqldb::OK; case BEGIN: /// Start transaction - clear_result(); + clear_result(this->_result); this->_result = ::PQexec(this->handle(), "BEGIN"); return postgresql_ok(this->_result); case COMMIT: /// Commit transaction - clear_result(); + clear_result(this->_result); this->_result = ::PQexec(this->handle(), "COMMIT"); return postgresql_ok(this->_result); case ROLLBACK: /// Rollback transaction - clear_result(); + clear_result(this->_result); this->_result = ::PQexec(this->handle(), "ROLLBACK"); return postgresql_ok(this->_result); case ERRNO: /// Access error number of last failed statement @@ -426,7 +538,7 @@ namespace sqldb { virtual int execute(std::string& err, const std::string& sql) override { err = ""; if( this->_handle ) { - clear_result(); + clear_result(this->_result); this->_result = ::PQexec(this->_handle, sql.c_str()); if ( postgresql_ok(this->_result) == sqldb::OK ) { return sqldb::OK; @@ -445,8 +557,10 @@ namespace sqldb { /// Prepare a new statement and allocate resources. virtual int prepare(statement& bs, const std::string& sql_str) override { auto* stm_imp = new statement_backend(); + auto* h = this->handle(); + auto sql = _escape(h, sql_str.c_str(), sql_str.length()); bs.intern.reset(stm_imp); - return stm_imp->prepare(this->handle(), sql_str); + return stm_imp->prepare(h, sql, ++this->_prepared_id); } /// Open database int open(const std::string& connect_string) { diff --git a/cmake/FindPostgreSQL.cmake b/cmake/FindPostgreSQL.cmake index 200c8c4eb..46f4a8642 100644 --- a/cmake/FindPostgreSQL.cmake +++ b/cmake/FindPostgreSQL.cmake @@ -1,32 +1,34 @@ -# - Locate MySQL library +# +# - Locate PostgreSQL library # Defines: # # PostgreSQL_FOUND -# PostgreSQL_INCLUDE_DIR -# PostgreSQL_INCLUDE_DIRS (not cached) -# PostgreSQL_LIBRARY -# PostgreSQL_LIBRARIES (not cached) -# PostgreSQL_LIBRARY_DIRS (not cached) -# PostgreSQL_EXECUTABLE +# POSTGRESQL_INCLUDE_DIR +# POSTGRESQL_INCLUDE_DIRS (not cached) +# POSTGRESQL_LIBRARY +# POSTGRESQL_LIBRARIES (not cached) +# POSTGRESQL_LIBRARY_DIRS (not cached) +# POSTGRESQL_EXECUTABLE # -SET(PostgreSQL_DIR /cvmfs/sft.cern.ch/lcg/external/postgresql/9.5.1/x86_64-slc6-gcc49-opt) +SET(POSTGRESQL_DIR /cvmfs/sft.cern.ch/lcg/external/postgresql/9.5.1/x86_64-slc6-gcc49-opt) # -find_path(PostgreSQL_INCLUDE_DIR libpq-fe.h - HINTS ${PostgreSQL_DIR}/include /usr/include +find_path(POSTGRESQL_INCLUDE_DIR libpq-fe.h + HINTS ${POSTGRESQL_DIR}/include /usr/include ) # -FIND_LIBRARY(PostgreSQL_LIBRARY NAMES mysqlclient_r libmysqlclient.lib) +FIND_LIBRARY(POSTGRESQL_LIBRARY NAMES pq libpq.so + HINTS ${POSTGRESQL_DIR}/lib + ) # -# handle the QUIETLY and REQUIRED arguments and set PostgreSQL_FOUND to TRUE if -# all listed variables are TRUE +# handle the QUIETLY and REQUIRED arguments and set POSTGRESQL_FOUND to TRUE if all listed variables are TRUE INCLUDE(FindPackageHandleStandardArgs) # -FIND_PACKAGE_HANDLE_STANDARD_ARGS(MySQL DEFAULT_MSG PostgreSQL_INCLUDE_DIR PostgreSQL_LIBRARY) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(PostgreSQL DEFAULT_MSG POSTGRESQL_INCLUDE_DIR POSTGRESQL_LIBRARY) # -MARK_AS_ADVANCED(PostgreSQL_FOUND PostgreSQL_INCLUDE_DIR PostgreSQL_LIBRARY) +MARK_AS_ADVANCED(PostgreSQL_FOUND POSTGRESQL_INCLUDE_DIR POSTGRESQL_LIBRARY) # -SET(PostgreSQL_INCLUDE_DIRS ${PostgreSQL_INCLUDE_DIR} ${PostgreSQL_INCLUDE_DIR}/mysql) +SET(POSTGRESQL_INCLUDE_DIRS ${POSTGRESQL_INCLUDE_DIR} ${POSTGRESQL_INCLUDE_DIR}) # -GET_FILENAME_COMPONENT(PostgreSQL_LIBRARY_DIRS ${PostgreSQL_LIBRARY} PATH) +GET_FILENAME_COMPONENT(POSTGRESQL_LIBRARY_DIRS ${POSTGRESQL_LIBRARY} PATH) # -SET(PostgreSQL_LIBRARIES ${PostgreSQL_LIBRARY}) +SET(POSTGRESQL_LIBRARIES ${POSTGRESQL_LIBRARY}) -- GitLab From 12e8f021fae876896a4e861f0072af9c93ba73ca Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Mon, 10 May 2021 02:17:45 +0200 Subject: [PATCH 22/26] Fix nightly build warnings --- Online/EventBuilding/options/EB_BU_algo.opts | 7 - Online/EventBuilding/options/EB_RU_algo.opts | 9 - .../options/EB_Transport_properties.opts | 9 - Online/HTTP/src/HttpServer.cpp | 316 +++++----- Online/PcSrv/PcSrv/TaskCheck.h | 5 +- Online/PcSrv/src/CacheDBSrv.cpp | 16 +- Online/PcSrv/src/TaskCheck.cpp | 83 +-- Online/RPC/src/HttpRpcClient.cpp | 60 +- Online/sqldb/sqldb/sqldb-imp.h | 93 +-- Online/sqldb/sqldb/sqldb.h | 75 ++- Online/sqldb/src/mysql.cpp | 28 +- Online/sqldb/src/oracle.cpp | 74 +-- Online/sqldb/src/postgresql.cpp | 591 +++++++++--------- Online/sqldb/src/sqlite.cpp | 36 +- 14 files changed, 719 insertions(+), 683 deletions(-) delete mode 100755 Online/EventBuilding/options/EB_BU_algo.opts delete mode 100644 Online/EventBuilding/options/EB_RU_algo.opts delete mode 100755 Online/EventBuilding/options/EB_Transport_properties.opts diff --git a/Online/EventBuilding/options/EB_BU_algo.opts b/Online/EventBuilding/options/EB_BU_algo.opts deleted file mode 100755 index 308ea7c2c..000000000 --- a/Online/EventBuilding/options/EB_BU_algo.opts +++ /dev/null @@ -1,7 +0,0 @@ -BU.MBM_name = { 'Input', 'Input' }; -BU.buffer_size = {}; -BU.buffer_type = { 2, 2 }; -BU.out_file_prefix = ""; -BU.shmem_prefix = "BU_"; -BU.stop_timeout = 60; -BU.write_to_file = { FALSE, FALSE }; diff --git a/Online/EventBuilding/options/EB_RU_algo.opts b/Online/EventBuilding/options/EB_RU_algo.opts deleted file mode 100644 index 0d55e62df..000000000 --- a/Online/EventBuilding/options/EB_RU_algo.opts +++ /dev/null @@ -1,9 +0,0 @@ -RU.MDF_filename = "/home/fpisani/mc_data/biger_file.mdf"; -RU.PCIe40_ids = {}; -RU.PCIe40_names = { 'tdtel051_1', 'tdtel052_0', 'tdtel052_1', 'tdtel053_0', 'tdtel053_1' }; -RU.buffer_sizes = {}; -RU.buffer_type = { 1 }; -RU.n_MFPs = 100; -RU.n_fragment = 3000; -RU.shmem_prefix = "RU_"; -RU.stop_timeout = 60; diff --git a/Online/EventBuilding/options/EB_Transport_properties.opts b/Online/EventBuilding/options/EB_Transport_properties.opts deleted file mode 100755 index 100f72393..000000000 --- a/Online/EventBuilding/options/EB_Transport_properties.opts +++ /dev/null @@ -1,9 +0,0 @@ -EB_transport.BU_ranks = {}; -EB_transport.MPI_errors_return = FALSE; -EB_transport.RU_ranks = {}; -EB_transport.RUs_per_nic = {}; -EB_transport.ib_config_file = "/home/lgranado/ebuilder/Online/tmp/PVSS.XM9928.json"; -EB_transport.n_par_mess = 0; -EB_transport.n_sources_per_ru = { 3 }; -EB_transport.shift_pattern = {}; -EB_transport.src_ids = {}; diff --git a/Online/HTTP/src/HttpServer.cpp b/Online/HTTP/src/HttpServer.cpp index 89109effd..4c2343411 100644 --- a/Online/HTTP/src/HttpServer.cpp +++ b/Online/HTTP/src/HttpServer.cpp @@ -137,8 +137,7 @@ void HttpConnection::send(const std::vector<boost::asio::const_buffer>& buffers, void HttpConnection::do_next(int next_action) { print_next(next_action); if ( next_action == HttpRequestHandler::read ) { - request.content.clear(); - if ( request.content_length - request.content_received > buffer.size() ) + if ( request.content_length - request.content_received > request.content.size() ) do_read(); else do_read_exact(request.content_length - request.content_received); @@ -182,148 +181,185 @@ void HttpConnection::do_shutdown() { /// Perform an asynchronous read operation. void HttpConnection::do_read() { auto self(shared_from_this()); - socket.async_read_some - (asio::buffer(buffer.data(), buffer.size()), - [this, self](system::error_code ec, std::size_t len) { - if ( !ec ) { - HttpRequestParser::result_type result = (HttpRequestParser::result_type)request.state; - bool do_move = (len > 0) && (result == HttpRequestParser::good); - request.content.clear(); - if ( result == HttpRequestParser::indeterminate ) { - std::tie(result, std::ignore) = parser.parse(request, buffer.data(), buffer.data() + len); - request.state = result; - } - if ( result == HttpRequestParser::good ) { - auto now = system_clock::now(); - request.netio += std::chrono::duration_cast<milliseconds>(now-request.start); - request.start = now; - if ( request.content_length == HttpRequest::invalid_content_length ) { - for(const auto& h : request.headers ) { - if ( h.name == http::constants::content_length ) { - request.content_length = ::atol(h.value.c_str()); - buffer.resize(request.content_length); - } - } - request.content_received = request.content.size(); - } - else { - request.content_received += len; - } - request.content.clear(); - if ( do_move ) { - // std::cout << "HttpConnection: read: move buffer len " << len << std::endl << std::flush; - request.content = std::move(buffer); - request.content.resize(len); - } - now = system_clock::now(); - auto next = handler.handle_request(request, reply); - if ( do_move ) buffer = std::move(request.content); - request.handling += std::chrono::duration_cast<milliseconds>(now-request.start); - request.start = now; - do_next(next); - } - else if ( result == HttpRequestParser::bad ) { - if ( handler.debug ) { - std::cout << "HttpConnection: read: parsing FAILED " << std::endl << std::flush; - } - reply = HttpReply::stock_reply(HttpReply::bad_request); - request.content_length = 0; - request.content.clear(); - parser.consumed = 0; - do_write(); - } - else { - do_read(); - } - } - else if ( ec != asio::error::operation_aborted ) { - system::error_code ignored_ec; - request.content_length = 0; - request.content.clear(); - parser.consumed = 0; - std::cout << "HttpConnection: SHUTDOWN (read)." << std::endl << std::flush; - socket.shutdown(asio::ip::tcp::socket::shutdown_both,ignored_ec); - manager.stop(shared_from_this()); - if ( self.get() ) {} - } - }); + auto handler_function = + [this, self](system::error_code ec, std::size_t len) { + auto& content = request.content; + if ( !ec ) { + HttpRequestParser::result_type result = (HttpRequestParser::result_type)request.state; + /// Parse transport header if not yet done + if ( result == HttpRequestParser::indeterminate ) { + content.clear(); + std::tie(result, std::ignore) = parser.parse(request, buffer.data(), buffer.data() + len); + request.state = result; + } + /// Transport header successfully parsed. Look at the content + if ( result == HttpRequestParser::good ) { + auto now = system_clock::now(); + request.netio += std::chrono::duration_cast<milliseconds>(now-request.start); + request.start = now; + /// First extract the HTTP header information + if ( request.content_length == HttpRequest::invalid_content_length ) { + buffer.clear(); + request.content_received = content.size(); + for(const auto& h : request.headers ) { + if ( h.name == http::constants::content_length ) { + request.content_length = ::atol(h.value.c_str()); + } + } + auto next = handler.handle_request(request, reply); + now = system_clock::now(); + request.handling += std::chrono::duration_cast<milliseconds>(now-request.start); + request.start = now; + do_next(next); + return; + } + else { + request.content_received += len; + } + /// All data were received. Call the handler + if ( request.content_received >= request.content_length ) { + now = system_clock::now(); + auto next = handler.handle_request(request, reply); + request.handling += std::chrono::duration_cast<milliseconds>(now-request.start); + request.start = now; + do_next(next); + return; + } + /// Restart reading cycle to fill the data buffer + request.start = now; + do_read_exact(request.content_length - request.content_received); + } + else if ( result == HttpRequestParser::bad ) { + if ( handler.debug ) { + std::cout << "HttpConnection: read: parsing FAILED " << std::endl << std::flush; + } + reply = HttpReply::stock_reply(HttpReply::bad_request); + request.content_length = 0; + content.clear(); + parser.consumed = 0; + do_write(); + } + else { + do_read(); + } + } + else if ( ec != asio::error::operation_aborted ) { + system::error_code ignored_ec; + request.content_length = 0; + content.clear(); + parser.consumed = 0; + std::cout << "HttpConnection: SHUTDOWN (read)." << std::endl << std::flush; + socket.shutdown(asio::ip::tcp::socket::shutdown_both,ignored_ec); + manager.stop(shared_from_this()); + if ( self.get() ) {} + } + }; + if ( request.state == HttpRequestParser::indeterminate ) { + socket.async_read_some(asio::buffer(buffer.data(), buffer.size()), handler_function); + } + else { + size_t cont_len = request.content.size(); + size_t buff_len = request.content_length-cont_len; + if ( request.content_length > cont_len ) { + request.content.resize(request.content_length); + } + // socket.async_read_some(asio::buffer(request.content.data()+cont_len, buff_len), handler_function); + asio::async_read(socket, asio::buffer(request.content.data()+cont_len, buff_len), + asio::transfer_exactly(buff_len), + handler_function); + } } /// Perform an asynchronous read operation with exact read length -void HttpConnection::do_read_exact(size_t len) { +void HttpConnection::do_read_exact(size_t record_length) { auto self(shared_from_this()); - asio::async_read - (socket, - asio::buffer(buffer.data(), buffer.size()), - asio::transfer_exactly(len), - [this, self](system::error_code ec, std::size_t len) { - if (!ec) { - HttpRequestParser::result_type result = (HttpRequestParser::result_type)request.state; - bool do_move = (len > 0) && (result == HttpRequestParser::good); - request.content.clear(); - if ( result == HttpRequestParser::indeterminate ) { - std::tie(result, std::ignore) = parser.parse(request, buffer.data(), buffer.data() + len); - request.state = result; - } - if ( result == HttpRequestParser::good ) { - auto now = system_clock::now(); - request.netio += std::chrono::duration_cast<milliseconds>(now-request.start); - request.start = now; - if ( request.content_length == HttpRequest::invalid_content_length ) { - for(const auto& h : request.headers ) { - if ( h.name == http::constants::content_length ) { - request.content_length = ::atol(h.value.c_str()); - buffer.resize(request.content_length); - } - } - request.content_received = request.content.size(); - } - else { - request.content_received += len; - } - request.content.clear(); - if ( request.content_received < request.content_length ) { - do_read_exact(request.content_length - request.content.size()); - } - else { - request.content.clear(); - if ( do_move ) { - // std::cout << "HttpConnection: read_exact: move buffer len " << len << std::endl << std::flush; - request.content = std::move(buffer); - request.content.resize(len); - } - now = system_clock::now(); - auto next = handler.handle_request(request, reply); - if ( do_move ) buffer = std::move(request.content); - request.handling += std::chrono::duration_cast<milliseconds>(now-request.start); - request.start = now; - do_next(next); - } - } - else if ( result == HttpRequestParser::bad ) { - reply = HttpReply::stock_reply(HttpReply::bad_request); - request.content_length = 0; - request.content.clear(); - parser.consumed = 0; - do_write(); - } - else { - do_read(); - } - } - else if (ec != asio::error::operation_aborted) { - system::error_code ignored_ec; - request.content_length = 0; - request.content.clear(); - parser.consumed = 0; - if ( handler.debug ) { - std::cout << "HttpConnection: SHUTDOWN (read_exact)." << std::endl << std::flush; - } - socket.shutdown(asio::ip::tcp::socket::shutdown_both,ignored_ec); - manager.stop(shared_from_this()); - if ( self.get() ) {} - } - }); + auto handler_function = + [this, self](system::error_code ec, std::size_t len) { + auto& content = request.content; + if ( !ec ) { + HttpRequestParser::result_type result = (HttpRequestParser::result_type)request.state; + /// Parse transport header if not yet done + if ( result == HttpRequestParser::indeterminate ) { + content.clear(); + std::tie(result, std::ignore) = parser.parse(request, buffer.data(), buffer.data() + len); + request.state = result; + buffer.clear(); + } + /// Transport header successfully parsed. Look at the content + if ( result == HttpRequestParser::good ) { + auto now = system_clock::now(); + auto& content = request.content; + request.netio += std::chrono::duration_cast<milliseconds>(now-request.start); + request.start = now; + /// First extract the HTTP header information + if ( request.content_length == HttpRequest::invalid_content_length ) { + buffer.clear(); + for(const auto& h : request.headers ) { + if ( h.name == http::constants::content_length ) { + request.content_length = ::atol(h.value.c_str()); + } + } + request.content_received = content.size(); + auto next = handler.handle_request(request, reply); + now = system_clock::now(); + request.handling += std::chrono::duration_cast<milliseconds>(now-request.start); + request.start = now; + do_next(next); + return; + } + else { + request.content_received += len; + } + /// All data were received. Call the handler + if ( request.content_received >= request.content_length ) { + now = system_clock::now(); + auto next = handler.handle_request(request, reply); + request.handling += std::chrono::duration_cast<milliseconds>(now-request.start); + request.start = now; + do_next(next); + return; + } + /// Restart reading cycle to fill the data buffer + request.start = now; + do_read_exact(request.content_length - request.content_received); + } + else if ( result == HttpRequestParser::bad ) { + reply = HttpReply::stock_reply(HttpReply::bad_request); + request.content_length = 0; + content.clear(); + parser.consumed = 0; + do_write(); + } + else { + do_read(); + } + } + else if (ec != asio::error::operation_aborted) { + system::error_code ignored_ec; + request.content_length = 0; + content.clear(); + parser.consumed = 0; + if ( handler.debug ) { + std::cout << "HttpConnection: SHUTDOWN (read_exact)." << std::endl << std::flush; + } + socket.shutdown(asio::ip::tcp::socket::shutdown_both,ignored_ec); + manager.stop(shared_from_this()); + if ( self.get() ) {} + } + }; + if ( request.state == HttpRequestParser::indeterminate ) { + asio::async_read(socket,asio::buffer(buffer.data(), buffer.size()), + asio::transfer_exactly(record_length), + handler_function); + } + else { + size_t cont_len = request.content.size(); + size_t buff_len = request.content_length-cont_len; + request.content.resize(request.content_length); + asio::async_read(socket, asio::buffer(request.content.data()+cont_len, buff_len), + asio::transfer_exactly(record_length), + handler_function); + } } /// Perform an asynchronous write operation. diff --git a/Online/PcSrv/PcSrv/TaskCheck.h b/Online/PcSrv/PcSrv/TaskCheck.h index d1101c823..36a1fb828 100644 --- a/Online/PcSrv/PcSrv/TaskCheck.h +++ b/Online/PcSrv/PcSrv/TaskCheck.h @@ -59,7 +59,7 @@ namespace taskdb { class TaskCheck : public CPP::Interactor { public: /// Definition of the task collection type - typedef std::vector<std::unique_ptr<xmlrpc::RPCClient> > servers_t; + typedef std::vector<std::unique_ptr<xmlrpc::HttpRpcClient> > servers_t; typedef std::chrono::duration<int> duration_t; typedef std::chrono::system_clock::time_point time_point_t; typedef std::map<std::string, Task> db_tasks_t; @@ -89,7 +89,6 @@ namespace taskdb { std::string m_dns; std::string m_node; std::string m_tmName; - long m_tmDicDns = -1; int m_tmInfo = -1; bool m_inited = false; bool m_info_changed = false; @@ -113,7 +112,7 @@ namespace taskdb { public: /// Default constructor - TaskCheck(const std::string& dns, const std::string& tms_dns, const std::string& node); + TaskCheck(const std::string& dns, const std::string& node); /// Default destructor virtual ~TaskCheck(); /// Access the task container from the tmSrv service diff --git a/Online/PcSrv/src/CacheDBSrv.cpp b/Online/PcSrv/src/CacheDBSrv.cpp index 8999ab9db..d75d6a831 100644 --- a/Online/PcSrv/src/CacheDBSrv.cpp +++ b/Online/PcSrv/src/CacheDBSrv.cpp @@ -172,16 +172,20 @@ string addr = "0.0.0.0", dis_server, server, mount = "/TDBDATA/XMLRPC", fifo; cl = new xmlrpc::HttpRpcClient(server, mount, port, tmo); m_servers.emplace_back(unique_ptr<xmlrpc::HttpRpcClient>(cl)); } - //cl = new xmlrpc::HttpRpcClient("slavetasks.service.consul.lhcb.cern.ch", mount, port, tmo); - //m_servers.emplace_back( unique_ptr<xmlrpc::HttpRpcClient>(cl)); - //lib_rtl_output(LIB_RTL_DEBUG,"Running Service on '%s'",RTL::str_upper(RTL::nodeNameShort()).c_str()); - if ( RTL::str_upper(RTL::nodeNameShort()) != "ECS03" ) { + + if ( RTL::str_upper(RTL::nodeNameShort()) == "ECS03" ) { + cl = new xmlrpc::HttpRpcClient("ecs03.lbdaq.cern.ch", mount, "3510", tmo); + m_servers.emplace_back( unique_ptr<xmlrpc::HttpRpcClient>(cl)); + } + else { cl = new xmlrpc::HttpRpcClient("ecs03.lbdaq.cern.ch", mount, "3501", tmo); m_servers.emplace_back( unique_ptr<xmlrpc::HttpRpcClient>(cl)); cl->debugFlag = client_debug; + + cl = new xmlrpc::HttpRpcClient("ecs03.lbdaq.cern.ch", mount, "3502", tmo); + m_servers.emplace_back( unique_ptr<xmlrpc::HttpRpcClient>(cl)); + cl->debugFlag = client_debug; } - cl = new xmlrpc::HttpRpcClient("ecs03.lbdaq.cern.ch", mount, "3500", tmo); - m_servers.emplace_back( unique_ptr<xmlrpc::HttpRpcClient>(cl)); { xmlrpc::HttpServer* server = new xmlrpc::HttpServer(addr,port,xmlrpc::HttpServer::SERVER,mount); server->setDebug(debug); diff --git a/Online/PcSrv/src/TaskCheck.cpp b/Online/PcSrv/src/TaskCheck.cpp index a2834bada..6c024a61a 100644 --- a/Online/PcSrv/src/TaskCheck.cpp +++ b/Online/PcSrv/src/TaskCheck.cpp @@ -19,9 +19,8 @@ #include "CPP/IocSensor.h" #include "RPC/HttpRpcClient.h" #include "RPC/XMLRPC.h" -#include "RTL/Logger.h" -#include "RTL/strdef.h" #include "RTL/rtl.h" +#include "RTL/strdef.h" #include "dim/dic.h" #include "dim/dis.h" @@ -51,11 +50,10 @@ enum Commands { typedef void* pvoid; /// Default constructor -TaskCheck::TaskCheck(const string& dns, const std::string& tms_dns, const string& node) { +TaskCheck::TaskCheck(const string& dns, const string& node) { m_dns = dns; m_node = node; m_connectTime = system_clock::from_time_t(0); - m_tmDicDns = ::dic_add_dns(tms_dns.c_str(), ::dim_get_dns_port()); } /// Default destructor @@ -77,10 +75,9 @@ bool TaskCheck::setStrictMatching(bool new_value) { /// Initalization callback bool TaskCheck::initialize() { if ( !m_inited ) { - m_tmName = "/FMC/"+RTL::str_upper(m_node)+"/task_manager/"; - m_tmInfo = ::dic_info_service_dns(m_tmDicDns, - (m_tmName+"longList").c_str(), MONITORED, - 0, 0, 0, tmSrvInfoHandler, long(this), 0, 0); + m_tmName = "/FMC/"+RTL::str_upper(m_node)+"/task_manager/"; + m_tmInfo = ::dic_info_service((m_tmName+"longList").c_str(), MONITORED, + 0, 0, 0, tmSrvInfoHandler, long(this), 0, 0); TimeSensor::instance().add(this, 1, pvoid(TASKCHECK_LOAD_DB)); TimeSensor::instance().add(this, m_dbCheckUpdateTmo, pvoid(TASKCHECK_CHECK_DB)); TimeSensor::instance().add(this, 1, pvoid(TASKCHECK_CHECK_TM)); @@ -156,6 +153,7 @@ void TaskCheck::load_tm_info(const vector<char>& data) { /// Load all relevant information from the database server void TaskCheck::load_db_info() { int count = 0; + std::string bad_servers; for( const auto& s : servers ) { try { m_api.handler = s.get(); @@ -177,28 +175,32 @@ void TaskCheck::load_db_info() { return; } catch(const exception& e) { + bad_servers += s->name() + " "; ::lib_rtl_output(LIB_RTL_DEBUG, - "+++ %s: Failed to connect database server: %s", s->name().c_str(), e.what()); + "+++ %s: Failed to connect server: %s", s->name().c_str(), e.what()); TimeSensor::instance().add(this, m_dbReconnectTmo, pvoid(TASKCHECK_LOAD_DB)); m_api.handler = 0; } catch(...) { + bad_servers += s->name() + " "; ::lib_rtl_output(LIB_RTL_DEBUG, - "+++ %s: Failed to connect database server.", s->name().c_str()); + "+++ %s: Failed to connect server.", s->name().c_str()); TimeSensor::instance().add(this, m_dbReconnectTmo, pvoid(TASKCHECK_LOAD_DB)); m_api.handler = 0; } } - ::lib_rtl_output(LIB_RTL_WARNING,"+++ Failed to connect database server for update. Will retry later."); + ::lib_rtl_output(LIB_RTL_WARNING,"+++ Failed to connect servers %sfor update." + " Will retry later.", bad_servers.c_str()); } void TaskCheck::check_db_for_update() { time_t stamp = system_clock::to_time_t(m_connectTime); + std::string bad_servers; for( const auto& s : servers ) { try { m_api.handler = s.get(); auto result = m_api.needDbUpdate(m_node, 1234, stamp); - if ( result ) { + if ( result || m_db_tasks.empty() ) { ::lib_rtl_output(LIB_RTL_INFO, "+++ Need to update tasks from DB (%s): %ld", s->name().c_str(), long(stamp)); IocSensor::instance().send(this, TASKCHECK_LOAD_DB); @@ -211,17 +213,20 @@ void TaskCheck::check_db_for_update() { } catch(const exception& e) { ::lib_rtl_output(LIB_RTL_DEBUG, - "+++ Failed to check database server %s for updates: %s", + "+++ Failed to check server %s for updates: %s", s->name().c_str(), e.what()); + bad_servers += s->name() + " "; } catch(...) { - ::lib_rtl_output(LIB_RTL_DEBUG,"+++ Failed to check database server %s for updates.", + ::lib_rtl_output(LIB_RTL_DEBUG,"+++ Failed to check server %s for updates.", s->name().c_str()); + bad_servers += s->name() + " "; } } m_api.handler = 0; TimeSensor::instance().add(this, m_dbCheckUpdateTmo, pvoid(TASKCHECK_CHECK_DB)); - ::lib_rtl_output(LIB_RTL_WARNING,"+++ Failed to check database server(s) for update. Will retry later."); + ::lib_rtl_output(LIB_RTL_WARNING,"+++ Failed to check server(s) %sfor update." + " Will retry later.", bad_servers.c_str()); } /// Kill task instance using the tmSrv @@ -277,7 +282,7 @@ bool TaskCheck::start_task(const Task& task) { command = cmd.str(); for(const auto& r : m_replacements) command = RTL::str_replace(command,r.first,r.second); - int ret = ::dic_cmnd_service_dns(m_tmDicDns, svc.c_str(), (void*)command.c_str(), command.length()+1); + int ret = ::dic_cmnd_service(svc.c_str(), (void*)command.c_str(), command.length()+1); if ( ret == 1 ) { ::lib_rtl_output(LIB_RTL_INFO, "======= Successfully STARTED %s [%s].",task.utgid.c_str(),command.c_str()); } @@ -408,11 +413,11 @@ void TaskCheck::handleTimer(const Event& event) { } extern "C" int run_task_check(int argc, char** argv) { - string node, dns, server; - string port = "3500"; - string tm_dns = "ecstms01"; - string mount = "/TDBDATA/XMLRPC"; - string logger = ::getenv("LOGFIFO") ? ::getenv("LOGFIFO") : "/proc/self/1"; + string node, dns; + string port = "3501"; + string mount = "/TDBDATA/XMLRPC"; + string server; + string tm_dns; int print_level = LIB_RTL_INFO; RTL::CLI cli(argc, argv, [] (int, char**) { cout << " run_task_check -opt [-opt] \n" @@ -431,22 +436,29 @@ extern "C" int run_task_check(int argc, char** argv) { cli.getopt("server", 3, server); cli.getopt("mount", 3, mount); cli.getopt("tm_dns", 3, tm_dns); - cli.getopt("logger", 3, logger); cli.getopt("print", 3, print_level); - { - RTL::Logger::log_args args(print_level); - RTL::Logger::install_rtl_printer(args.level); - RTL::Logger::getGlobalDevice()->compileFormat("%-36PROCESS %-8LEVEL %-20SOURCE"); - RTL::Logger::getGlobalDevice()->set_io_buffering(RTL::Logger::LINE_BUFFERING); + + TaskCheck chk(dns, node); + ::lib_rtl_set_log_level(print_level); + ::lib_rtl_install_printer(print_msg2, 0); + lib_rtl_output(LIB_RTL_ALWAYS,"-%s- Domain RPC service of type: TaskCheck started. DNS:%s TM:%s %s%s", + node.c_str(), dns.c_str(), tm_dns.c_str(), server.empty() ? "" : "Closest Server:", + server.empty() ? "" : server.c_str()); + if ( cli.getopt("debug", 3) != 0 ) { + int wait = 1; + while ( wait ) ::lib_rtl_sleep(100); + } + if ( !server.empty() ) { + auto ports = RTL::str_split(port,','); + for( const auto& p : ports ) { + lib_rtl_output(LIB_RTL_ALWAYS,"-%s- Use TaskDB cache: %s : %s", + node.c_str(), server.c_str(), p.c_str()); + chk.servers.emplace_back(make_unique<HttpRpcClient>(server, mount, p)); + } } - TaskCheck chk(dns, tm_dns, node); - ::lib_rtl_output(LIB_RTL_ALWAYS,"Domain RPC service of type: TaskCheck started. DNS:%s TM:%s %s%s", - node.c_str(), dns.c_str(), tm_dns.c_str(), server.empty() ? "" : "Closest Server:", - server.empty() ? "" : server.c_str()); - if ( !server.empty() ) chk.servers.emplace_back(make_unique<HttpRpcClient>(server, mount, port)); chk.servers.emplace_back(make_unique<HttpRpcClient>("ecs03.lbdaq.cern.ch", mount, "3501")); - chk.servers.emplace_back(make_unique<HttpRpcClient>("slavetasks.service.consul.lhcb.cern.ch", mount, port)); - chk.servers.emplace_back(make_unique<HttpRpcClient>("ecs03.lbdaq.cern.ch", mount, "3500")); + chk.servers.emplace_back(make_unique<HttpRpcClient>("ecs03.lbdaq.cern.ch", mount, "3502")); + // chk.servers.emplace_back(make_unique<HttpRpcClient>("slavetasks.service.consul.lhcb.cern.ch", mount, port)); chk.replaceString("<NODE>", node); chk.replaceString("<DIM_DNS>", dns); chk.replaceString("<TOP_LOGGER>", "ecs03"); @@ -454,7 +466,8 @@ extern "C" int run_task_check(int argc, char** argv) { chk.initialize(); server = '/' + RTL::str_upper(RTL::nodeNameShort()) + '/' + RTL::processName(); - ::dis_set_dns_node(dns.c_str()); + ::dis_set_dns_node(tm_dns.c_str()); + ::dic_set_dns_node(tm_dns.c_str()); ::dis_start_serving(server.c_str()); IocSensor::instance().run(); return 1; diff --git a/Online/RPC/src/HttpRpcClient.cpp b/Online/RPC/src/HttpRpcClient.cpp index b2d110bfc..6353d9323 100644 --- a/Online/RPC/src/HttpRpcClient.cpp +++ b/Online/RPC/src/HttpRpcClient.cpp @@ -197,16 +197,8 @@ vector<unsigned char> HttpRpcClient::call(const void* req, size_t len) const } vector<unsigned char> resp; int resp_len = 0, got_bytes = 0; -#if 0 if ( contentLength > 0 ) { - resp.reserve(std::max(contentLength,1024*1024)); - resp_len = asio::read(socket, - response, - boost::asio::transfer_exactly(contentLength), - ec); - } - else { - resp_len = asio::read(socket, response, asio::transfer_all(), ec); + resp.reserve(contentLength); } do { char c = (char)response_stream.get(); @@ -214,38 +206,36 @@ vector<unsigned char> HttpRpcClient::call(const void* req, size_t len) const ++got_bytes; resp.push_back(c); } while ( true ); -#endif - if ( contentLength > 0 ) { - resp.reserve(contentLength); - } - do { - char c = (char)response_stream.get(); - if ( !response_stream.good() ) break; - ++got_bytes; - resp.push_back(c); - } while ( true ); - if ( contentLength > 0 ) { + if ( contentLength > 0 ) { + if ( got_bytes < contentLength ) { unsigned char chunk[0x4000]; - int num_bytes = 0; + int num_bytes = 0; do { - num_bytes = asio::read(socket,asio::buffer(chunk, contentLength-got_bytes), ec); - if ( ec ) break; - if ( num_bytes < 0 ) break; - copy(chunk,chunk+num_bytes,back_inserter(resp)); - got_bytes += num_bytes; + size_t nb = std::min(size_t(contentLength-got_bytes), sizeof(chunk)); + if ( nb > 0 ) { + num_bytes = asio::read(socket, asio::buffer(chunk, nb), asio::transfer_exactly(nb), ec); + if ( ec ) break; + if ( num_bytes < 0 ) break; + copy(chunk, chunk+num_bytes, back_inserter(resp)); + got_bytes += num_bytes; + } } while ( !ec && got_bytes < contentLength ); } - else { - resp_len = asio::read(socket, response, asio::transfer_all(), ec); - istream resp_stream(&response); + resp.resize(resp_len=contentLength); + } + else { + resp_len = asio::read(socket, response, asio::transfer_all(), ec); + istream resp_stream(&response); + if ( 0 == contentLength ) { contentLength = resp_len + got_bytes; - do { - char c = (char)resp_stream.get(); - if ( !resp_stream.good() ) break; - ++got_bytes; - resp.push_back(c); - } while ( true ); } + do { + char c = (char)resp_stream.get(); + if ( !resp_stream.good() ) break; + ++got_bytes; + resp.push_back(c); + } while ( true ); + } if ( int(resp_len) != contentLength && int(got_bytes) != contentLength ) { printout(ERROR,"RPCClient","Handling response[%ld bytes]: got %ld bytes. [error:%d %s]", diff --git a/Online/sqldb/sqldb/sqldb-imp.h b/Online/sqldb/sqldb/sqldb-imp.h index f643c3e1d..113d9e450 100644 --- a/Online/sqldb/sqldb/sqldb-imp.h +++ b/Online/sqldb/sqldb/sqldb-imp.h @@ -24,12 +24,12 @@ namespace sqldb { /// Call on failure: Throws exception - SQLDB_INLINE int invalid_statement(const char* msg) { + SQLDB_INLINE sqldb_status invalid_statement(const char* msg) { throw std::runtime_error(std::string("Statement Error: ") + (msg ? msg : " [Unknwon Error]")); } /// Call on failure: Throws exception - SQLDB_INLINE int invalid_statement(const std::string& msg) { + SQLDB_INLINE sqldb_status invalid_statement(const std::string& msg) { throw std::runtime_error("Statement Error: " + msg); } @@ -148,14 +148,15 @@ namespace sqldb { } /// Bind pointers for writing - SQLDB_INLINE void column_data::bind_pointer(const unsigned char* ptr, size_t len) { + SQLDB_INLINE void column_data::bind_pointer(const void* ptr, size_t len) { if ( len < sizeof(column_data::buf) ) { ::memcpy(this->buf, ptr, len); this->item.pointer = this->buf; } else { + pointers_t p(ptr); this->vbuf.resize(len); - this->vbuf.assign(ptr, ptr+len); + this->vbuf.assign(p.puint8, p.puint8 + len); this->item.pointer = &this->vbuf.front(); } this->length = len; @@ -187,7 +188,7 @@ namespace sqldb { } /// Close the connection to the database - SQLDB_INLINE int database::close() { + SQLDB_INLINE sqldb_status database::close() { if ( intern.get() ) { intern->action(backend::CLOSE); intern->db = nullptr; @@ -197,18 +198,18 @@ namespace sqldb { } /// Start transaction - SQLDB_INLINE int database::begin() const { - return intern->action(backend::BEGIN); + SQLDB_INLINE sqldb_status database::begin() const { + return (sqldb_status)intern->action(backend::BEGIN); } /// Commit transaction - SQLDB_INLINE int database::commit() const { - return intern->action(backend::COMMIT); + SQLDB_INLINE sqldb_status database::commit() const { + return (sqldb_status)intern->action(backend::COMMIT); } /// Roll-back transaction - SQLDB_INLINE int database::rollback() const { - return intern->action(backend::ROLLBACK); + SQLDB_INLINE sqldb_status database::rollback() const { + return (sqldb_status)intern->action(backend::ROLLBACK); } /// Access the number of rows affected by the last changes @@ -217,12 +218,12 @@ namespace sqldb { } /// Execute SQL statement - SQLDB_INLINE int database::execute(std::string& error, const char* sql) const { + SQLDB_INLINE sqldb_status database::execute(std::string& error, const char* sql) const { return intern->execute(error, sql); } /// Execute SQL statement - SQLDB_INLINE int database::execute_sql(std::string& error, const char* fmt, ...) const { + SQLDB_INLINE sqldb_status database::execute_sql(std::string& error, const char* fmt, ...) const { char str[4096]; va_list args; va_start(args, fmt); @@ -232,12 +233,12 @@ namespace sqldb { } /// Prepare a new statement and allocate resources. - SQLDB_INLINE int database::prepare(statement& stmt, const std::string& sql) const { + SQLDB_INLINE sqldb_status database::prepare(statement& stmt, const std::string& sql) const { return intern->prepare(stmt, sql); } /// Fetch next result row. Stop if result is <> traits::OK - SQLDB_INLINE int record_set::fetch_one() const { + SQLDB_INLINE sqldb_status record_set::fetch_one() const { if ( this->stmt ) return this->stmt->fetch_one(); return invalid_statement("fetch_one: Invalid recordset statement handle"); } @@ -347,100 +348,100 @@ namespace sqldb { } /// Generic bind of a single parameter column - SQLDB_INLINE int record_bind::bind (size_t column, field_type typ, const void* data, size_t len) const { + SQLDB_INLINE sqldb_status record_bind::bind (size_t column, field_type typ, const void* data, size_t len) const { return stmt->_bind(column, typ, data, len); } /// Type specific bind operator for blobs - SQLDB_INLINE int record_bind::bind(size_t col, const blob_t& data) const { + SQLDB_INLINE sqldb_status record_bind::bind(size_t col, const blob_t& data) const { return stmt->_bind(col, TYPE_BLOB, data.first, data.second); } /// Type specific bind operator for blobs - SQLDB_INLINE int record_bind::bind(size_t col, const vblob_t& data) const { + SQLDB_INLINE sqldb_status record_bind::bind(size_t col, const vblob_t& data) const { return stmt->_bind(col, TYPE_BLOB, &data.at(0), data.size()); } /// Type specific bind operator for blobs - SQLDB_INLINE int record_bind::bind(size_t col, const uint8_t* data, size_t len) const { + SQLDB_INLINE sqldb_status record_bind::bind(size_t col, const uint8_t* data, size_t len) const { return stmt->_bind(col, TYPE_BLOB, data, len); } /// Type specific bind operator for strings - SQLDB_INLINE int record_bind::bind(size_t col, std::string& data) const { + SQLDB_INLINE sqldb_status record_bind::bind(size_t col, std::string& data) const { return stmt->_bind(col, TYPE_STRING, data.c_str(), data.length()); } /// Type specific bind operator for strings - SQLDB_INLINE int record_bind::bind(size_t col, const std::string& data) const { + SQLDB_INLINE sqldb_status record_bind::bind(size_t col, const std::string& data) const { return stmt->_bind(col, TYPE_STRING, data.c_str(), data.length()); } /// Type specific bind operator for strings - SQLDB_INLINE int record_bind::bind(size_t col, char* const data) const { + SQLDB_INLINE sqldb_status record_bind::bind(size_t col, char* const data) const { return stmt->_bind(col, TYPE_STRING, data ? data : "", data ? ::strlen(data) : 0); } /// Type specific bind operator for strings - SQLDB_INLINE int record_bind::bind(size_t col, const char* const data) const { + SQLDB_INLINE sqldb_status record_bind::bind(size_t col, const char* const data) const { return stmt->_bind(col, TYPE_STRING, data ? data : "", data ? ::strlen(data) : 0); } /// Type specific bind operator for integer type - template <> SQLDB_INLINE int record_bind::bind<char>(size_t col, char data) const { + template <> SQLDB_INLINE sqldb_status record_bind::bind<char>(size_t col, char data) const { return stmt->_bind(col, TYPE_TINY, &data, sizeof(data)); } /// Type specific bind operator for integer type - template <> SQLDB_INLINE int record_bind::bind<signed char>(size_t col, signed char data) const { + template <> SQLDB_INLINE sqldb_status record_bind::bind<signed char>(size_t col, signed char data) const { return stmt->_bind(col, TYPE_TINY, &data, sizeof(data)); } /// Type specific bind operator for integer type - template <> SQLDB_INLINE int record_bind::bind<unsigned char>(size_t col, unsigned char data) const { + template <> SQLDB_INLINE sqldb_status record_bind::bind<unsigned char>(size_t col, unsigned char data) const { return stmt->_bind(col, TYPE_TINY, &data, sizeof(data)); } /// Type specific bind operator for integer type - template <> SQLDB_INLINE int record_bind::bind<int16_t>(size_t col, int16_t data) const { + template <> SQLDB_INLINE sqldb_status record_bind::bind<int16_t>(size_t col, int16_t data) const { return stmt->_bind(col, TYPE_SHORT, &data, sizeof(data)); } /// Type specific bind operator for integer type - template <> SQLDB_INLINE int record_bind::bind<uint16_t>(size_t col, uint16_t data) const { + template <> SQLDB_INLINE sqldb_status record_bind::bind<uint16_t>(size_t col, uint16_t data) const { return stmt->_bind(col, TYPE_SHORT, &data, sizeof(data)); } /// Type specific bind operator for integer type - template <> SQLDB_INLINE int record_bind::bind<int32_t>(size_t col, int32_t data) const { + template <> SQLDB_INLINE sqldb_status record_bind::bind<int32_t>(size_t col, int32_t data) const { return stmt->_bind(col, TYPE_LONG, &data, sizeof(data)); } /// Type specific bind operator for integer type - template <> SQLDB_INLINE int record_bind::bind<uint32_t>(size_t col, uint32_t data) const { + template <> SQLDB_INLINE sqldb_status record_bind::bind<uint32_t>(size_t col, uint32_t data) const { return stmt->_bind(col, TYPE_LONG, &data, sizeof(data)); } /// Type specific bind operator for integer type - template <> SQLDB_INLINE int record_bind::bind<int64_t>(size_t col, int64_t data) const { + template <> SQLDB_INLINE sqldb_status record_bind::bind<int64_t>(size_t col, int64_t data) const { return stmt->_bind(col, TYPE_LONGLONG, &data, sizeof(data)); } /// Type specific bind operator for integer type - template <> SQLDB_INLINE int record_bind::bind<uint64_t>(size_t col, uint64_t data) const { + template <> SQLDB_INLINE sqldb_status record_bind::bind<uint64_t>(size_t col, uint64_t data) const { return stmt->_bind(col, TYPE_LONGLONG, &data, sizeof(data)); } /// Type specific bind operator for - template <> SQLDB_INLINE int record_bind::bind<float>(size_t col, float data) const { + template <> SQLDB_INLINE sqldb_status record_bind::bind<float>(size_t col, float data) const { return stmt->_bind(col, TYPE_FLOAT, &data, sizeof(data)); } - template <> SQLDB_INLINE int record_bind::bind<double>(size_t col, double data) const { + template <> SQLDB_INLINE sqldb_status record_bind::bind<double>(size_t col, double data) const { return stmt->_bind(col, TYPE_DOUBLE, &data, sizeof(data)); } - template <> SQLDB_INLINE int record_bind::bind<struct time>(size_t col, struct time data) const { + template <> SQLDB_INLINE sqldb_status record_bind::bind<struct time>(size_t col, struct time data) const { return stmt->_bind(col, TYPE_TIME, &data.stamp, sizeof(data.stamp)); } @@ -468,12 +469,12 @@ namespace sqldb { } /// Create fully prepared statement in a database context - SQLDB_INLINE int statement::prepare(const database& db, const std::string& sql) { + SQLDB_INLINE sqldb_status statement::prepare(const database& db, const std::string& sql) { return db.prepare(*this, sql); } /// Execute SQL statement with variable number of arguments - SQLDB_INLINE int statement::prepare(const database& db, const char* fmt, ...) { + SQLDB_INLINE sqldb_status statement::prepare(const database& db, const char* fmt, ...) { char sql[4096]; va_list args; va_start(args, fmt); @@ -490,29 +491,29 @@ namespace sqldb { ::vsnprintf(sql, sizeof(sql), fmt, args); va_end(args); statement stmt; - int ret = db.prepare(stmt, sql); + sqldb_status ret = db.prepare(stmt, sql); if ( ret == sqldb::OK ) return stmt; return statement(); } /// Execute prepared statement - SQLDB_INLINE int statement::execute() const { - return intern->action(backend::EXEC); + SQLDB_INLINE sqldb_status statement::execute() const { + return (sqldb_status)intern->action(backend::EXEC); } /// Fetch next result row. Stop if result is <> traits::OK - SQLDB_INLINE int statement::fetch_one() const { - return intern->action(backend::FETCH); + SQLDB_INLINE sqldb_status statement::fetch_one() const { + return (sqldb_status)intern->action(backend::FETCH); } /// Reset SQL statement: Reset all data to re-execute the statement with new parameters - SQLDB_INLINE int statement::reset() const { - return intern->action(backend::RESET); + SQLDB_INLINE sqldb_status statement::reset() const { + return (sqldb_status)intern->action(backend::RESET); } /// Finalize and run-down the statement. Release all resources. - SQLDB_INLINE int statement::finalize() { + SQLDB_INLINE sqldb_status statement::finalize() { if ( intern.get() ) { intern->action(backend::FINALIZE); intern.reset(); @@ -521,7 +522,7 @@ namespace sqldb { } /// Generic bind of a single parameter column - SQLDB_INLINE int statement::_bind(size_t column, field_type typ, const void* data, size_t len) const { + SQLDB_INLINE sqldb_status statement::_bind(size_t column, field_type typ, const void* data, size_t len) const { return intern->bind(column, typ, data, len); } diff --git a/Online/sqldb/sqldb/sqldb.h b/Online/sqldb/sqldb/sqldb.h index 83aa2866c..bebac399b 100644 --- a/Online/sqldb/sqldb/sqldb.h +++ b/Online/sqldb/sqldb/sqldb.h @@ -71,8 +71,8 @@ namespace sqldb { class time; /// Call on failure: Throws exception - int invalid_statement(const char* msg = "[Unknown Error]"); - int invalid_statement(const std::string& msg); + sqldb_status invalid_statement(const char* msg = "[Unknown Error]"); + sqldb_status invalid_statement(const std::string& msg); /// Connection argument parser /** @@ -256,9 +256,16 @@ namespace sqldb { bool is_null() const { return length == 0 || type == TYPE_NULL; } - + /// Clear data structure + void clear() { + this->length = 0; + this->item.int64 = 0; + this->type = TYPE_NULL; + this->vbuf.clear(); + this->buf[0] = 0; + } /// Bind pointers for writing - void bind_pointer(const unsigned char* ptr, size_t len); + void bind_pointer(const void* ptr, size_t len); }; /// Database handle class @@ -302,12 +309,12 @@ namespace sqldb { /// Perform multiple actions virtual int action(action_type type) = 0; /// Execute prepared statement - virtual int execute(std::string& err, const std::string& sql) = 0; + virtual sqldb_status execute(std::string& err, const std::string& sql) = 0; /// Prepare a new statement and allocate resources. - virtual int prepare(statement& stmt, const std::string& sql) = 0; + virtual sqldb_status prepare(statement& stmt, const std::string& sql) = 0; /// Open the database connection using all information in passed string template <typename BACKEND> - static std::pair<std::shared_ptr<backend>, int> open(const std::string& name); + static std::pair<std::shared_ptr<backend>, sqldb_status> open(const std::string& name); }; std::shared_ptr<backend> intern; @@ -342,7 +349,7 @@ namespace sqldb { int error() const; /// Open the database connection using all information in passed string - template <typename BACKEND> int open(const std::string& con) { + template <typename BACKEND> sqldb_status open(const std::string& con) { auto result = backend::open<BACKEND>(con); this->intern = result.first; if ( this->intern.get() ) { @@ -361,21 +368,21 @@ namespace sqldb { return db; } /// Close the connection to the database - int close(); + sqldb_status close(); /// Start transaction - int begin() const; + sqldb_status begin() const; /// Commit transaction - int commit() const; + sqldb_status commit() const; /// Roll-back transaction - int rollback() const; + sqldb_status rollback() const; /// Access the number of rows affected by the last changes long rows_affected() const; /// Execute SQL statement - int execute(std::string& error, const char* sql) const; + sqldb_status execute(std::string& error, const char* sql) const; /// Execute SQL statement with variable number of arguments - int execute_sql(std::string& error, const char* fmt, ...) const; + sqldb_status execute_sql(std::string& error, const char* fmt, ...) const; /// Prepare a new statement and allocate resources. - int prepare(statement& stmt, const std::string& sql) const; + sqldb_status prepare(statement& stmt, const std::string& sql) const; }; /// Check handle validity @@ -452,7 +459,7 @@ namespace sqldb { record_set& operator = (const record_set& copy) = default; /// Fetch next result row. Stop if result is <> traits::OK - int fetch_one() const; + sqldb_status fetch_one() const; }; /// Record binder class @@ -483,23 +490,23 @@ namespace sqldb { record_bind& operator = (const record_bind& copy) = default; /// Generic bind of a single parameter column - int bind(size_t column, field_type typ, const void* data, size_t len) const; + sqldb_status bind(size_t column, field_type typ, const void* data, size_t len) const; /// Type specific bind operator for blobs - int bind(size_t column, const blob_t& data) const; + sqldb_status bind(size_t column, const blob_t& data) const; /// Type specific bind operator for blobs - int bind(size_t column, const vblob_t& data) const; + sqldb_status bind(size_t column, const vblob_t& data) const; /// Type specific bind operator for blobs - int bind(size_t column, const uint8_t* data, size_t len) const; + sqldb_status bind(size_t column, const uint8_t* data, size_t len) const; /// Type specific bind operator for strings - int bind(size_t column, char* const data) const; + sqldb_status bind(size_t column, char* const data) const; /// Type specific bind operator for strings - int bind(size_t column, const char* const data) const; + sqldb_status bind(size_t column, const char* const data) const; /// Type specific bind operator for strings - int bind(size_t column, std::string& data) const; + sqldb_status bind(size_t column, std::string& data) const; /// Type specific bind operator for strings - int bind(size_t column, const std::string& data) const; + sqldb_status bind(size_t column, const std::string& data) const; /// Type specific bind operator for blobs - template <typename T> int bind(size_t column, T data) const; + template <typename T> sqldb_status bind(size_t column, T data) const; }; /// Statement handle class @@ -544,9 +551,9 @@ namespace sqldb { /// Access the number of rows affected by the last statement virtual long rows_affected() = 0; /// Perform multiple backend operations - virtual int action(action_type type) = 0; + virtual int action(action_type type) = 0; /// Bind a single column to the data passed - virtual int bind(size_t col, field_type typ, const void* data, size_t len) = 0; + virtual sqldb_status bind(size_t col, field_type typ, const void* data, size_t len) = 0; }; std::shared_ptr<backend> intern; @@ -555,7 +562,7 @@ namespace sqldb { /// Access raw data from result handle column_data& _field(size_t column) const; /// Generic bind of a single parameter column - int _bind(size_t column, field_type typ, const void* data, size_t len) const; + sqldb_status _bind(size_t column, field_type typ, const void* data, size_t len) const; public: @@ -577,9 +584,9 @@ namespace sqldb { static statement create(const database& db, const char* fmt, ...); /// Prepare a new statement and allocate resources. - int prepare(const database& db, const std::string& sql); + sqldb_status prepare(const database& db, const std::string& sql); /// Execute SQL statement with variable number of arguments - int prepare(const database& db, const char* fmt, ...); + sqldb_status prepare(const database& db, const char* fmt, ...); /// Access the number of result fields of the prepared statement size_t param_count() const; @@ -597,16 +604,16 @@ namespace sqldb { long rows_affected() const; /// Execute prepared statement - int execute() const; + sqldb_status execute() const; /// Fetch next result row. Stop if result is <> traits::OK - int fetch_one() const; + sqldb_status fetch_one() const; /// Reset SQL statement: Reset all data to re-execute the statement with new parameters - int reset() const; + sqldb_status reset() const; /// Finalize and run-down the statement. Release all resources. - int finalize(); + sqldb_status finalize(); /// Access to the current selected row record current_row() const; diff --git a/Online/sqldb/src/mysql.cpp b/Online/sqldb/src/mysql.cpp index 5e458a2a8..bf5a1bad1 100644 --- a/Online/sqldb/src/mysql.cpp +++ b/Online/sqldb/src/mysql.cpp @@ -76,7 +76,7 @@ namespace sqldb { struct column_bind : public MYSQL_BIND { public: /// Bind parameter to null value - int bind_store_null(column_data* d) { + sqldb_status bind_store_null(column_data* d) { d->null = 1; this->is_null = &d->null; this->buffer = nullptr; @@ -85,7 +85,7 @@ namespace sqldb { return sqldb::OK; } /// Generic parameter binding - int bind_store(column_data* d, const unsigned char* ptr, size_t len) { + sqldb_status bind_store(column_data* d, const unsigned char* ptr, size_t len) { if ( ptr ) { d->bind_pointer(ptr, len); this->buffer = d->item.string; @@ -97,7 +97,7 @@ namespace sqldb { return this->bind_store_null(d); } /// Type specific parameter bind - template <typename T> int bind_store(column_data* d, T& to, const T* from) { + template <typename T> sqldb_status bind_store(column_data* d, T& to, const T* from) { if ( from ) { to = *from; d->length = sizeof(T); @@ -110,7 +110,7 @@ namespace sqldb { return this->bind_store_null(d); } /// Bind output field - int bind_fetch(column_data* d, void* to, size_t len) { + sqldb_status bind_fetch(column_data* d, void* to, size_t len) { d->null = false; d->error = false; d->length = 0; @@ -161,7 +161,7 @@ namespace sqldb { } /// Prepare a new statement and allocate resources. - int prepare(mysql::dbase_t* db, const std::string& sql_str) { + sqldb_status prepare(mysql::dbase_t* db, const std::string& sql_str) { MYSQL_STMT* stmt = nullptr; if ( _imp ) ::mysql_stmt_reset(_imp); if ( _imp ) ::mysql_stmt_close(_imp); @@ -193,7 +193,7 @@ namespace sqldb { } /// Execute prepared statement - virtual int execute() { + virtual sqldb_status execute() { if( auto* h = this->_imp ) { if( this->_pbind.size() ) { if( ::mysql_stmt_bind_param(h, &this->_pbind.front()) ) { @@ -215,7 +215,7 @@ namespace sqldb { } /// Fetch next result row. Stop if result is NON-zero - virtual int fetch_one() { + virtual sqldb_status fetch_one() { if ( this->_imp ) { size_t nfld = this->_fbind.size(); if ( nfld > 0 ) { @@ -251,7 +251,7 @@ namespace sqldb { } /// Bind output field identified by column number - int bind_fetch(size_t col) { + sqldb_status bind_fetch(size_t col) { auto* p = this; auto [d, b, f] = p->field(col); @@ -274,7 +274,7 @@ namespace sqldb { } /// Bind a single column to the data passed - virtual int bind(size_t col, field_type typ, const void* data, size_t len) override { + virtual sqldb_status bind(size_t col, field_type typ, const void* data, size_t len) override { pointers_t ptr(data); auto [d, b] = this->param(col); ::memset(b, 0, sizeof(column_bind)); @@ -372,7 +372,7 @@ namespace sqldb { } } /// Execute prepared statement - virtual int execute(std::string& err, const std::string& sql) override { + virtual sqldb_status execute(std::string& err, const std::string& sql) override { if( this->_handle ) { if( ::mysql_real_query(this->_handle, sql.c_str(), sql.length()) ) { err = ::mysql_error(this->_handle); @@ -383,13 +383,13 @@ namespace sqldb { return sqldb::INVALID_HANDLE; } /// Prepare a new statement and allocate resources. - int prepare(statement& bs, const std::string& sql_str) override { + sqldb_status prepare(statement& bs, const std::string& sql_str) override { auto* stm_imp = new statement_backend(); bs.intern.reset(stm_imp); return stm_imp->prepare(this->handle(), sql_str); } /// Open database - int open(const std::string& connect_string) { + sqldb_status open(const std::string& connect_string) { auto args = connection_args().parse(connect_string); this->_handle = nullptr; if ( nullptr == ::mysql_real_connect(&_imp, @@ -416,10 +416,10 @@ namespace sqldb { /// Open the database connection using all information in passed string template <> - std::pair<std::shared_ptr<database::backend>, int> + std::pair<std::shared_ptr<database::backend>, sqldb_status> database::backend::open<mysql>(const std::string& connect_string) { auto db = std::make_shared<mysql_imp::dbase_backend>(); - int ret = db->open(connect_string); + sqldb_status ret = db->open(connect_string); if ( ret == sqldb::OK ) return make_pair(db, sqldb::OK); return make_pair(std::shared_ptr<database::backend>(), ret); diff --git a/Online/sqldb/src/oracle.cpp b/Online/sqldb/src/oracle.cpp index d8b2d73ec..5bb61a68d 100644 --- a/Online/sqldb/src/oracle.cpp +++ b/Online/sqldb/src/oracle.cpp @@ -147,19 +147,19 @@ namespace sqldb { /// Default destructor ~column_bind(); /// Generic parameter binding - int bind_str(column_data& d, const char* ptr, size_t len); + sqldb_status bind_str(column_data& d, const char* ptr, size_t len); /// Generic parameter binding - int bind_blob(column_data& d, const unsigned char* ptr, size_t len); + sqldb_status bind_blob(column_data& d, const unsigned char* ptr, size_t len); /// Type specific parameter bind - int bind_int(column_data& d, int32_t from); + sqldb_status bind_int(column_data& d, int32_t from); /// Type specific parameter bind - int bind_int64(column_data& d, int64_t from); + sqldb_status bind_int64(column_data& d, int64_t from); /// Type specific parameter bind - int bind_flt(column_data& d, double from); + sqldb_status bind_flt(column_data& d, double from); /// Read lob data - int fetch_data(column_data& d); + sqldb_status fetch_data(column_data& d); /// Bind output field - int bind_fetch(column_data& d, void* ptr, size_t len); + sqldb_status bind_fetch(column_data& d, void* ptr, size_t len); }; /// Technology abstraction layer @@ -209,14 +209,6 @@ namespace sqldb { field_t field(int col) { return field_t(&this->fields[col], &_fbind[col], _types[col]); } - /// Execute prepared statement - int execute(); - /// Fetch next result row. Stop if result is NON-zero - int fetch_one(); - /// Bind output field identified by column number - int bind_fetch(size_t col); - /// Prepare a new statement and allocate resources. - int prepare(const std::string& sql_str); /// Access error message on failure std::string errmsg() override { return error_message(_error_code, _error); @@ -234,10 +226,18 @@ namespace sqldb { _error_code = ret; return -1; } + /// Execute prepared statement + sqldb_status execute(); + /// Fetch next result row. Stop if result is NON-zero + sqldb_status fetch_one(); + /// Bind output field identified by column number + sqldb_status bind_fetch(size_t col); + /// Prepare a new statement and allocate resources. + sqldb_status prepare(const std::string& sql_str); /// Bind a single column to the data passed - virtual int bind(size_t col, field_type typ, const void* data, size_t len) override; + virtual sqldb_status bind(size_t col, field_type typ, const void* data, size_t len) override; /// - int action(action_type type) override; + virtual int action(action_type type) override; }; /// Technology abstraction layer for the database @@ -264,9 +264,9 @@ namespace sqldb { throw std::runtime_error("Invalid MySQL database handle"); } /// Prepare a new statement and allocate resources. - int prepare(statement& bs, const std::string& sql_str) override; + sqldb_status prepare(statement& bs, const std::string& sql_str) override; /// Open database - int open(const std::string& connect_string); + sqldb_status open(const std::string& connect_string); /// Access error message on failure std::string errmsg() override { return error_message(_error_code, _error); @@ -276,7 +276,7 @@ namespace sqldb { return -1; } /// Execute prepared statement - virtual int execute(std::string& error, const std::string& sql_str) override; + virtual sqldb_status execute(std::string& error, const std::string& sql_str) override; /// Perform multiple actions virtual int action(action_type type) override; }; @@ -290,7 +290,7 @@ namespace sqldb { } /// Generic parameter binding - int column_bind::bind_str(column_data& d, const char* ptr, size_t len) { + sqldb_status column_bind::bind_str(column_data& d, const char* ptr, size_t len) { pointers_t p(ptr); d.bind_pointer(p.puint8, len); auto ret = ::OCIBindByPos(_stm->_stmt, &_bind, _stm->_error, @@ -299,7 +299,7 @@ namespace sqldb { return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; } /// Generic parameter binding - int column_bind::bind_blob(column_data& d, const unsigned char* ptr, size_t len) { + sqldb_status column_bind::bind_blob(column_data& d, const unsigned char* ptr, size_t len) { pointers_t p(ptr); d.bind_pointer(ptr, len); auto ret = ::OCIBindByPos(_stm->_stmt, &_bind, _stm->_error, @@ -308,7 +308,7 @@ namespace sqldb { return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; } /// Type specific parameter bind - int column_bind::bind_int(column_data& d, int32_t from) { + sqldb_status column_bind::bind_int(column_data& d, int32_t from) { d.item.int32 = from; d.length = sizeof(int); result_t ret = ::OCIBindByPos(_stm->_stmt, &_bind, _stm->_error, @@ -317,7 +317,7 @@ namespace sqldb { return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; } /// Type specific parameter bind - int column_bind::bind_int64(column_data& d, int64_t from) { + sqldb_status column_bind::bind_int64(column_data& d, int64_t from) { d.item.int64 = from; d.length = sizeof(int64_t); result_t ret = ::OCIBindByPos(_stm->_stmt, &_bind, _stm->_error, @@ -326,7 +326,7 @@ namespace sqldb { return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; } /// Type specific parameter bind - int column_bind::bind_flt(column_data& d, double from) { + sqldb_status column_bind::bind_flt(column_data& d, double from) { d.item.real64 = from; d.length = sizeof(double); result_t ret = ::OCIBindByPos(_stm->_stmt, &_bind, _stm->_error, @@ -335,7 +335,7 @@ namespace sqldb { return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; } /// Read lob data - int column_bind::fetch_data(column_data& d) { + sqldb_status column_bind::fetch_data(column_data& d) { result_t rc = OCI_SUCCESS; DbcH db = _stm->_dbc; ErrorH err = _stm->_error; @@ -356,7 +356,7 @@ namespace sqldb { return oracle_ok(rc) ? sqldb::OK : sqldb::ERROR; } /// Bind output field - int column_bind::bind_fetch(column_data& d, void* ptr, size_t len) { + sqldb_status column_bind::bind_fetch(column_data& d, void* ptr, size_t len) { result_t rc = OCI_SUCCESS; d.null = false; d.error = false; @@ -380,7 +380,7 @@ namespace sqldb { } /// Fetch next result row. Stop if result is NON-zero - int statement_backend::fetch_one() { + sqldb_status statement_backend::fetch_one() { if ( _stmt ) { size_t nfld = _fbind.size(); if ( nfld > 0 ) { @@ -407,7 +407,7 @@ namespace sqldb { } /// Bind output field identified by column number - int statement_backend::bind_fetch(size_t col) { + sqldb_status statement_backend::bind_fetch(size_t col) { auto* p = this; auto [d, b, type] = p->field(col); b->_column = col; @@ -432,7 +432,7 @@ namespace sqldb { } /// Bind a single column to the data passed - int statement_backend::bind(size_t col, field_type typ, const void* data, size_t len) { + sqldb_status statement_backend::bind(size_t col, field_type typ, const void* data, size_t len) { pointers_t ptr(data); auto [d, b] = param(col); d->type = typ; @@ -457,7 +457,7 @@ namespace sqldb { } /// Prepare a new statement and allocate resources. - int statement_backend::prepare(const std::string& sql_str) { + sqldb_status statement_backend::prepare(const std::string& sql_str) { StmH stm; ub4 count; if ( _stmt ) { @@ -555,16 +555,16 @@ namespace sqldb { _error = nullptr; } /// Execute prepared statement - int dbase_backend::execute(std::string& error, const std::string& sql_str) { + sqldb_status dbase_backend::execute(std::string& error, const std::string& sql_str) { char* err = nullptr; - int ret = OCI_SUCCESS; //do_exec(handle(), sql_str.c_str(), &err); + sqldb_status ret = sqldb::ERROR; //do_exec(handle(), sql_str.c_str(), &err); if ( !sql_str.empty() ) { if ( ret != sqldb::OK && err != nullptr ) error = err; } return ret; } /// Prepare a new statement and allocate resources. - int dbase_backend::prepare(statement& bs, const std::string& sql_str) { + sqldb_status dbase_backend::prepare(statement& bs, const std::string& sql_str) { ErrorH err {nullptr}; if ( bs.intern.get() ) { bs.intern.reset(); @@ -579,7 +579,7 @@ namespace sqldb { return sqldb::ERROR; } /// Open database - int dbase_backend::open(const std::string& connect_string) { + sqldb_status dbase_backend::open(const std::string& connect_string) { auto args = connection_args().parse(connect_string); if ( _dbc ) { ::OCILogoff(_dbc, _error); @@ -638,10 +638,10 @@ namespace sqldb { namespace sqldb { /// Open the database connection using all information in passed string - template <> std::pair<std::shared_ptr<database::backend>, int> + template <> std::pair<std::shared_ptr<database::backend>, sqldb_status> database::backend::open<oracle>(const std::string& connect_string) { auto db = std::make_shared<oracle_imp::dbase_backend>(); - int ret = db->open(connect_string); + sqldb_status ret = db->open(connect_string); if ( ret == sqldb::OK ) return make_pair(db, sqldb::OK); return make_pair(std::shared_ptr<database::backend>(), ret); diff --git a/Online/sqldb/src/postgresql.cpp b/Online/sqldb/src/postgresql.cpp index 0c3d86761..7e2688e7a 100644 --- a/Online/sqldb/src/postgresql.cpp +++ b/Online/sqldb/src/postgresql.cpp @@ -29,6 +29,8 @@ namespace sqldb { namespace postgresql_imp { + struct statement_backend; + /// std::string _escape(postgresql::dbase_t* hdl, const char* str, size_t length) { std::vector<char> buf(2*length+1); @@ -43,7 +45,7 @@ namespace sqldb { } } /// - inline int postgresql_ok(const PGresult* status) { + inline sqldb_status postgresql_ok(const PGresult* status) { if ( status ) { ExecStatusType sc = ::PQresultStatus(status); switch( sc ) { @@ -103,63 +105,47 @@ namespace sqldb { throw std::runtime_error("Invalid data type encountered!"); } -#if 0 + /// Technology abstraction layer /** * \author M.Frank * \version 1.0 * \date 02.05.2017 */ - struct column_bind : public POSTGRESQL_BIND { + struct column_bind { + public: + statement_backend* stmt { nullptr }; + int col { 0 }; + public: + /// Initializing constructor + column_bind(statement_backend* s, int c) : stmt(s), col(c) {} + /// Default constructor + column_bind() = default; + /// Move constructor + column_bind(column_bind&& copy) = default; + /// Copy constructor + column_bind(const column_bind& copy) = default; + /// Move assignment + column_bind& operator=(column_bind&& copy) = default; + /// Copy assigment + column_bind& operator=(const column_bind& copy) = default; + /// Default destructor + ~column_bind() = default; /// Bind parameter to null value - int bind_store_null(column_data* d) { - d->null = true; - this->is_null = &d->null; - this->buffer = nullptr; - this->length = nullptr; - this->error = nullptr; - return sqldb::OK; - } + sqldb_status bind_store_null(column_data& d); /// Generic parameter binding - int bind_store(column_data* d, const unsigned char* ptr, size_t len) { - if ( ptr ) { - d->bind_pointer(ptr, len); - this->buffer = d->item.string; - this->length = &d->length; - this->is_null = nullptr; - this->error = nullptr; - return sqldb::OK; - } - return this->bind_store_null(d); - } + sqldb_status bind_store(column_data& d, const char* ptr, size_t len); + /// Generic parameter binding + sqldb_status bind_store(column_data& d, const unsigned char* ptr, size_t len); /// Type specific parameter bind - template <typename T> int bind_store(column_data* d, T& to, const T* from) { - if ( from ) { - to = *from; - d->length = sizeof(T); - this->buffer = &to; - this->length = nullptr; - this->is_null = nullptr; - this->error = nullptr; - return sqldb::OK; - } - return this->bind_store_null(d); - } + template <typename T> sqldb_status bind_store(column_data& d, T& to, const T* from); /// Bind output field - int bind_fetch(column_data* d, void* to, size_t len) { - d->null = false; - d->error = false; - d->length = 0; - this->buffer = to; - this->length = &d->length; - this->is_null = &d->null; - this->error = &d->error; - this->buffer_length = len; - return sqldb::OK; - } + sqldb_status bind_fetch_blob(column_data& d); + /// Bind output field + template <typename T> sqldb_status bind_fetch(column_data& d, T* to, size_t len); }; -#endif + /// Technology abstraction layer for a Postgresql prepared statement /** * \author M.Frank @@ -174,29 +160,21 @@ namespace sqldb { binary_param } param_type; + typedef std::tuple<column_data*, column_bind*, int> field_t; postgresql::dbase_t* _handle { nullptr }; postgresql::statement_t* _result { nullptr }; - std::string _sql; std::string _id; + std::string _sql; + std::string _err; std::stringstream _fmt; - std::vector<std::string> _params; - std::vector<char const *> _params_p; - std::vector<size_t> _params_len; - std::vector<param_type> _params_typ; - int _current {0}; + std::vector<char const *> _param_ptr; + std::vector<int> _param_len; + std::vector<int> _param_fmt; + std::vector<Oid> _param_typ; + std::vector<column_bind> _fbind; + size_t _current_row {0}; + size_t _num_param {0}; -#if 0 - typedef std::tuple<column_data*, column_bind*, POSTGRESQL_FIELD*> field_t; - std::vector<column_bind> _pbind, _fbind; - POSTGRESQL_STMT* _imp { nullptr }; - - std::pair<column_data*, column_bind*> param(int col) { - return std::make_pair(¶ms[col], &_pbind[col]); - } - field_t field(int col) { - return field_t(&this->fields[col], &_fbind[col], &_imp->fields[col]); - } -#endif template <typename T> T _to(const char* s) { T result; this->_fmt.clear(); @@ -204,14 +182,25 @@ namespace sqldb { this->_fmt >> result; return result; } + field_t field(int col) { + return field_t(&this->fields[col], &_fbind[col], col); + } /// Access to the internal statement handle. Throws exception if the handle is invalid postgresql::dbase_t* handle() const { if ( this->_handle ) return this->_handle; throw std::runtime_error("Failed to access statement handle. [Invalid handle]"); } /// Access error message on failure + std::string error_msg(const postgresql::dbase_t* h) { + return ::PQerrorMessage(h); + } + /// Access error message on failure + std::string error_msg(const postgresql::statement_t* h) { + return ::PQresultErrorMessage(h); + } + /// Access error message on failure virtual std::string errmsg() override { - return ::PQerrorMessage(this->handle()); + return (this->_result) ? this->error_msg(this->_result) : this->error_msg(this->_handle); } /// Character string of the prepared statement virtual const char* sql() override { @@ -230,248 +219,269 @@ namespace sqldb { sqldb_status reset() { if( this->_handle ) { clear_result(this->_result); - std::vector<std::string> vals(_params); - std::vector<size_t> lens(_params.size(),0); - std::vector<char const *> pval(_params.size(),0); - std::vector<param_type> typs(_params.size(),null_param); - _params.swap(vals); - _params_p.swap(pval); - _params_len.swap(lens); - _params_typ.swap(typs); + std::vector<int> lens(_num_param,0); + std::vector<char const *> pval(_num_param,0); + std::vector<Oid> typs(_num_param,null_param); + this->_param_ptr.swap(pval); + this->_param_len.swap(lens); + this->_param_typ.swap(typs); return sqldb::OK; } return sqldb::ERROR; } - /// Prepare a new statement and allocate resources. - sqldb_status prepare(postgresql::dbase_t* db, const std::string& sql_str, uint64_t id) { - this->_handle = db; - this->_current = 0; - this->_fmt.imbue(std::locale::classic()); - this->_fmt.str(std::string()); - this->_fmt.clear(); - this->_fmt << "sqldb_pq_stmt_" << id; - this->_id = this->_fmt.str(); - this->_fmt.str(std::string()); - this->_fmt.clear(); - this->_sql.clear(); - this->_sql.reserve(sql_str.size()); - int num_params = 0; - bool inside = false; - for(unsigned i=0;i<sql_str.size();i++) { - char c = sql_str[i]; - switch(c) { - case '\'': - inside = !inside; - break; - case '?': - if( !inside ) { - this->_sql += '$'; - ++num_params; - this->_fmt << num_params; - this->_sql += this->_fmt.str(); - this->_fmt.str(std::string()); - this->_fmt.clear(); - } - break; - default: - this->_sql += c; - break; - } - } - - _params.resize(num_params,std::string()); - _params_typ.resize(num_params, null_param); - _params_p.resize(num_params, nullptr); - _params_len.resize(num_params, 0); - this->reset(); + sqldb_status prepare(postgresql::dbase_t* db, const std::string& sql_str, uint64_t id); + /// Execute prepared statement + virtual sqldb_status execute(); + /// Bind output field identified by column number + sqldb_status bind_fetch(size_t col); + /// Bind a single column to the data passed + virtual sqldb_status bind(size_t col, field_type typ, const void* data, size_t len) override; + /// Action routine for various functions + virtual int action(action_type type) override; + }; - this->_result = ::PQprepare(this->_handle,_id.c_str(),this->_sql.c_str(),0,0); - try { - if( !this->_result ) { - throw std::runtime_error("sqldb(postgres): Failed to create prepared statement object!"); - } - if ( ::PQresultStatus(this->_result) != PGRES_COMMAND_OK ) { - throw std::runtime_error("sqldb(postgres): statement preparation failed: "+errmsg()); - } - } - catch(...) { - clear_result(this->_result); - throw; - } - clear_result(this->_result); + /// Bind parameter to null value + sqldb_status column_bind::bind_store_null(column_data& d) { + d.null = true; + d.length = 0; + stmt->_param_ptr[col] = nullptr; + stmt->_param_len[col] = d.length; + stmt->_param_fmt[col] = 1; + return sqldb::OK; + } + + /// Generic parameter binding + sqldb_status column_bind::bind_store(column_data& d, const unsigned char* ptr, size_t len) { + if ( ptr ) { + d.bind_pointer(ptr, len); + stmt->_param_ptr[col] = d.item.string; + stmt->_param_len[col] = d.length; + stmt->_param_fmt[col] = 1; return sqldb::OK; } -#if 0 - - POSTGRESQL_STMT* stmt = nullptr; - if ( _imp ) ::postgresql_stmt_reset(_imp); - if ( _imp ) ::postgresql_stmt_close(_imp); - stmt = ::postgresql_stmt_init(db); - if ( stmt ) { - int ret = ::postgresql_stmt_prepare(stmt, sql_str.c_str(), sql_str.length()); - if ( ret == sqldb::OK ) { - int num_param = ::postgresql_stmt_param_count(stmt); - int num_field = ::postgresql_stmt_field_count(stmt); - this->_pbind.resize(num_param, column_bind()); - this->params.resize(num_param, column_data()); - this->_fbind.resize(num_field, column_bind()); - this->fields.resize(num_field, column_data()); - this->_sql = sql_str; - this->_imp = stmt; - this->stmt = this; - return sqldb::OK; - } - return sqldb::ERROR; - } -#endif + return this->bind_store_null(d); + } - /// Execute prepared statement - virtual int execute() { -#if 0 - if( auto* h = this->_imp ) { - if( this->_pbind.size() ) { - if( ::postgresql_stmt_bind_param(h, &this->_pbind.front()) ) { - return invalid_statement(::postgresql_stmt_error(h)); - } - } - if( sqldb::OK == ::postgresql_stmt_execute(h) ) { - if( sqldb::OK != ::postgresql_stmt_store_result(h) ) { - if( auto* r = ::postgresql_stmt_result_metadata(h) ) { - ::postgresql_free_result(r); - return invalid_statement(::postgresql_stmt_error(h)); - } - } - return sqldb::OK; - } - return invalid_statement(::postgresql_stmt_error(h)); - } -#endif - return invalid_statement(); + /// Generic parameter binding + sqldb_status column_bind::bind_store(column_data& d, const char* ptr, size_t len) { + if ( ptr ) { + d.bind_pointer((const unsigned char*)ptr, len); + stmt->_param_ptr[col] = d.item.string; + stmt->_param_len[col] = d.length; + stmt->_param_fmt[col] = 0; + return sqldb::OK; } + return this->bind_store_null(d); + } - /// Fetch next result row. Stop if result is NON-zero - virtual int fetch_one() { -#if 0 - if ( this->_imp ) { - size_t nfld = this->_fbind.size(); - if ( nfld > 0 ) { - ::memset(&this->_fbind.front(), 0, nfld*sizeof(column_bind)); - for( size_t i=0; i < nfld; i++ ) { - this->bind_fetch(i); - if ( !postgresql_ok(::postgresql_stmt_bind_result(this->_imp, &this->_fbind.front())) ) - return invalid_statement(::postgresql_stmt_error(this->_imp)); - } - } - int ret = ::postgresql_stmt_fetch(this->_imp); - if( ret == POSTGRESQL_NO_DATA ) { - return sqldb::NO_DATA; - } - else if( ret == POSTGRESQL_DATA_TRUNCATED ) { - for( size_t i=0; i < nfld; i++ ) { - auto& d = this->fields[i]; - if( d.error && !d.null && d.length >= sizeof(d.buf) ) { - auto& b = this->_fbind[i]; - d.vbuf.resize(d.length); - d.item.blob = &d.vbuf.front(); - b.buffer = d.item.blob; - b.buffer_length = d.length; - if( !postgresql_ok(::postgresql_stmt_fetch_column(this->_imp,&b,i,0)) ) { - return invalid_statement(::postgresql_stmt_error(this->_imp)); - } - } - } + /// Type specific parameter bind + template <typename T> sqldb_status column_bind::bind_store(column_data& d, T& to, const T* from) { + if ( from ) { + to = *from; + d.length = sizeof(T); + stmt->_param_ptr[col] = (char*)&to; + stmt->_param_len[col] = d.length; + stmt->_param_fmt[col] = 1; + return sqldb::OK; + } + return this->bind_store_null(d); + } + + /// Bind output field + template <typename T> sqldb_status column_bind::bind_fetch(column_data& d, T* to, size_t len) { + T* ptr = (T*)::PQgetvalue(stmt->_result, stmt->_current_row, col); + d.null = false; + d.error = false; + d.length = len; + *to = *ptr; + return sqldb::OK; + } + + /// Bind output field + sqldb_status column_bind::bind_fetch_blob(column_data& d) { + unsigned char* ptr = (unsigned char*)::PQgetvalue(stmt->_result, stmt->_current_row, col); + size_t length = ::PQgetlength(stmt->_result, stmt->_current_row, col); + d.null = false; + d.error = false; + d.length = length; + d.vbuf.clear(); + if ( length <= sizeof(d.buf) ) { + ::memcpy(d.buf, ptr, length); + d.item.string = d.buf; + return sqldb::OK; + } + d.vbuf.insert(d.vbuf.begin(), ptr, ptr+length); + d.item.blob = &d.vbuf.front(); + return sqldb::OK; + } + + /// Prepare a new statement and allocate resources. + sqldb_status statement_backend::prepare(postgresql::dbase_t* db, const std::string& sql_str, uint64_t id) { + this->_handle = db; + this->_current_row = -1; + this->_fmt.imbue(std::locale::classic()); + this->_fmt.str(std::string()); + this->_fmt.clear(); + this->_fmt << "sqldb_pq_stmt_" << id; + this->_id = this->_fmt.str(); + this->_fmt.str(std::string()); + this->_fmt.clear(); + this->_sql.clear(); + this->_sql.reserve(sql_str.size()); + this->_num_param = 0; + bool inside = false; + for(unsigned i=0;i<sql_str.size();i++) { + char c = sql_str[i]; + switch(c) { + case '\'': + inside = !inside; + break; + case '?': + if( !inside ) { + this->_sql += '$'; + ++this->_num_param; + this->_fmt << this->_num_param; + this->_sql += this->_fmt.str(); + this->_fmt.str(std::string()); + this->_fmt.clear(); } - - return sqldb::OK; + break; + default: + this->_sql += c; + break; } -#endif - return invalid_statement(); } + this->_param_ptr.resize( this->_num_param, nullptr); + this->_param_typ.resize(this->_num_param, null_param); + this->_param_len.resize(this->_num_param, 0); + this->reset(); - /// Bind output field identified by column number - int bind_fetch(size_t col) { -#if 0 - auto* p = this; - auto [d, b, f] = p->field(col); + this->_result = ::PQprepare(this->_handle,this->_id.c_str(),this->_sql.c_str(),0,0); + if( !this->_result ) { + _err = "sqldb(postgres): Failed to create prepared statement object!"; + return sqldb::ERROR; + } + if ( ::PQresultStatus(this->_result) != PGRES_COMMAND_OK ) { + _err = "sqldb(postgres): statement preparation failed: "+error_msg(this->_result); + return sqldb::ERROR; + } + + auto* res = ::PQdescribePrepared(this->_handle,this->_id.c_str()); + if( !this->_result ) { + _err = "sqldb(postgres): Failed to inspect prepared statement object!"; + return sqldb::ERROR; + } + if ( ::PQresultStatus(this->_result) != PGRES_COMMAND_OK ) { + _err = "sqldb(postgres): statement inspection failed: "+error_msg(res); + return sqldb::ERROR; + } + this->params.resize(this->_num_param); + for(size_t col=0; col < this->_num_param; ++col) { + Oid native_type = ::PQparamtype(res, col); + this->_param_typ[col] = native_type; + this->params[col].type = gen_field_type(native_type); + } + this->_err.clear(); + clear_result(res); + clear_result(this->_result); + return sqldb::OK; + } - ::memset(b, 0, sizeof(column_bind)); - b->buffer_type = f->type; - d->type = gen_field_type(f->type); - switch(d->type) { - case TYPE_TINY: return b->bind_fetch(d, &d->item.int8, sizeof(d->item.int8)); - case TYPE_SHORT: return b->bind_fetch(d, &d->item.int16, sizeof(d->item.int16)); - case TYPE_LONG: return b->bind_fetch(d, &d->item.int32, sizeof(d->item.int32)); - case TYPE_LONGLONG: return b->bind_fetch(d, &d->item.int64, sizeof(d->item.int64)); - case TYPE_FLOAT: return b->bind_fetch(d, &d->item.real32, sizeof(d->item.real32)); - case TYPE_DOUBLE: return b->bind_fetch(d, &d->item.real64, sizeof(d->item.real64)); - case TYPE_STRING: return b->bind_fetch(d, d->buf, sizeof(d->buf)); - case TYPE_BLOB: return b->bind_fetch(d, d->buf, sizeof(d->buf)); - case TYPE_TIME: return b->bind_fetch(d, &d->item.time.stamp, sizeof(d->item.int64)); - default: return invalid_statement("Invalid Postgresql Field data type"); - } -#endif - return sqldb::OK; + /// Bind a single column to the data passed + sqldb_status statement_backend::bind(size_t col, field_type typ, const void* data, size_t len) { + pointers_t ptr(data); + column_bind b(this, col); + auto& d = this->params[col]; + switch(typ) { + case TYPE_NULL: return b.bind_store_null(d); + case TYPE_TINY: return b.bind_store(d, d.item.uint8, ptr.puint8); + case TYPE_SHORT: return b.bind_store(d, d.item.uint16, ptr.puint16); + case TYPE_LONG: return b.bind_store(d, d.item.uint32, ptr.puint32); + case TYPE_LONGLONG: return b.bind_store(d, d.item.uint64, ptr.puint64); + case TYPE_FLOAT: return b.bind_store(d, d.item.real32, ptr.preal32); + case TYPE_DOUBLE: return b.bind_store(d, d.item.real64, ptr.preal64); + case TYPE_STRING: return b.bind_store(d, ptr.pstring, len); + case TYPE_BLOB: return b.bind_store(d, ptr.puint8, len); + /// For now store time stamps as long integers (time_t) + case TYPE_TIME: return b.bind_store(d, d.item.time.stamp, ptr.pint64); + default: return invalid_statement("Invalid Postgresql parameter data type"); } + return sqldb::OK; + } - /// Bind a single column to the data passed - virtual int bind(size_t col, field_type typ, const void* data, size_t len) override { -#if 0 - pointers_t ptr(data); - auto [d, b] = this->param(col); + /// Bind output field identified by column number + sqldb_status statement_backend::bind_fetch(size_t col) { + auto typ = ::PQftype(this->_result, col); + auto& d = this->fields[col]; + column_bind b(this, col); + d.clear(); + d.type = gen_field_type(typ); + switch(d.type) { + case TYPE_TINY: return b.bind_fetch(d, &d.item.int8, sizeof(d.item.int8)); + case TYPE_SHORT: return b.bind_fetch(d, &d.item.int16, sizeof(d.item.int16)); + case TYPE_LONG: return b.bind_fetch(d, &d.item.int32, sizeof(d.item.int32)); + case TYPE_LONGLONG: return b.bind_fetch(d, &d.item.int64, sizeof(d.item.int64)); + case TYPE_FLOAT: return b.bind_fetch(d, &d.item.real32, sizeof(d.item.real32)); + case TYPE_DOUBLE: return b.bind_fetch(d, &d.item.real64, sizeof(d.item.real64)); + case TYPE_TIME: return b.bind_fetch(d, &d.item.time.stamp, sizeof(d.item.int64)); + case TYPE_STRING: return b.bind_fetch_blob(d); + case TYPE_BLOB: return b.bind_fetch_blob(d); + default: return invalid_statement("Invalid Postgresql Field data type"); + } + return sqldb::OK; + } + + /// Execute prepared statement + sqldb_status statement_backend::execute() { + this->_current_row = -1; + clear_result(this->_result); + this->_result = ::PQexecPrepared(this->handle(), + this->_id.c_str(), + this->_num_param, + &this->_param_ptr[0], + &this->_param_len[0], + &this->_param_fmt[0], + 1); + if( !this->_result ) { + _err = "sqldb(postgres): Failed to create prepared statement object!"; + return sqldb::ERROR; + } + if ( ::PQresultStatus(this->_result) != PGRES_COMMAND_OK ) { + _err = "sqldb(postgres): statement preparation failed: "+error_msg(this->_result); + return sqldb::ERROR; + } + return sqldb::OK; + } - ::memset(&b, 0, sizeof(column_bind)); - d->type = typ; - b->buffer_type = postgresql_field_type(typ); - switch(typ) { - case TYPE_NULL: return b->bind_store_null(d); - case TYPE_TINY: return b->bind_store(d, d->item.uint8, ptr.puint8); - case TYPE_SHORT: return b->bind_store(d, d->item.uint16, ptr.puint16); - case TYPE_LONG: return b->bind_store(d, d->item.uint32, ptr.puint32); - case TYPE_LONGLONG: return b->bind_store(d, d->item.uint64, ptr.puint64); - case TYPE_FLOAT: return b->bind_store(d, d->item.real32, ptr.preal32); - case TYPE_DOUBLE: return b->bind_store(d, d->item.real64, ptr.preal64); - case TYPE_STRING: return b->bind_store(d, ptr.puint8, len); - case TYPE_BLOB: return b->bind_store(d, ptr.puint8, len); - /// For now store time stamps as long integers (time_t) - case TYPE_TIME: return b->bind_store(d, d->item.time.stamp, ptr.puint64); - default: return invalid_statement("Invalid Postgresql parameter data type"); + /// Action routine for various functions + int statement_backend::action(action_type type) { + switch(type) { + case EXEC: /// Execute prepared statement + return this->execute(); + case FETCH: /// Fetch next result row. Stop if result is NON-zero + if ( this->_result ) { + ++_current_row; + return ::PQgetisnull(this->_result, this->_current_row, 0) ? sqldb::OK : sqldb::ERROR; + } + return sqldb::ERROR; + case RESET: /// Reset all data to re-execute the statement with new parameters + return this->reset(); + case FINALIZE: /// Finalize and run-down the statement. Release all resources. + if ( this->_handle != nullptr && !this->_id.empty() ) { + this->_result = ::PQexec(this->_handle, ("DEALLOCATE "+this->_id).c_str()); + clear_result(this->_result); } -#endif return sqldb::OK; - } - - /// Action routine for various functions - int action(action_type type) override { - switch(type) { - case EXEC: /// Execute prepared statement - return sqldb::OK; - case FETCH: /// Fetch next result row. Stop if result is NON-zero - if ( this->_result ) { - return ::PQgetisnull(this->_result, this->_current, 0) ? sqldb::OK : sqldb::ERROR; - } - return sqldb::ERROR; - case RESET: /// Reset all data to re-execute the statement with new parameters - return this->reset(); - case FINALIZE: /// Finalize and run-down the statement. Release all resources. - if ( this->_handle != nullptr && !this->_id.empty() ) { - this->_fmt.str(""); - this->_fmt.clear(); - this->_fmt << "DEALLOCATE " << this->_id; - this->_result = ::PQexec(this->_handle, this->_fmt.str().c_str()); - clear_result(this->_result); - } - return sqldb::OK; - case ERRNO: /// Access error number of last failed statement - if ( this->_result != nullptr ) { - return ::PQresultStatus(this->_result); - } - return ::PQstatus(this->handle()); - default: - throw std::runtime_error("Invalid action request received!"); + case ERRNO: /// Access error number of last failed statement + if ( this->_result != nullptr ) { + return ::PQresultStatus(this->_result); } + return ::PQstatus(this->handle()); + default: + throw std::runtime_error("Invalid action request received!"); } - }; + } /// Technology abstraction layer for a Postgresql database /** @@ -535,7 +545,7 @@ namespace sqldb { } } /// Execute prepared statement - virtual int execute(std::string& err, const std::string& sql) override { + virtual sqldb_status execute(std::string& err, const std::string& sql) override { err = ""; if( this->_handle ) { clear_result(this->_result); @@ -555,7 +565,7 @@ namespace sqldb { return sqldb::INVALID_HANDLE; } /// Prepare a new statement and allocate resources. - virtual int prepare(statement& bs, const std::string& sql_str) override { + virtual sqldb_status prepare(statement& bs, const std::string& sql_str) override { auto* stm_imp = new statement_backend(); auto* h = this->handle(); auto sql = _escape(h, sql_str.c_str(), sql_str.length()); @@ -563,7 +573,7 @@ namespace sqldb { return stm_imp->prepare(h, sql, ++this->_prepared_id); } /// Open database - int open(const std::string& connect_string) { + sqldb_status open(const std::string& connect_string) { auto args = connection_args().parse(connect_string); this->_handle = ::PQsetdbLogin(args["host"].c_str(), args["port"].c_str(), @@ -587,16 +597,17 @@ namespace sqldb { /// SQLDB namespace declaration namespace sqldb { + /// Open the database connection using all information in passed string template <> - std::pair<std::shared_ptr<database::backend>, int> + std::pair<std::shared_ptr<database::backend>, sqldb_status> database::backend::open<postgresql>(const std::string& connect_string) { auto db = std::make_shared<postgresql_imp::dbase_backend>(); - int ret = db->open(connect_string); + sqldb_status ret = db->open(connect_string); if ( ret == sqldb::OK ) return make_pair(db, sqldb::OK); return make_pair(std::shared_ptr<database::backend>(), ret); } -} // End namespace sqldb +} // End namespace sqldb #endif diff --git a/Online/sqldb/src/sqlite.cpp b/Online/sqldb/src/sqlite.cpp index 1f249f811..8c54476aa 100644 --- a/Online/sqldb/src/sqlite.cpp +++ b/Online/sqldb/src/sqlite.cpp @@ -45,12 +45,12 @@ namespace sqldb { throw std::runtime_error("Invalid data type encountered!"); } /// - int do_exec(sqlite3* h, const char* sql, char** err=0) { + sqldb_status do_exec(sqlite3* h, const char* sql, char** err=0) { if ( h ) { char* e = nullptr; if ( !sqlite_ok(::sqlite3_exec(h, sql, nullptr, nullptr, &e)) ) { if ( err != nullptr ) *err = e; - return ::sqlite3_extended_errcode(h); + return (sqldb_status)::sqlite3_extended_errcode(h); } return sqldb::OK; } @@ -69,13 +69,13 @@ namespace sqldb { sqlite::statement_t* stmt { nullptr }; size_t column { 0 }; public: - int bind_null(column_data& d) { + sqldb_status bind_null(column_data& d) { d.length = 0; d.item.pointer = nullptr; return sqldb::OK; } /// Generic parameter binding - int bind_str(column_data& d, const char* ptr, size_t len) { + sqldb_status bind_str(column_data& d, const char* ptr, size_t len) { if ( ptr ) { pointers_t p(ptr); d.bind_pointer(p.puint8, len); @@ -85,7 +85,7 @@ namespace sqldb { return bind_null(d); } /// Generic parameter binding - int bind_blob(column_data& d, const unsigned char* ptr, size_t len) { + sqldb_status bind_blob(column_data& d, const unsigned char* ptr, size_t len) { if ( ptr ) { d.bind_pointer(ptr, len); ::sqlite3_bind_blob(this->stmt, this->column+1, d.item.pointer, len, SQLITE_STATIC); @@ -94,21 +94,21 @@ namespace sqldb { return bind_null(d); } /// Type specific parameter bind - template <typename T> int bind_int(column_data& d, T& to, const T* from) { + template <typename T> sqldb_status bind_int(column_data& d, T& to, const T* from) { to = *from; d.length = sizeof(int); ::sqlite3_bind_int(this->stmt, this->column+1, int(to)); return sqldb::OK; } /// Type specific parameter bind - int bind_int64(column_data& d, int64_t& to, const int64_t* from) { + sqldb_status bind_int64(column_data& d, int64_t& to, const int64_t* from) { to = *from; d.length = sizeof(int64_t); ::sqlite3_bind_int64(this->stmt, this->column+1, to); return sqldb::OK; } /// Type specific parameter bind - template <typename T> int bind_flt(column_data& d, T& to, const T* from) { + template <typename T> sqldb_status bind_flt(column_data& d, T& to, const T* from) { to = *from; d.length = sizeof(double); ::sqlite3_bind_double(this->stmt, this->column+1, double(to)); @@ -147,7 +147,7 @@ namespace sqldb { return ::sqlite3_expanded_sql(this->_handle); } /// - int bind(size_t col, field_type typ, const void* data, size_t len) override { + sqldb_status bind(size_t col, field_type typ, const void* data, size_t len) override { pointers_t ptr(data); column_data& d = this->params[col]; column_bind b {this->_handle, col}; @@ -167,7 +167,7 @@ namespace sqldb { return sqldb::OK; } /// Fetch next result row. Stop if result is NON-zero - int fetch_one() { + sqldb_status fetch_one() { if ( auto* h = this->_handle ) { int ret = ::sqlite3_step(h); if( ret == SQLITE_ROW ) { @@ -204,7 +204,7 @@ namespace sqldb { return invalid_statement(); } /// Prepare a new statement and allocate resources. - int prepare(sqlite::dbase_t* db, const std::string& sql_str) { + sqldb_status prepare(sqlite::dbase_t* db, const std::string& sql_str) { sqlite::statement_t* stmt = nullptr; if ( auto* h = this->_handle ) { ::sqlite3_reset(h); @@ -288,20 +288,20 @@ namespace sqldb { return ::sqlite3_changes(this->_handle); } /// Execute prepared statement - virtual int execute(std::string& error, const std::string& sql_str) override { - char* err = nullptr; - int ret = do_exec(this->handle(), sql_str.c_str(), &err); + virtual sqldb_status execute(std::string& error, const std::string& sql_str) override { + char* err = nullptr; + sqldb_status ret = do_exec(this->handle(), sql_str.c_str(), &err); if ( ret != sqldb::OK && err != nullptr ) error = err; return ret; } /// Prepare a new statement and allocate resources. - int prepare(statement& bs, const std::string& sql_str) override { + sqldb_status prepare(statement& bs, const std::string& sql_str) override { auto* stm_imp = new sqlite_statement_backend(); bs.intern.reset(stm_imp); return stm_imp->prepare(this->_handle, sql_str); } /// Open database - int open(const std::string& connect_string) { + sqldb_status open(const std::string& connect_string) { this->_handle = nullptr; if ( !sqlite_ok(::sqlite3_open(connect_string.c_str(), &this->_handle)) ) return sqldb::ERROR; @@ -336,11 +336,11 @@ namespace sqldb { namespace sqldb { /// Open the database connection using all information in passed string - template <> std::pair<std::shared_ptr<database::backend>, int> + template <> std::pair<std::shared_ptr<database::backend>, sqldb_status> database::backend::open<sqlite>(const std::string& connect_string) { typedef sqlite_imp::sqlite_dbase_backend _imp_t; auto db = std::make_shared<_imp_t>(); - int ret = db->open(connect_string); + sqldb_status ret = db->open(connect_string); if ( ret == sqldb::OK ) return make_pair(std::move(db), sqldb::OK); return make_pair(std::shared_ptr<database::backend>(), ret); -- GitLab From d197c4ccc482c3c6283752f0d08e519617745916 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Mon, 10 May 2021 09:58:20 +0200 Subject: [PATCH 23/26] Fix nightly build warnings --- Online/EventBuilding/CMakeLists.txt | 11 +++- Online/PyDIM/src/dimcppmodule.cpp | 4 +- Online/ROLogger/kafka/LogServer.h | 8 ++- Online/Storage/src/server/sqlite_test.cpp | 34 +++++----- Online/sqldb/sqldb/sqldb-imp.h | 42 ++++++++----- Online/sqldb/sqldb/sqldb.h | 10 +-- Online/sqldb/src/oracle.cpp | 77 ++++++++++++++++++----- 7 files changed, 127 insertions(+), 59 deletions(-) diff --git a/Online/EventBuilding/CMakeLists.txt b/Online/EventBuilding/CMakeLists.txt index a2e9be0ce..102bb598b 100644 --- a/Online/EventBuilding/CMakeLists.txt +++ b/Online/EventBuilding/CMakeLists.txt @@ -74,8 +74,10 @@ gaudi_add_library(EventBuildingLib ${INFINIBAND_LIBRARIES}) target_include_directories(EventBuildingLib BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) target_include_directories(EventBuildingLib BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include/infiniband_net) -target_compile_options(EventBuildingLib PRIVATE -std=c++2a -fconcepts) - +target_compile_options(EventBuildingLib PRIVATE -std=c++2a) +if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + target_compile_options(EventBuildingLib PRIVATE -fconcepts) +endif() # --------------------------------------------------------------------------------------- # Commonly used DataflowExample components @@ -88,7 +90,10 @@ gaudi_add_library(EventBuilding ${MPI_CXX_LIBRARIES} ${PCIE_40_LIBRARIES}) target_include_directories(EventBuilding BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) target_include_directories(EventBuilding BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include/infiniband_net) -target_compile_options(EventBuilding PRIVATE -std=c++2a -fconcepts) +target_compile_options(EventBuilding PRIVATE -std=c++2a) +if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + target_compile_options(EventBuildingLib PRIVATE -fconcepts) +endif() # # ---> Steer the property parser: diff --git a/Online/PyDIM/src/dimcppmodule.cpp b/Online/PyDIM/src/dimcppmodule.cpp index 05e798f8f..fe1748ef5 100755 --- a/Online/PyDIM/src/dimcppmodule.cpp +++ b/Online/PyDIM/src/dimcppmodule.cpp @@ -441,7 +441,7 @@ static PyMethodDef dimcpp_methods[] = { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wmissing-field-initializers" #pragma GCC diagnostic ignored "-Wpedantic" -#ifdef __clang__ +#if defined(__clang__) && __clang_major__ >= 10 #pragma clang diagnostic ignored "-Wc99-designator" #endif @@ -810,7 +810,7 @@ static PyMethodDef DimRpcInfo_methods[] = { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wmissing-field-initializers" #pragma GCC diagnostic ignored "-Wpedantic" -#ifdef __clang__ +#if defined(__clang__) && __clang_major__ >= 10 #pragma clang diagnostic ignored "-Wc99-designator" #endif diff --git a/Online/ROLogger/kafka/LogServer.h b/Online/ROLogger/kafka/LogServer.h index 26248b832..3a3d1852c 100644 --- a/Online/ROLogger/kafka/LogServer.h +++ b/Online/ROLogger/kafka/LogServer.h @@ -118,7 +118,7 @@ namespace kafka { private: /// Reference to implementation - std::unique_ptr<Implementation> consumer = 0; + std::unique_ptr<Implementation> consumer {}; public: /// Access to the monitoring structure @@ -131,8 +131,12 @@ namespace kafka { std::unique_ptr<Implementation> create(int argc, char** argv); /// Consumer creator static std::unique_ptr<Implementation> create(int consumer_type, std::vector<char*>& args); - + protected: /// Default constructor + Consumer() = default; + + public: + /// Initializing constructor Consumer(std::unique_ptr<Implementation>&& impl); /// Default destructor virtual ~Consumer(); diff --git a/Online/Storage/src/server/sqlite_test.cpp b/Online/Storage/src/server/sqlite_test.cpp index 876e04f57..9eaace9b3 100644 --- a/Online/Storage/src/server/sqlite_test.cpp +++ b/Online/Storage/src/server/sqlite_test.cpp @@ -72,7 +72,8 @@ namespace { sqldb::database d; d.open<sqlite>(this->database); if ( d.intern->db == nullptr ) { - ::lib_rtl_output(LIB_RTL_ERROR,"%s: Can't open database [%s]", name.c_str(), d.errmsg()); + ::lib_rtl_output(LIB_RTL_ERROR,"%s: Can't open database [%s]", + name.c_str(), d.errmsg().c_str()); } return d; } @@ -126,7 +127,8 @@ extern "C" int sqlite_populate_database(int argc, char** argv) { insert.bind(5, host); int ret = insert.execute(); if ( ret != sqldb::OK ) { - ::lib_rtl_output(LIB_RTL_ERROR,"%s: failed INSERT: %s",db.name(),insert.errmsg()); + ::lib_rtl_output(LIB_RTL_ERROR,"%s: failed INSERT: %s", + db.name(), insert.errmsg().c_str()); return EINVAL; } insert.reset(); @@ -160,7 +162,8 @@ extern "C" int sqlite_populate_database_direct(int argc, char** argv) { "VALUES ( '%s' , %d , %ld , '%s' , '%s' )", obj, 0, length, date.c_str(), host.c_str()); if ( ret != sqldb::OK ) { - ::lib_rtl_output(LIB_RTL_ERROR,"%s: failed INSERT: %s",db.name(),db.errmsg()); + ::lib_rtl_output(LIB_RTL_ERROR,"%s: failed INSERT: %s", + db.name(), db.errmsg().c_str()); return EINVAL; } } @@ -168,7 +171,7 @@ extern "C" int sqlite_populate_database_direct(int argc, char** argv) { db.close(); } catch(const exception& e) { - ::lib_rtl_output(LIB_RTL_ERROR,"%s: exception: %s",db.name(),e.what()); + ::lib_rtl_output(LIB_RTL_ERROR,"%s: exception: %s", db.name(), e.what()); } return 0; } @@ -218,7 +221,8 @@ extern "C" int sqlite_update_database_direct(int argc, char** argv) { std::stringstream s; std::vector<file_t> files; sqldb::statement query = - sqldb::statement::create(db, "SELECT Name, State, Size, Date, Host" + sqldb::statement::create(db, + "SELECT Name, State, Size, Date, Host" " FROM Files" " WHERE Size>%d",0); if ( query.is_valid() ) { @@ -239,17 +243,17 @@ extern "C" int sqlite_update_database_direct(int argc, char** argv) { ret = sqldb::ERROR; if ( f.state == 0 ) { ret = db.execute_sql(error, - "UPDATE Files" - " SET State=1" - " WHERE Name='%s'", - f.name.c_str()); + "UPDATE Files" + " SET State=1" + " WHERE Name='%s'", + f.name.c_str()); } else if ( f.state == 1 ) { ret = db.execute_sql(error, - "UPDATE Files" - " SET State=0" - " WHERE Name='%s'", - f.name.c_str()); + "UPDATE Files" + " SET State=0" + " WHERE Name='%s'", + f.name.c_str()); } if ( ret != sqldb::OK ) { ::lib_rtl_output(LIB_RTL_INFO, "SQL Failure: %s", error.c_str()); @@ -259,7 +263,7 @@ extern "C" int sqlite_update_database_direct(int argc, char** argv) { db.close(); } catch(const exception& e) { - ::lib_rtl_output(LIB_RTL_ERROR,"%s: exception: %s",db.name(),e.what()); + ::lib_rtl_output(LIB_RTL_ERROR,"%s: exception: %s", db.name(), e.what()); } return 0; } @@ -279,7 +283,7 @@ extern "C" int fdb_cli_dumpdb(int argc, char** argv) { sqldb::database database = setup.db(); if ( !database.is_valid() ) { ::lib_rtl_output(LIB_RTL_ERROR,"%s: Can't open database [%s]", - setup.database.c_str(), database.errmsg()); + setup.database.c_str(), database.errmsg().c_str()); return EINVAL; } diff --git a/Online/sqldb/sqldb/sqldb-imp.h b/Online/sqldb/sqldb/sqldb-imp.h index 113d9e450..06ff71dc5 100644 --- a/Online/sqldb/sqldb/sqldb-imp.h +++ b/Online/sqldb/sqldb/sqldb-imp.h @@ -12,8 +12,8 @@ // // Author : Markus Frank //========================================================================== -#ifndef SQLDB_SQLDB_IMP_INTERFACE_H -#define SQLDB_SQLDB_IMP_INTERFACE_H +#ifndef SQLDB_SQLDB_IMP_H +#define SQLDB_SQLDB_IMP_H /// C/C++ include files #include <sstream> @@ -129,7 +129,7 @@ namespace sqldb { } /// Generic conversion to integer number - SQLDB_INLINE struct time to_time(const column_data& column) { + SQLDB_INLINE class time to_time(const column_data& column) { switch( column.type ) { case TYPE_BLOB: case TYPE_STRING: @@ -229,7 +229,7 @@ namespace sqldb { va_start(args, fmt); ::vsnprintf(str, sizeof(str), fmt, args); va_end(args); - return this->execute_sql(error, str); + return intern->execute(error, str); } /// Prepare a new statement and allocate resources. @@ -243,6 +243,7 @@ namespace sqldb { return invalid_statement("fetch_one: Invalid recordset statement handle"); } + /// Access type specific field data template <> SQLDB_INLINE blob_t record::get<blob_t>(size_t col) const { auto& column = this->field(col); switch( column.type ) { @@ -255,6 +256,7 @@ namespace sqldb { } } + /// Access type specific field data template <> SQLDB_INLINE vblob_t record::get<vblob_t>(size_t col) const { const auto& column = this->field(col); switch( column.type ) { @@ -267,6 +269,7 @@ namespace sqldb { } } + /// Access type specific field data template <> SQLDB_INLINE std::string record::get<std::string>(size_t col) const { const auto& column = this->field(col); switch( column.type ) { @@ -299,51 +302,63 @@ namespace sqldb { } } + /// Access type specific field data template <> SQLDB_INLINE signed char record::get<signed char>(size_t col) const { return to_integer<signed char>(this->field(col)); } + /// Access type specific field data template <> SQLDB_INLINE char record::get<char>(size_t col) const { return to_integer<char>(this->field(col)); } + /// Access type specific field data template <> SQLDB_INLINE unsigned char record::get<unsigned char>(size_t col) const { return to_integer<unsigned char>(this->field(col)); } + /// Access type specific field data template <> SQLDB_INLINE int16_t record::get<int16_t>(size_t col) const { return to_integer<int16_t>(this->field(col)); } + /// Access type specific field data template <> SQLDB_INLINE uint16_t record::get<uint16_t>(size_t col) const { return to_integer<uint16_t>(this->field(col)); } + /// Access type specific field data template <> SQLDB_INLINE int32_t record::get<int32_t>(size_t col) const { return to_integer<int32_t>(this->field(col)); } + /// Access type specific field data template <> SQLDB_INLINE uint32_t record::get<uint32_t>(size_t col) const { return to_integer<uint32_t>(this->field(col)); } + /// Access type specific field data template <> SQLDB_INLINE int64_t record::get<int64_t>(size_t col) const { return to_integer<int64_t>(this->field(col)); } + /// Access type specific field data template <> SQLDB_INLINE uint64_t record::get<uint64_t>(size_t col) const { return to_integer<uint64_t>(this->field(col)); } + /// Access type specific field data template <> SQLDB_INLINE float record::get<float>(size_t col) const { return to_real<float>(this->field(col)); } + /// Access type specific field data template <> SQLDB_INLINE double record::get<double>(size_t col) const { return to_real<double>(this->field(col)); } - template <> SQLDB_INLINE time record::get<struct time>(size_t col) const { + /// Access type specific field data + template <> SQLDB_INLINE time record::get<class time>(size_t col) const { return to_time(this->field(col)); } @@ -432,28 +447,21 @@ namespace sqldb { return stmt->_bind(col, TYPE_LONGLONG, &data, sizeof(data)); } - /// Type specific bind operator for + /// Type specific bind operator for float type template <> SQLDB_INLINE sqldb_status record_bind::bind<float>(size_t col, float data) const { return stmt->_bind(col, TYPE_FLOAT, &data, sizeof(data)); } + /// Type specific bind operator for double type template <> SQLDB_INLINE sqldb_status record_bind::bind<double>(size_t col, double data) const { return stmt->_bind(col, TYPE_DOUBLE, &data, sizeof(data)); } - template <> SQLDB_INLINE sqldb_status record_bind::bind<struct time>(size_t col, struct time data) const { + /// Type specific bind operator for time type + template <> SQLDB_INLINE sqldb_status record_bind::bind<class time>(size_t col, class time data) const { return stmt->_bind(col, TYPE_TIME, &data.stamp, sizeof(data.stamp)); } - /// Copy assignment - SQLDB_INLINE statement& statement::operator = (statement&& copy) { - if ( © != this ) { - this->stmt = this; - this->intern = move(copy.intern); - } - return *this; - } - /// Copy assignment SQLDB_INLINE statement& statement::operator = (const statement& copy) { if ( © != this ) { @@ -527,4 +535,4 @@ namespace sqldb { } } /// End namespace sqldb -#endif // SQLDB_SQLDB_IMP_INTERFACE_H +#endif // SQLDB_SQLDB_IMP_H diff --git a/Online/sqldb/sqldb/sqldb.h b/Online/sqldb/sqldb/sqldb.h index bebac399b..4184fd2e1 100644 --- a/Online/sqldb/sqldb/sqldb.h +++ b/Online/sqldb/sqldb/sqldb.h @@ -427,7 +427,7 @@ namespace sqldb { column_data& field(size_t column) const; /// Check if the field is NULL bool is_null(size_t column) const; - /// Access field data + /// Access type specific field data template <typename T> T get (size_t column) const; }; @@ -447,14 +447,14 @@ namespace sqldb { /// Initializing constructor record_set(const statement* s) : record(s) {} /// Move constructor - record_set(record_set&& copy) = default; + record_set(record_set&& copy) = delete; /// Copy constructor record_set(const record_set& copy) = default; /// Destructor ~record_set() = default; /// Move assignment - record_set& operator = (record_set&& copy) = default; + record_set& operator = (record_set&& copy) = delete; /// Copy assignment record_set& operator = (const record_set& copy) = default; @@ -533,7 +533,7 @@ namespace sqldb { */ class backend { public: - backend* stmt {nullptr}; + backend* stmt { nullptr }; columns_t params; columns_t fields; @@ -576,7 +576,7 @@ namespace sqldb { ~statement() = default; /// Move assignment - statement& operator = (statement&& copy); + statement& operator = (statement&& copy) = delete; /// Copy assignment statement& operator = (const statement& copy); diff --git a/Online/sqldb/src/oracle.cpp b/Online/sqldb/src/oracle.cpp index 5bb61a68d..33e1806e2 100644 --- a/Online/sqldb/src/oracle.cpp +++ b/Online/sqldb/src/oracle.cpp @@ -144,8 +144,20 @@ namespace sqldb { ub2 _sql_type; public: + /// Default constructor + column_bind() = default; + /// Move constructor + column_bind(column_bind&& copy) = delete; + /// Copy constructor + column_bind(const column_bind& copy) = delete; /// Default destructor ~column_bind(); + + /// Move assignment + column_bind& operator = (column_bind&& copy) = delete; + /// Copy assignment + column_bind& operator = (const column_bind& copy) = delete; + /// Generic parameter binding sqldb_status bind_str(column_data& d, const char* ptr, size_t len); /// Generic parameter binding @@ -169,11 +181,11 @@ namespace sqldb { * \version 1.0 * \date 02.05.2017 */ - struct statement_backend : public statement::backend { + class statement_backend : public statement::backend { public: - typedef std::tuple<column_data*, column_bind*, ub2> field_t; typedef std::vector<column_bind> Binders; + std::vector<ub2> _types; Binders _pbind, _fbind; EnvH _env { nullptr }; @@ -183,10 +195,16 @@ namespace sqldb { std::string _sql_str { }; int _error_code { 0 }; + public: /// Initializing constructor statement_backend(EnvH env, DbcH dbc, ErrorH err) : _env(env), _dbc(dbc), _error(err) { } - + /// Default constructor + statement_backend() = delete; + /// Move constructor + statement_backend(statement_backend&& copy) = delete; + /// Copy constructor + statement_backend(const statement_backend& copy) = delete; /// Default destructor virtual ~statement_backend() { if ( _stmt ) { @@ -198,6 +216,11 @@ namespace sqldb { _error = nullptr; } } + /// Move assignment + statement_backend& operator = (statement_backend&& copy) = delete; + /// Copy assignment + statement_backend& operator = (const statement_backend& copy) = delete; + /// Access to the internal statement handle. Throws exception if the handle is invalid StmH handle() const { if ( this->_stmt ) return this->_stmt; @@ -218,14 +241,7 @@ namespace sqldb { return _sql_str.c_str(); } /// Access the number of rows affected by the last statement - long rows_affected() override { - ub4 row_count = 0; - result_t ret = ::OCIAttrGet(_stmt, OCI_HTYPE_STMT, &row_count, 0, OCI_ATTR_ROW_COUNT, _error); - if ( oracle_ok(ret) ) - return row_count; - _error_code = ret; - return -1; - } + long rows_affected() override; /// Execute prepared statement sqldb_status execute(); /// Fetch next result row. Stop if result is NON-zero @@ -253,10 +269,18 @@ namespace sqldb { ErrorH _error; int _error_code; - /// Initializing constructor + /// Default constructor dbase_backend(); + /// Move constructor + dbase_backend(dbase_backend&& copy) = delete; + /// Copy constructor + dbase_backend(const dbase_backend& copy) = delete; /// Default destructor virtual ~dbase_backend(); + /// Move assignment + dbase_backend& operator = (dbase_backend&& copy) = delete; + /// Copy assignment + dbase_backend& operator = (const dbase_backend& copy) = delete; /// Access to the internal database handle. Throws exception if the handle is invalid DbcH handle() const { @@ -288,7 +312,7 @@ namespace sqldb { _lob = nullptr; } } - + /// Generic parameter binding sqldb_status column_bind::bind_str(column_data& d, const char* ptr, size_t len) { pointers_t p(ptr); @@ -298,6 +322,7 @@ namespace sqldb { 0, 0, 0, 0, 0, OCI_DEFAULT); return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; } + /// Generic parameter binding sqldb_status column_bind::bind_blob(column_data& d, const unsigned char* ptr, size_t len) { pointers_t p(ptr); @@ -307,6 +332,7 @@ namespace sqldb { 0, 0, 0, 0, 0, OCI_DEFAULT); return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; } + /// Type specific parameter bind sqldb_status column_bind::bind_int(column_data& d, int32_t from) { d.item.int32 = from; @@ -316,6 +342,7 @@ namespace sqldb { 0, 0, 0, 0, 0, OCI_DEFAULT); return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; } + /// Type specific parameter bind sqldb_status column_bind::bind_int64(column_data& d, int64_t from) { d.item.int64 = from; @@ -325,6 +352,7 @@ namespace sqldb { 0, 0, 0, 0, 0, OCI_DEFAULT); return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; } + /// Type specific parameter bind sqldb_status column_bind::bind_flt(column_data& d, double from) { d.item.real64 = from; @@ -334,6 +362,7 @@ namespace sqldb { 0, 0, 0, 0, 0, OCI_DEFAULT); return oracle_ok(ret) ? sqldb::OK : sqldb::ERROR; } + /// Read lob data sqldb_status column_bind::fetch_data(column_data& d) { result_t rc = OCI_SUCCESS; @@ -355,6 +384,7 @@ namespace sqldb { } return oracle_ok(rc) ? sqldb::OK : sqldb::ERROR; } + /// Bind output field sqldb_status column_bind::bind_fetch(column_data& d, void* ptr, size_t len) { result_t rc = OCI_SUCCESS; @@ -379,6 +409,16 @@ namespace sqldb { return oracle_ok(rc) ? sqldb::OK : sqldb::ERROR; } + /// Access the number of rows affected by the last statement + long statement_backend::rows_affected() { + ub4 row_count = 0; + result_t ret = ::OCIAttrGet(_stmt, OCI_HTYPE_STMT, &row_count, 0, OCI_ATTR_ROW_COUNT, _error); + if ( oracle_ok(ret) ) + return row_count; + _error_code = ret; + return -1; + } + /// Fetch next result row. Stop if result is NON-zero sqldb_status statement_backend::fetch_one() { if ( _stmt ) { @@ -390,7 +430,7 @@ namespace sqldb { } } } - result_t ret = OCIStmtFetch(_stmt, _error, 1, OCI_FETCH_NEXT, OCI_DEFAULT); + result_t ret = ::OCIStmtFetch(_stmt, _error, 1, OCI_FETCH_NEXT, OCI_DEFAULT); if( ret == OCI_SUCCESS ) { for( size_t i=0; i < nfld; i++ ) { auto& d = fields[i]; @@ -503,6 +543,7 @@ namespace sqldb { stmt = this; return sqldb::OK; } + /// int statement_backend::action(action_type type) { switch(type) { @@ -534,6 +575,7 @@ namespace sqldb { } } + /// Default constructor dbase_backend::dbase_backend() { result_t rc; rc = OCIEnvCreate(&_env, OCI_OBJECT, 0, 0, 0, 0, 0, 0); @@ -546,6 +588,7 @@ namespace sqldb { } _error_code = rc; } + /// Default destructor dbase_backend::~dbase_backend() { if ( _dbc ) { @@ -554,6 +597,7 @@ namespace sqldb { ::OCIHandleFree(_error, OCI_HTYPE_ERROR); _error = nullptr; } + /// Execute prepared statement sqldb_status dbase_backend::execute(std::string& error, const std::string& sql_str) { char* err = nullptr; @@ -563,6 +607,7 @@ namespace sqldb { } return ret; } + /// Prepare a new statement and allocate resources. sqldb_status dbase_backend::prepare(statement& bs, const std::string& sql_str) { ErrorH err {nullptr}; @@ -578,6 +623,7 @@ namespace sqldb { _error_code = rc; return sqldb::ERROR; } + /// Open database sqldb_status dbase_backend::open(const std::string& connect_string) { auto args = connection_args().parse(connect_string); @@ -602,6 +648,7 @@ namespace sqldb { _error_code = OCI_SUCCESS; return sqldb::OK; } + /// Perform multiple actions int dbase_backend::action(action_type type) { result_t rc = OCI_SUCCESS; @@ -646,4 +693,4 @@ namespace sqldb { return make_pair(db, sqldb::OK); return make_pair(std::shared_ptr<database::backend>(), ret); } -} // End namespace sqldb +} /// End namespace sqldb -- GitLab From 1832f1f8d1bdb832041573dece11ca3775ce1844 Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Sat, 15 May 2021 23:01:04 +0200 Subject: [PATCH 24/26] Fix task checker --- Online/PcSrv/src/TaskCheck.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Online/PcSrv/src/TaskCheck.cpp b/Online/PcSrv/src/TaskCheck.cpp index 6c024a61a..4d38f6830 100644 --- a/Online/PcSrv/src/TaskCheck.cpp +++ b/Online/PcSrv/src/TaskCheck.cpp @@ -19,8 +19,9 @@ #include "CPP/IocSensor.h" #include "RPC/HttpRpcClient.h" #include "RPC/XMLRPC.h" -#include "RTL/rtl.h" +#include "RTL/Logger.h" #include "RTL/strdef.h" +#include "RTL/rtl.h" #include "dim/dic.h" #include "dim/dis.h" @@ -438,9 +439,12 @@ extern "C" int run_task_check(int argc, char** argv) { cli.getopt("tm_dns", 3, tm_dns); cli.getopt("print", 3, print_level); + ::dis_set_dns_node(tm_dns.c_str()); + ::dic_set_dns_node(tm_dns.c_str()); + RTL::Logger::install_rtl_printer(print_level); + RTL::Logger::getGlobalDevice()->compileFormat("%LEVEL %-19SOURCE'"); + TaskCheck chk(dns, node); - ::lib_rtl_set_log_level(print_level); - ::lib_rtl_install_printer(print_msg2, 0); lib_rtl_output(LIB_RTL_ALWAYS,"-%s- Domain RPC service of type: TaskCheck started. DNS:%s TM:%s %s%s", node.c_str(), dns.c_str(), tm_dns.c_str(), server.empty() ? "" : "Closest Server:", server.empty() ? "" : server.c_str()); @@ -466,8 +470,6 @@ extern "C" int run_task_check(int argc, char** argv) { chk.initialize(); server = '/' + RTL::str_upper(RTL::nodeNameShort()) + '/' + RTL::processName(); - ::dis_set_dns_node(tm_dns.c_str()); - ::dic_set_dns_node(tm_dns.c_str()); ::dis_start_serving(server.c_str()); IocSensor::instance().run(); return 1; -- GitLab From f25cc37dbe45fb42d0c959f5f3db1534ff935a3a Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Thu, 20 May 2021 14:56:50 +0200 Subject: [PATCH 25/26] Update event builder options --- Online/EventBuilding/options/MFP_generator_params.opts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Online/EventBuilding/options/MFP_generator_params.opts b/Online/EventBuilding/options/MFP_generator_params.opts index b75ef17ce..2ef58e6b5 100644 --- a/Online/EventBuilding/options/MFP_generator_params.opts +++ b/Online/EventBuilding/options/MFP_generator_params.opts @@ -5,4 +5,4 @@ MFP_generator.n_banks = 30000; MFP_generator.numa_layout = { 4, 2 }; MFP_generator.random_sizes = FALSE; MFP_generator.reconnect = FALSE; -MFP_generator.shmem_prefix = "RU"; \ No newline at end of file +MFP_generator.shmem_prefix = "RU"; -- GitLab From 0011ccbf261ff3660949765c0834e864475c41aa Mon Sep 17 00:00:00 2001 From: Markus Frank <Markus.Frank@cern.ch> Date: Thu, 20 May 2021 15:08:46 +0200 Subject: [PATCH 26/26] Online v7r7 --- .../FarmConfig/job/AddersFromArchitecture.py | 452 ++++++++++++++++++ 1 file changed, 452 insertions(+) create mode 100644 Online/FarmConfig/job/AddersFromArchitecture.py diff --git a/Online/FarmConfig/job/AddersFromArchitecture.py b/Online/FarmConfig/job/AddersFromArchitecture.py new file mode 100644 index 000000000..2a0b1b9c3 --- /dev/null +++ b/Online/FarmConfig/job/AddersFromArchitecture.py @@ -0,0 +1,452 @@ +from xml.dom import minidom +import sys +import os +import OnlineEnvBase +import re +import socket +CounterDebugSvcs = ["DskWriter"] +HistDebugSvcs =["AligWrk"] +vnode=False + +def VictimNode(): + victimnodes_re = [] + vnodes="" + vnodes=os.getenv("victimnodes","") + if vnodes!="": + victimnodes = vnodes.split(",") + for i in victimnodes: + victimnodes_re.append(re.compile(i,re.IGNORECASE)) + hname=socket.gethostname().split('.')[0] + for i in victimnodes_re: + if i.match(hname) != None: + return True + +def TaskListfromArch(arch, tasklist): + xmldoc = minidom.parse(arch) + itemlist = xmldoc.getElementsByTagName('task') +# tasklist = [] + for s in itemlist: + nam = s.attributes['name'] +# if not nam.find('MEPR')>=0: + tasklist.append(s.attributes['name'].value) +def AddersfromTasks +def OptionsfromTasks(tasklist,level,ofile,pname,dohostdns): + f = open(ofile,'w') + f.write("// Adder Level "+level+"=====================\n") + if vnode: + f.write("// This is a victim node\n") + if level=="1": + if pname=="LHCb": + f.write("""#include "$INFO_OPTIONS" +ApplicationMgr.ExtSvc += {"MonitorSvc"}; + +ApplicationMgr.EventLoop = "LHCb::OnlineRunable/EmptyEventLoop"; +ApplicationMgr.Runable = "LHCb::OnlineRunable/Runable"; +ApplicationMgr.HistogramPersistency = "NONE"; +ApplicationMgr.EvtSel = "NONE"; + +Runable.Wait = 3; // 1 of running as daemon (Class1 task) + +MessageSvc.fifoPath = "$LOGFIFO"; +MessageSvc.OutputLevel = @OnlineEnv.OutputLevel; +MonitorSvc.OutputLevel = @OnlineEnv.OutputLevel; +HistogramPersistencySvc.Warnings = false; +MonitorSvc.CounterUpdateInterval = 5; +""") + else: + f.write("""#include "$INFO_OPTIONS" +ApplicationMgr.ExtSvc += {"MonitorSvc"}; + +ApplicationMgr.EventLoop = "LHCb::OnlineRunable/EmptyEventLoop"; +ApplicationMgr.Runable = "LHCb::OnlineRunable/Runable"; +ApplicationMgr.HistogramPersistency = "NONE"; +ApplicationMgr.EvtSel = "NONE"; + +Runable.Wait = 3; // 1 of running as daemon (Class1 task) + +MessageSvc.fifoPath = "$LOGFIFO"; +MessageSvc.OutputLevel = @OnlineEnv.OutputLevel; +MonitorSvc.OutputLevel = @OnlineEnv.OutputLevel; +HistogramPersistencySvc.Warnings = false; +MonitorSvc.CounterUpdateInterval = 5; +""") + else: + f.write("""#include "$INFO_OPTIONS" +ApplicationMgr.ExtSvc += {"MonitorSvc"}; +ApplicationMgr.EventLoop = "LHCb::OnlineRunable/EmptyEventLoop"; +ApplicationMgr.Runable = "LHCb::OnlineRunable/Runable"; +ApplicationMgr.HistogramPersistency = "NONE"; +ApplicationMgr.EvtSel = "NONE"; + +Runable.Wait = 3; // 1 of running as daemon (Class1 task) + +MessageSvc.fifoPath = "$LOGFIFO"; +MessageSvc.OutputLevel = @OnlineEnv.OutputLevel; +MonitorSvc.OutputLevel = @OnlineEnv.OutputLevel; +HistogramPersistencySvc.Warnings = false; +MonitorSvc.CounterUpdateInterval = 5; +""") + + if level=="1": + InDns = os.getenv("InDns","<dns>") + OutDns = os.getenv("OutDns","<dns>") + InDns = "\""+InDns+"\"" + OutDns = "\""+OutDns+"\"" + if level =="2": + InDns = os.getenv("InDns","<node>") + OutDns = os.getenv("OutDns","hlt01") + InDns = "\""+InDns+"\"" + OutDns = "\""+OutDns+"\"" + elif level == "3": + InDns = os.getenv("InDns","hlt01") + OutDns = os.getenv("OutDns","mona08") + InDns = "\""+InDns+"\"" + OutDns = "\""+OutDns+"\"" + elif level == "3.1": + InDns = os.getenv("InDns","mona08") + OutDns = os.getenv("OutDns","mona08") + InDns = "\""+InDns+"\"" + OutDns = "\""+OutDns+"\"" + elif level == "4": + InDns = os.getenv("InDns","mona08") + OutDns = os.getenv("OutDns","mona08") + InDns = "\""+InDns+"\"" + OutDns = "\""+OutDns+"\"" + if level == "1": + histsvc = [] + cntsvc = [] + for s in tasklist: + if 'NodeAdder' in s: + continue + if 'AligAdder' in s: + continue + hsvc = s#+"HistAdder" + f.write("ApplicationMgr.ExtSvc += {\"AdderSvc/"+hsvc+"HistAdder\"};\n") + csvc = s#+"CountAdder" + f.write("ApplicationMgr.ExtSvc += {\"AdderSvc/"+csvc+"CountAdder\"};\n") + histsvc.append(hsvc) + cntsvc.append(csvc) + f.write("\n") + + for s in histsvc: + svc = s+"HistAdder" + f.write(svc+".PartitionName = @OnlineEnv.PartitionName;\n") + f.write(svc+".MyName = \"<part>_<node>_"+s+"\";\n") + f.write(svc+".TaskPattern = \"<part>_<node>_"+s+"_(.*)\";\n") + f.write(svc+".ServicePattern = \"MON_<part>_<node>_"+s+"_(.*)/Histos/\";\n") + f.write(svc+".AdderClass = \"hists\";\n") + f.write(svc+".ReceiveTimeout = 5;\n") + f.write("\n") + if dohostdns: + f.write(svc+".InDNS = "+InDns+";\n") + f.write(svc+".OutDNS = "+OutDns+";\n") + if pname == "LHCbA": # overwrite certain options for the Alignment... + f.write(svc+".SaveInterval = -1;\n"); + f.write(svc+".SaveonUpdate = false;\n"); + f.write(svc+".SaveSetTaskName= \""+svc+"\";\n"); + f.write(svc+".ReceiveTimeout = 1;\n") + if pname =="LHCbA": + f.write(svc+".EoRTmoFactor = 2;\n") +# f.write(svc+".DebugOn = true;\n") + f.write(svc+".GotoPause = false;\n") +# f.write(svc+".DebugOn = true;\n") + if vnode: + if s in HistDebugSvcs: + f.write(svc+".DebugOn = true;\n") + for s in cntsvc: + svc = s+"CountAdder" + f.write(svc+".PartitionName = @OnlineEnv.PartitionName;\n") + f.write(svc+".MyName = \"<part>_<node>_"+s+"\";\n") + f.write(svc+".TaskPattern = \"<part>_<node>_"+s+"_(.*)\";\n") + f.write(svc+".ServicePattern = \"MON_<part>_<node>_"+s+"_(.*)/Counter/\";\n") + f.write(svc+".AdderClass = \"Counter\";\n") + f.write(svc+".ReceiveTimeout = 4;\n") + if vnode: + if s in CounterDebugSvcs: + f.write(svc+".DebugOn = true;\n") + if pname == "LHCbA": + f.write(svc+".GotoPause = false;\n") + f.write(svc+".ReceiveTimeout = 0;\n") + if dohostdns: + f.write(svc+".InDNS = "+InDns+";\n") + f.write(svc+".OutDNS = "+OutDns+";\n") + f.write("\n") +# s = "BusyMon" +# svc = s+"CountAdder" +# f.write(svc+".PartitionName = @OnlineEnv.PartitionName;\n") +# f.write(svc+".MyName = \"<part>_<node>_"+s+"\";\n") +# f.write(svc+".TaskPattern = \"GEN_<node>_"+s+"\";\n") +# f.write(svc+".ServicePattern = \"MON_GEN_<node>_"+s+"/Counter/\";\n") +# f.write(svc+".AdderClass = \"Counter\";\n") +# f.write(svc+".ReceiveTimeout = 2;\n") + if vnode: + if s in CounterDebugSvcs: + f.write(svc+".DebugOn = true;\n") + if pname == "LHCbA": + f.write(svc+".GotoPause = false;\n") + f.write(svc+".ReceiveTimeout = 0;\n") + if dohostdns: + f.write(svc+".InDNS = "+InDns+";\n") + f.write(svc+".OutDNS = "+OutDns+";\n") + f.write("\n") + f.write("ApplicationMgr.ExtSvc += {\"AdderSvc/"+svc+"\"};\n") + elif level == "2": +# f.write("ApplicationMgr.ExtSvc += {\"AdderSvc/"+"BusyAdder"+"CountAdder\"};\n") +# f.write(""" +# BusyAdderCountAdder.MyName = "<part>_<node>_BusySvc"; +# BusyAdderCountAdder.PartitionName = @OnlineEnv.PartitionName; +# BusyAdderCountAdder.TaskPattern = "<part>_<node>[0-9][0-9]_NodeAdder_0"; +# BusyAdderCountAdder.ServicePattern = "MON_<part>_<node>[0-9][0-9]_BusyMon/Counter/"; +# BusyAdderCountAdder.AdderClass = "Counter"; +# BusyAdderCountAdder.ReceiveTimeout = 6; +# """) +# f.write("BusyAdderCountAdder.InDns = "+InDns+";\n") +# f.write("BusyAdderCountAdder.OutDns = "+OutDns+";\n") + histsvc = [] + cntsvc = [] + for s in tasklist: + if 'SubFarmAdder' in s: + continue + hsvc = s#+"HistAdder" + f.write("ApplicationMgr.ExtSvc += {\"AdderSvc/"+hsvc+"HistAdder\"};\n") + csvc = s#+"CountAdder" + f.write("ApplicationMgr.ExtSvc += {\"AdderSvc/"+csvc+"CountAdder\"};\n") + histsvc.append(hsvc) + cntsvc.append(csvc) + f.write("\n") + + for s in histsvc: + svc = s+"HistAdder" + f.write(svc+".PartitionName = @OnlineEnv.PartitionName;\n") + f.write(svc+".MyName = \"<part>_<node>_"+s+"\";\n") + f.write(svc+".TaskPattern = \"<part>_<node>[0-9][0-9]_NodeAdder_0\";\n") + f.write(svc+".ServicePattern = \"MON_<part>_<node>[0-9][0-9]_"+s+"/Histos/\";\n") + f.write(svc+".AdderClass = \"hists\";\n") + f.write(svc+".ReceiveTimeout = 8;\n") + f.write(svc+".InDNS = "+InDns+";\n") + f.write(svc+".OutDNS = "+OutDns+";\n") + if vnode: + if s in HistDebugSvcs: + f.write(svc+".DebugOn = true;\n") + if pname == "LHCbA": # overwrite certain options for the Alignment... + f.write(svc+".SaveInterval = -1;\n"); + f.write(svc+".SaveonUpdate = false;\n"); + f.write(svc+".SaveSetTaskName= \""+svc+"\";\n"); + f.write(svc+".ReceiveTimeout = 2;\n") + f.write(svc+".EoRTmoFactor = 8;\n") +# f.write(svc+".DebugOn = true;\n") + f.write("\n") + + + for s in cntsvc: + svc = s+"CountAdder" + f.write(svc+".PartitionName = @OnlineEnv.PartitionName;\n") + f.write(svc+".MyName = \"<part>_<node>_"+s+"\";\n") + f.write(svc+".TaskPattern = \"<part>_<node>[0-9][0-9]_NodeAdder_0\";\n") + f.write(svc+".ServicePattern = \"MON_<part>_<node>[0-9][0-9]_"+s+"/Counter/\";\n") + f.write(svc+".ReceiveTimeout = 8;\n") + f.write(svc+".AdderClass = \"Counter\";\n") + f.write(svc+".InDNS = "+InDns+";\n") + f.write(svc+".OutDNS = "+OutDns+";\n") + if vnode: + if s in CounterDebugSvcs: + f.write(svc+".DebugOn = true;\n") + if pname == "LHCbA": + f.write(svc+".GotoPause = true;\n") + f.write(svc+".ReceiveTimeout = 0;\n") + f.write("\n") + elif level == "3": +# f.write("ApplicationMgr.ExtSvc += {\"AdderSvc/"+"BusyAdder"+"CountAdder\"};\n") +# f.write(""" +# BusyAdderCountAdder.MyName = "<part>_Busy_00"; +# BusyAdderCountAdder.PartitionName = @OnlineEnv.PartitionName; +# BusyAdderCountAdder.TaskPattern = "<part>_hlt[a-z][0-9][0-9]_SubFarmAdder_0"; +# BusyAdderCountAdder.ServicePattern = "MON_<part>_hlt[a-z][0-9][0-9]_BusySvc/Counter/"; +# BusyAdderCountAdder.AdderClass = "Counter"; +# BusyAdderCountAdder.ReceiveTimeout = 8; +# """) +# f.write("BusyAdderCountAdder.InDns = "+InDns+";\n") +# f.write("BusyAdderCountAdder.OutDns = "+OutDns+";\n") + histsvc = [] + cntsvc = [] +# histsvc.append("Adder") +# cntsvc.append("BusySvc") +# tasklist.remove("MEPrx") + for s in tasklist: + hsvc = s#+"HistAdder" + f.write("ApplicationMgr.ExtSvc += {\"AdderSvc/"+hsvc+"HistAdder\"};\n") + csvc = s#+"CountAdder" + f.write("ApplicationMgr.ExtSvc += {\"AdderSvc/"+csvc+"CountAdder\"};\n") + histsvc.append(hsvc) + cntsvc.append(csvc) + f.write("\n") + + for s in histsvc: + svc = s+"HistAdder" + f.write(svc+".PartitionName = @OnlineEnv.PartitionName;\n") + f.write(svc+".MyName = \"<part>_hlt01_"+s+"_00\";\n") + f.write(svc+".TaskPattern = \"<part>_HLT[a-z][0-9][0-9]_SubFarmAdder_(.*)\";\n") + f.write(svc+".ServicePattern = \"MON_<part>_hlt[a-z][0-9][0-9]_"+s+"/Histos/\";\n") + f.write(svc+".AdderClass = \"hists\";\n") + f.write(svc+".ReceiveTimeout = 12;\n") + f.write(svc+".InDNS = "+InDns+";\n") + f.write(svc+".OutDNS = "+OutDns+";\n") +# f.write(svc+".SaveRootDir = \"/hist/Savesets\";\n"); +# f.write(svc+".IsSaver = true;\n"); +# f.write(svc+".SaveInterval = -1;\n"); +# f.write(svc+".SaveonUpdate = false;\n"); +# f.write(svc+".SaveSetTaskName= \""+svc+"\";\n"); + if vnode: + if s in HistDebugSvcs: + f.write(svc+".DebugOn = true;\n") + if pname == "LHCbA" and s=="AligWrk": # overwrite certain options for the Alignment... +# f.write(svc+".DebugOn = true;\n") + act = "" + act=OnlineEnvBase.Activity + acts = act.split('|') + if len(acts) > 1: + altype = acts[1] + else: + altype = acts[0] +# f.write(svc+".SaveRootDir = \"/hist/Savesets\";\n"); +# f.write(svc+".IsSaver = true;\n"); +# f.write(svc+".SaveInterval = -1;\n"); +# f.write(svc+".SaveonUpdate = false;\n"); +# f.write(svc+".SaveSetTaskName= \""+svc+"\";\n"); +# f.write(svc+".SaveInterval = -1;\n"); +# f.write(svc+".SaveonUpdate = false;\n"); +# f.write(svc+".SaveSetTaskName= \"AligWrk_"+altype+"\";\n"); +# f.write(svc+".ReceiveTimeout = 3;\n") +# # f.write(svc+".PauseOnEOR = true;\n") +# f.write(svc+".EoRTmoFactor = 15;\n") + f.write("\n") + + for s in cntsvc: + svc = s+"CountAdder" + f.write(svc+".PartitionName = @OnlineEnv.PartitionName;\n") + f.write(svc+".MyName = \"<part>_"+s+"_00\";\n") + f.write(svc+".TaskPattern = \"<part>_HLT[a-z][0-9][0-9]_SubFarmAdder_(.*)\";\n") + f.write(svc+".ServicePattern = \"MON_<part>_hlt[a-z][0-9][0-9]_"+s+"/Counter/\";\n") + f.write(svc+".AdderClass = \"Counter\";\n") + f.write(svc+".ReceiveTimeout = 12;\n") + f.write(svc+".InDNS = "+InDns+";\n") + f.write(svc+".OutDNS = "+OutDns+";\n") + if vnode: + if s in CounterDebugSvcs: + f.write(svc+".DebugOn = true;\n") + if pname == "LHCbA": + f.write(svc+".ReceiveTimeout = 0;\n") +# f.write(svc+".GotoPause = true;\n") + f.write("\n") + elif level == "3.1": #Top level saver + histsvc = [] + histsvc.append("Adder") +# tasklist.remove("MEPrx") + for s in tasklist: + hsvc = s#+"HistAdder" + f.write("ApplicationMgr.ExtSvc += {\"AdderSvc/"+hsvc+"HistAdder\"};\n") + histsvc.append(hsvc) + f.write("\n") + + for s in histsvc: + svc = s+"HistAdder" + f.write(svc+".PartitionName = @OnlineEnv.PartitionName;\n") + f.write(svc+".MyName = \"<part>_Saverhlt01_"+s+"_00\";\n") + f.write(svc+".TaskPattern = \"MON_<part>_HLT02_PartAdder_0\";\n") + f.write(svc+".ServicePattern = \"MON_<part>_hlt01_"+s+"_00/Histos/\";\n") + f.write(svc+".AdderClass = \"hists\";\n") + f.write(svc+".ReceiveTimeout = 12;\n") + f.write(svc+".InDNS = "+InDns+";\n") + f.write(svc+".OutDNS = "+OutDns+";\n") + f.write(svc+".SaveRootDir = \"/hist/Savesets\";\n"); + f.write(svc+".IsSaver = true;\n"); + f.write(svc+".SaveInterval = 900;\n"); + f.write(svc+".SaveonUpdate = false;\n"); + f.write(svc+".SaveSetTaskName= \""+s+"\";\n"); + if vnode: + if s in HistDebugSvcs: + f.write(svc+".DebugOn = true;\n") + if pname == "LHCbA" and s=="AligWrk": # overwrite certain options for the Alignment... +# f.write(svc+".DebugOn = true;\n") + act = "" + act=OnlineEnvBase.Activity + acts = act.split('|') + if len(acts) > 1: + altype = acts[1] + else: + altype = acts[0] + f.write(svc+".SaveInterval = -1;\n"); + f.write(svc+".SaveonUpdate = true;\n"); + f.write(svc+".SaveSetTaskName= \"AligWrk_"+altype+"\";\n"); + f.write(svc+".ReceiveTimeout = 3;\n") +# f.write(svc+".PauseOnEOR = true;\n") + f.write(svc+".EoRTmoFactor = 15;\n") +# f.write(svc+".DebugOn = true;\n") + f.write("\n") + elif level == "4": + histsvc = [] + cntsvc = [] +# histsvc.append("Adder") +# cntsvc.append("Busy") + for s in tasklist: + hsvc = s#+"HistAdder" + f.write("ApplicationMgr.ExtSvc += {\"AdderSvc/"+hsvc+"HistAdder\"};\n") + csvc = s#+"CountAdder" + f.write("ApplicationMgr.ExtSvc += {\"AdderSvc/"+csvc+"CountAdder\"};\n") + histsvc.append(hsvc) + cntsvc.append(csvc) + f.write("\n") + + for s in histsvc: + svc = s+"HistAdder" + f.write(svc+".PartitionName = @OnlineEnv.PartitionName;\n") + f.write(svc+".MyName = \"<part>_X_"+s+"\";\n") + f.write(svc+".TaskPattern = \"<part>_HLT01_PartAdder_(.*)\";\n") + f.write(svc+".ServicePattern = \"MON_<part>_hlt01_"+s+"/Histos/\";\n") + f.write(svc+".AdderClass = \"hists\";\n") + f.write(svc+".ReceiveTimeout = 12;\n") + f.write(svc+".InDNS = "+InDns+";\n") + f.write(svc+".OutDNS = "+OutDns+";\n") + if vnode: + if s in HistDebugSvcs: + f.write(svc+".DebugOn = true;\n") + f.write("\n") + + for s in cntsvc: + svc = s+"CountAdder" + f.write(svc+".PartitionName = @OnlineEnv.PartitionName;\n") + f.write(svc+".MyName = \"<part>_X_"+s+"\";\n") + f.write(svc+".TaskPattern = \"<part>_HLT01_PartAdder_(.*)\";\n") + f.write(svc+".ServicePattern = \"MON_<part>_hlt01_"+s+"/Counter/\";\n") + f.write(svc+".AdderClass = \"Counter\";\n") + f.write(svc+".ReceiveTimeout = 12;\n") + f.write(svc+".InDNS = "+InDns+";\n") + f.write(svc+".OutDNS = "+OutDns+";\n") + if vnode: + if s in CounterDebugSvcs: + f.write(svc+".DebugOn = true;\n") + if pname == "LHCbA": + f.write(svc+".ReceiveTimeout = 1000000;\n") + f.write("\n") + # Add UI options + ##f.write("//\n// RPC UI options for adders:\n//\n") + ##f.write("ApplicationMgr.ExtSvc += { \"Online::OnlineUISvc/UI\" };\n") + ##f.write("UI.When = \"initialize\";\n") + +tasklist = [] +vnode= VictimNode() +arch = OnlineEnvBase.HltArchitecture +part = OnlineEnvBase.PartitionName +arch = os.getenv("ARCH",arch) +hostdns = False +if arch == "Calib": + hostdns = True +arch = "/group/online/dataflow/architectures/lbDataflowArch_"+arch+".xml" +level = sys.argv[1] +TaskListfromArch(arch, tasklist) +ofile = "" +if len(sys.argv) >= 3: + ofile = sys.argv[2] +if ofile == "": + ofile = "/tmp/AdderOptions.opts" +OptionsfromTasks(tasklist,level,ofile,part,hostdns) -- GitLab