From bbb5ee4286622df9e4249c911fd063241f787f13 Mon Sep 17 00:00:00 2001 From: Constantin Heidegger Date: Mon, 20 Jun 2022 16:53:11 +0200 Subject: [PATCH] changing default RT and T0 conditions folders for MDT calibration --- .../CoralUtilities/blobaccess.h | 16 ++-- Database/CoralUtilities/src/blobaccess.cxx | 76 +++++++++++++++++-- .../python/CalibrationTargetFolderConfig.py | 12 ++- .../share/CalibDbDefaults.py | 4 +- .../MuonCnvExample/python/MuonCalibConfig.py | 5 +- .../MuonCondAlg/MuonCondAlg/MdtCalibDbAlg.h | 6 +- .../MuonCondAlg/src/MdtCalibDbAlg.cxx | 12 ++- .../MuonCondAlg/src/MdtCalibFormatAlgTest.cxx | 17 ++++- .../python/MuonCalibrationConfig.py | 2 +- 9 files changed, 125 insertions(+), 25 deletions(-) diff --git a/Database/CoralUtilities/CoralUtilities/blobaccess.h b/Database/CoralUtilities/CoralUtilities/blobaccess.h index 46d1d55d489..1e6c1b13050 100644 --- a/Database/CoralUtilities/CoralUtilities/blobaccess.h +++ b/Database/CoralUtilities/CoralUtilities/blobaccess.h @@ -18,15 +18,21 @@ namespace coral { namespace CoralUtilities { - bool compressBlob (const char* , coral::Blob&); - bool writeBlobFromString (const std::string_view, coral::Blob&); - bool writeBlobFromJson (const nlohmann::json& , coral::Blob&); - bool writeBlobFromTTree ( TTree* , coral::Blob&); + bool compressString (const std::string, std::string&); + bool uncompressString (const std::string, std::string&); + + bool insertBlob (const char* , coral::Blob&, bool=true); + bool extractBlob (const coral::Blob&, char*& , bool=true); + + bool compressBlob (const char* , coral::Blob& ); + bool writeBlobFromString (const std::string_view, coral::Blob&, bool=true); + bool writeBlobFromJson (const nlohmann::json& , coral::Blob&, bool=true); + bool writeBlobFromTTree ( TTree* , coral::Blob& ); bool uncompressBlob (const coral::Blob&, unsigned char*&, unsigned long&); bool readBlobAsString (const coral::Blob&, std::string& ); bool readBlobAsJson (const coral::Blob&, nlohmann::json& ); - bool readBlobAsTTree (const coral::Blob&, std::unique_ptr& ); + bool readBlobAsTTree (const coral::Blob&, std::unique_ptr& ); } diff --git a/Database/CoralUtilities/src/blobaccess.cxx b/Database/CoralUtilities/src/blobaccess.cxx index 39214c7ad59..7d8d2a17d5a 100644 --- a/Database/CoralUtilities/src/blobaccess.cxx +++ b/Database/CoralUtilities/src/blobaccess.cxx @@ -1,5 +1,5 @@ /* - Copyright (C) 2002-2021 CERN for the benefit of the ATLAS collaboration + Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration */ #include "CoralUtilities/blobaccess.h" @@ -14,7 +14,71 @@ namespace CoralUtilities { -// Compression Functions +// String Compression and Uncompression Functions +// ------------------------------------------------- + +/* TODO! placeholder for now +bool compressString(const std::string in, std::string& out){ + return false; +} +*/ + +bool uncompressString(const std::string in, std::string& out){ + uLongf uncompLen = 50000; + std::unique_ptr uncompBuf(new unsigned char[uncompLen+1]); + uLongf actualLen; + std::vector bdata = CxxUtils::base64_decode(in); + while(true) { + actualLen = uncompLen; + int res(uncompress(uncompBuf.get(), &actualLen, reinterpret_cast(bdata.data()), static_cast(bdata.size()))); + if(res == Z_OK ) break; + if(res == Z_BUF_ERROR) { // double buffer if it was not big enough + uncompLen *= 2; + std::cout << "ATTENTION: Increasing buffer to " << uncompLen << std::endl; + uncompBuf.reset(new unsigned char[uncompLen+1]); + continue; + } + // something else is wrong + uncompBuf.reset(); + return false; + } + uncompBuf.get()[actualLen]=0; // append 0 to terminate string + out = reinterpret_cast(uncompBuf.release()); + return true; +} + + + +// Blob Insertion and Extraction Functions +// ------------------------------------------------- + +bool insertBlob(const char* in, coral::Blob& out, bool decode) { + if(!decode){ + std::vector bdata = CxxUtils::base64_decode(in); + char* enc = reinterpret_cast(bdata.data()); + out.resize(strlen(enc)); + char* addressToElement = reinterpret_cast(out.startingAddress()); + for(size_t i = 0; i < strlen(in); ++i, ++addressToElement) + *addressToElement = enc[i]; + } + else { + out.resize(strlen(in)); + char* addressToElement = static_cast(out.startingAddress()); + for(size_t i = 0; i < strlen(in); ++i, ++addressToElement) + *addressToElement = in[i]; + } + return true; +} + +/* TODO! placeholder for now +bool extractBlob(const coral::Blob& in, char*& out, bool encode){ + return false; +} +*/ + + + +// Blob Compression Functions // ------------------------------------------------- bool compressBlob(const char* in, coral::Blob& out) { @@ -28,11 +92,13 @@ bool compressBlob(const char* in, coral::Blob& out) { return true; } -bool writeBlobFromString(const std::string_view in, coral::Blob& out) { +bool writeBlobFromString(const std::string_view in, coral::Blob& out, bool compress) { + if(!compress) return insertBlob(in.data(), out); return compressBlob(in.data(), out); } -bool writeBlobFromJson(const nlohmann::json& in, coral::Blob& out) { +bool writeBlobFromJson(const nlohmann::json& in, coral::Blob& out, bool compress) { + if(!compress) return insertBlob(in.dump().data(), out); return writeBlobFromString(in.dump(), out); } @@ -55,7 +121,7 @@ bool writeBlobFromTTree(TTree* tree, coral::Blob& out) { -// Reading and Uncompressing Functions +// Blob Uncompression Functions // ------------------------------------------------- bool uncompressBlob(const coral::Blob &blob, unsigned char*& out, unsigned long& len){ diff --git a/MuonSpectrometer/MuonCalib/MuonCalibDbOperations/python/CalibrationTargetFolderConfig.py b/MuonSpectrometer/MuonCalib/MuonCalibDbOperations/python/CalibrationTargetFolderConfig.py index a6638e6fbb6..1f719499597 100644 --- a/MuonSpectrometer/MuonCalib/MuonCalibDbOperations/python/CalibrationTargetFolderConfig.py +++ b/MuonSpectrometer/MuonCalib/MuonCalibDbOperations/python/CalibrationTargetFolderConfig.py @@ -12,8 +12,8 @@ MuonCalib__gCalibrationTargetConfigs = [] class MuonCalib__CalibrationTargetConfig: def __init__(self, FolderType="T0"): - if FolderType not in ["T0", "RT", "T0BLOB", "RTBLOB"]: - print ("FATAL Folder type must be 'T0[BLOB]' or 'RT[BLOB]'") + if FolderType not in ["T0", "RT", "T0BLOB", "RTBLOB", "T0JSONS", "RTJSONS"]: + print ("FATAL Folder type must be 'T0[BLOB,JSONS]' or 'RT[BLOB,JSONS]'") sys.exit(1) #set defaults @@ -29,6 +29,10 @@ class MuonCalib__CalibrationTargetConfig: self.__is_t0=True self.Compress=True self.Folder = "/MDT/T0BLOB" + elif FolderType == "T0JSONS": + self.__is_t0=True + self.Compress=True + self.Folder = "/MDT/T0JSONS" if FolderType=="RT": self.__is_t0=False self.Compress=False @@ -37,6 +41,10 @@ class MuonCalib__CalibrationTargetConfig: self.__is_t0=False self.Compress=True self.Folder = "/MDT/RTBLOB" + elif FolderType == "RTJSONS": + self.__is_t0=False + self.Compress=True + self.Folder = "/MDT/RTJSONS" MuonCalib__gCalibrationTargetConfigs.append(self) diff --git a/MuonSpectrometer/MuonCalib/MuonCalibDbOperations/share/CalibDbDefaults.py b/MuonSpectrometer/MuonCalib/MuonCalibDbOperations/share/CalibDbDefaults.py index 21f443f1cd6..1db486d6564 100644 --- a/MuonSpectrometer/MuonCalib/MuonCalibDbOperations/share/CalibDbDefaults.py +++ b/MuonSpectrometer/MuonCalib/MuonCalibDbOperations/share/CalibDbDefaults.py @@ -7,9 +7,9 @@ cool_dbname = 'CONDBR2' cool_dbname_mc = 'OFLP200' cool_database_string = 'COOLOFL_MDT/' + cool_dbname -cool_folders['t0'] = '/MDT/T0BLOB' +cool_folders['t0'] = '/MDT/T0JSONS' cool_tags['t0'] = 'CURRENT' -cool_folders['rt'] = '/MDT/RTBLOB' +cool_folders['rt'] = '/MDT/RTJSONS' cool_tags['rt'] ='CURRENT' diff --git a/MuonSpectrometer/MuonCnv/MuonCnvExample/python/MuonCalibConfig.py b/MuonSpectrometer/MuonCnv/MuonCnvExample/python/MuonCalibConfig.py index 98445870c5b..6de4e2dc7db 100644 --- a/MuonSpectrometer/MuonCnv/MuonCnvExample/python/MuonCalibConfig.py +++ b/MuonSpectrometer/MuonCnv/MuonCnvExample/python/MuonCalibConfig.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration +# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration #-------------------------------------------------------------- # JobOption fragments for Condition Database access @@ -130,9 +130,10 @@ def CscCalibTool(name,**kwargs): # MDT calibration ################################################################################ +from AtlasGeoModel.CommonGMJobProperties import CommonGeometryFlags if mdtCalibFlags.readMDTCalibFromBlob(): - mdt_folder_name_appendix = "BLOB" + mdt_folder_name_appendix = "BLOB" if CommonGeometryFlags.Run in ["RUN1","RUN2"] else "JSONS" else: mdt_folder_name_appendix="" diff --git a/MuonSpectrometer/MuonConditions/MuonCondGeneral/MuonCondAlg/MuonCondAlg/MdtCalibDbAlg.h b/MuonSpectrometer/MuonConditions/MuonCondGeneral/MuonCondAlg/MuonCondAlg/MdtCalibDbAlg.h index 668c4df638b..f0f4cf5416d 100644 --- a/MuonSpectrometer/MuonConditions/MuonCondGeneral/MuonCondAlg/MuonCondAlg/MdtCalibDbAlg.h +++ b/MuonSpectrometer/MuonConditions/MuonCondGeneral/MuonCondAlg/MuonCondAlg/MdtCalibDbAlg.h @@ -64,7 +64,7 @@ private: Gaudi::Property m_checkTubes{this, "checkTubes", true,"If true the number of tubes must agree between the conditions DB & geometry"}; // new conditions format 2020 - Gaudi::Property m_newFormat2020{this, "NewFormat2020", false, "Use the new calibration data format "}; + Gaudi::Property m_newFormat2020{this, "NewFormat2020", true, "Use the new calibration data format "}; // like MdtCalibrationDbSvc // for corData in loadRt @@ -117,8 +117,8 @@ private: static inline MuonCalib::RtResolutionLookUp* getRtResolutionInterpolation(const std::vector& sample_points); inline StatusCode extractString(std::string& input, std::string& output, const std::string& separator); - SG::ReadCondHandleKey m_readKeyRt{this, "ReadKeyRt", "/MDT/RTBLOB", "DB folder containing the RT calibrations"}; - SG::ReadCondHandleKey m_readKeyTube{this, "ReadKeyTube", "/MDT/T0BLOB", + SG::ReadCondHandleKey m_readKeyRt{this, "ReadKeyRt", "/MDT/RTJSONS", "DB folder containing the RT calibrations"}; + SG::ReadCondHandleKey m_readKeyTube{this, "ReadKeyTube", "/MDT/T0JSONS", "DB folder containing the tube constants"}; SG::WriteCondHandleKey m_writeKeyRt{this, "MdtRtRelationCollection", "MdtRtRelationCollection", "MDT RT relations"}; diff --git a/MuonSpectrometer/MuonConditions/MuonCondGeneral/MuonCondAlg/src/MdtCalibDbAlg.cxx b/MuonSpectrometer/MuonConditions/MuonCondGeneral/MuonCondAlg/src/MdtCalibDbAlg.cxx index bdf55f5aaca..0e223797889 100644 --- a/MuonSpectrometer/MuonConditions/MuonCondGeneral/MuonCondAlg/src/MdtCalibDbAlg.cxx +++ b/MuonSpectrometer/MuonConditions/MuonCondGeneral/MuonCondAlg/src/MdtCalibDbAlg.cxx @@ -318,7 +318,9 @@ StatusCode MdtCalibDbAlg::loadRt() { al.extend("data", "blob"); al["tech"].data() = jt.value()[1]; al["file"].data() = static_cast(jt.value()[2]); - std::string data = jt.value()[3]; + std::string compr = jt.value()[3]; + std::string data; + CoralUtilities::uncompressString(compr, data); if (!CoralUtilities::writeBlobFromString(data, al["data"].data())) { ATH_MSG_FATAL("Cannot compress BLOB!"); return StatusCode::FAILURE; @@ -347,7 +349,7 @@ StatusCode MdtCalibDbAlg::loadRt() { std::string istr; ATH_MSG_VERBOSE("Loading data as a BLOB, uncompressing..."); if (!CoralUtilities::readBlobAsString(atr["data"].data(), istr)) { - ATH_MSG_FATAL("Cannot uncompress BLOB! Aborting..."); + ATH_MSG_FATAL("Cannot uncompress RT BLOB! Aborting..."); return StatusCode::FAILURE; } ATH_CHECK(extractString(istr, header, "\n")); @@ -764,7 +766,9 @@ StatusCode MdtCalibDbAlg::loadTube() { al.extend("data", "blob"); al["tech"].data() = jt.value()[1]; al["file"].data() = static_cast(jt.value()[2]); - std::string data = jt.value()[3]; + std::string compr = jt.value()[3]; + std::string data; + CoralUtilities::uncompressString(compr, data); if (!CoralUtilities::writeBlobFromString(data, al["data"].data())) { ATH_MSG_FATAL("Cannot compress BLOB!"); return StatusCode::FAILURE; @@ -798,7 +802,7 @@ StatusCode MdtCalibDbAlg::loadTube() { std::string istr; ATH_MSG_VERBOSE("Loading data as a BLOB, uncompressing..."); if (!CoralUtilities::readBlobAsString(atr["data"].data(), istr)) { - ATH_MSG_FATAL("Cannot uncompress BLOB! Aborting..."); + ATH_MSG_FATAL("Cannot uncompress T0 BLOB! Aborting..."); return StatusCode::FAILURE; } ATH_CHECK(extractString(istr, header, "\n")); diff --git a/MuonSpectrometer/MuonConditions/MuonCondGeneral/MuonCondAlg/src/MdtCalibFormatAlgTest.cxx b/MuonSpectrometer/MuonConditions/MuonCondGeneral/MuonCondAlg/src/MdtCalibFormatAlgTest.cxx index 612cec935fe..2e04e218fce 100644 --- a/MuonSpectrometer/MuonConditions/MuonCondGeneral/MuonCondAlg/src/MdtCalibFormatAlgTest.cxx +++ b/MuonSpectrometer/MuonConditions/MuonCondGeneral/MuonCondAlg/src/MdtCalibFormatAlgTest.cxx @@ -1,5 +1,5 @@ /* - Copyright (C) 2002-2021 CERN for the benefit of the ATLAS collaboration + Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration */ #include "MuonCondAlg/MdtCalibFormatAlgTest.h" @@ -125,6 +125,21 @@ StatusCode MdtCalibFormatAlgTest::processBlob(const std::string& folder, const s // ATH_CHECK( extractString(istr, payload, "\n") ); // if( istr.size() ) ATH_CHECK( extractString(istr, trailer, "\n") ); ATH_MSG_DEBUG("processing Blob " << folder << "," << setup << "," << data); // leave for now + + if(setup=="new"){ + // unwrap the json and build the data vector + nlohmann::json yy = nlohmann::json::parse(data); + for (auto &it : yy.items()) { + nlohmann::json yx = it.value(); + for (auto &jt : yx.items()) { + std::string data = jt.value()[3]; + std::string test; + CoralUtilities::uncompressString(data, test); + std::cout << test << std::endl; + } + } + } + return StatusCode::SUCCESS; } diff --git a/MuonSpectrometer/MuonConfig/python/MuonCalibrationConfig.py b/MuonSpectrometer/MuonConfig/python/MuonCalibrationConfig.py index 80b71bfa718..e8b1325fd04 100644 --- a/MuonSpectrometer/MuonConfig/python/MuonCalibrationConfig.py +++ b/MuonSpectrometer/MuonConfig/python/MuonCalibrationConfig.py @@ -49,7 +49,7 @@ def _setupMdtCondDB(flags): result=ComponentAccumulator() if flags.Muon.Calib.readMDTCalibFromBlob: - mdt_folder_name_appendix = "BLOB" + mdt_folder_name_appendix = "JSONS" if flags.GeoModel.Run>=LHCPeriod.Run3 else "BLOB" else: mdt_folder_name_appendix="" -- GitLab