From ad4440c2fd5d7e67ba7fb99cd4d03285ff6a3d47 Mon Sep 17 00:00:00 2001 From: LHCbDIRAC Bot <dirac.bot@cern.ch> Date: Tue, 15 Oct 2024 22:56:40 +0200 Subject: [PATCH] Merge branch 'roneil-extendedDQok-bug' into 'master' Fix extendedDQOK See merge request lhcb-dirac/LHCbDIRAC!1646 (cherry picked from commit 85ed60ee4385400cf8d9b25471432d746bb480d0) a0cc57ba set extendedDQOK properly in BK query dict c38112bb Fix other locations to support all manner of references to DQ flag fe93da3f remove whitespace 1705a46b Reformat. d76f6330 fix also for getFiles d1e275e2 inExtendedDQOK is a list that should be turned into a comma-separated string. 5d22d965 another one! (DQflag should be a list in the BKQuery dict!) 3ce2c3d2 Handle correctly dataquality flag if list or string, make sure to add ";;;" delim a0a492d4 Undo semicolon mess c01e1ba7 Clean up ExtendedDQOK error message. Co-authored-by: Chris Burr <christopher.burr@cern.ch> --- .../BookkeepingSystem/DB/LegacyOracleBookkeepingDB.py | 2 +- .../Service/BookkeepingManagerHandler.py | 8 ++++---- .../Client/ProductionRequest.py | 2 +- .../Utilities/ModelCompatibility.py | 4 ++-- .../scripts/dirac_production_request_run_local.py | 4 ++++ 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/LHCbDIRAC/BookkeepingSystem/DB/LegacyOracleBookkeepingDB.py b/src/LHCbDIRAC/BookkeepingSystem/DB/LegacyOracleBookkeepingDB.py index 9b6438ced0..6ff382c02d 100644 --- a/src/LHCbDIRAC/BookkeepingSystem/DB/LegacyOracleBookkeepingDB.py +++ b/src/LHCbDIRAC/BookkeepingSystem/DB/LegacyOracleBookkeepingDB.py @@ -4080,7 +4080,7 @@ prod.production>0 AND prod.production=cont.production AND cont.processingid=%d" if len(quality) != 1 or str(quality[0]) != "OK": return S_ERROR("ExtendedDQOK can be specified with DataQuality=OK only") else: - return S_ERROR("DataQuality OK must be explicitly specified to use ExtendedDQOK " + str(quality)) + return S_ERROR(f"DataQuality OK must be explicitly specified to use ExtendedDQOK: {quality=}") # I leave it here if we wabt change the logic (set 'OK' when not specified at all) # mimic __buildDataquality("OK") when not specified # hardcoded into DB Schema as 2. But current BK test wipe it and set to 1... diff --git a/src/LHCbDIRAC/BookkeepingSystem/Service/BookkeepingManagerHandler.py b/src/LHCbDIRAC/BookkeepingSystem/Service/BookkeepingManagerHandler.py index aef4830e5d..ebea52179c 100644 --- a/src/LHCbDIRAC/BookkeepingSystem/Service/BookkeepingManagerHandler.py +++ b/src/LHCbDIRAC/BookkeepingSystem/Service/BookkeepingManagerHandler.py @@ -573,7 +573,7 @@ class BookkeepingManagerHandlerMixin: evt = in_dict.get("EventType", in_dict.get("EventTypeId", default)) production = in_dict.get("Production", default) filetype = in_dict.get("FileType", default) - quality = in_dict.get("DataQuality", in_dict.get("Quality", default)) + quality = in_dict.get("DataQuality", in_dict.get("DataQualityFlag", in_dict.get("Quality", default))) visible = in_dict.get("Visible", "Y") replicaFlag = in_dict.get("ReplicaFlag", "Yes") startDate = in_dict.get("StartDate", None) @@ -701,7 +701,7 @@ class BookkeepingManagerHandlerMixin: eventType = in_dict.get("EventType", in_dict.get("EventTypeId", default)) production = in_dict.get("Production", default) fileType = in_dict.get("FileType", default) - dataQuality = in_dict.get("DataQuality", in_dict.get("Quality", default)) + dataQuality = in_dict.get("DataQuality", in_dict.get("DataQualityFlag", in_dict.get("Quality", default))) startRun = in_dict.get("StartRun", None) endRun = in_dict.get("EndRun", None) visible = in_dict.get("Visible", "Y") @@ -770,7 +770,7 @@ class BookkeepingManagerHandlerMixin: evt = in_dict.get("EventType", in_dict.get("EventTypeId", default)) production = in_dict.get("Production", default) filetype = in_dict.get("FileType", default) - quality = in_dict.get("DataQuality", in_dict.get("Quality", default)) + quality = in_dict.get("DataQuality", in_dict.get("DataQualityFlag", in_dict.get("Quality", default))) runnb = in_dict.get("RunNumbers", in_dict.get("RunNumber", default)) start = in_dict.get("StartItem", 0) maxValue = in_dict.get("MaxItem", 10) @@ -2114,7 +2114,7 @@ class BookkeepingManagerHandlerMixin: evt = in_dict.get("EventType", in_dict.get("EventTypeId", default)) production = in_dict.get("Production", default) filetype = in_dict.get("FileType", default) - quality = in_dict.get("DataQuality", in_dict.get("Quality", default)) + quality = in_dict.get("DataQuality", in_dict.get("DataQualityFlag", in_dict.get("Quality", default))) runnb = in_dict.get("RunNumber", default) if "Quality" in in_dict: cls.log.verbose("The Quality has to be replaced by DataQuality!") diff --git a/src/LHCbDIRAC/ProductionManagementSystem/Client/ProductionRequest.py b/src/LHCbDIRAC/ProductionManagementSystem/Client/ProductionRequest.py index b9554e07e6..6b06d3b852 100644 --- a/src/LHCbDIRAC/ProductionManagementSystem/Client/ProductionRequest.py +++ b/src/LHCbDIRAC/ProductionManagementSystem/Client/ProductionRequest.py @@ -929,7 +929,7 @@ class ProductionRequest: bkQuery["DataQualityFlag"] = self.dqFlag.replace(",", ";;;").replace(" ", "") if self.extendedDQOK: - bkQuery["ExtendedDQOK"] = self.dqFlag.replace(",", ";;;").replace(" ", "") + bkQuery["ExtendedDQOK"] = self.extendedDQOK.replace(",", ";;;").replace(" ", "") if self.smog2State: bkQuery["SMOG2"] = self.smog2State diff --git a/src/LHCbDIRAC/ProductionManagementSystem/Utilities/ModelCompatibility.py b/src/LHCbDIRAC/ProductionManagementSystem/Utilities/ModelCompatibility.py index faafe6ffef..4c55e3329b 100644 --- a/src/LHCbDIRAC/ProductionManagementSystem/Utilities/ModelCompatibility.py +++ b/src/LHCbDIRAC/ProductionManagementSystem/Utilities/ModelCompatibility.py @@ -428,9 +428,9 @@ def configure_input(pr, data, *, runs=None, startRun=None, endRun=None): pr.configName = scd["configName"] pr.configVersion = scd["configVersion"] - pr.dqFlag = scd["inDataQualityFlag"] + pr.dqFlag = scd["inDataQualityFlag"].replace(" ", "") if scd["inExtendedDQOK"]: - pr.extendedDQOK = scd["inExtendedDQOK"] + pr.extendedDQOK = ",".join(scd["inExtendedDQOK"]) if scd["inSMOG2State"]: pr.smog2State = scd["inSMOG2State"] pr.dataTakingConditions = data["SimCondition"] diff --git a/src/LHCbDIRAC/ProductionManagementSystem/scripts/dirac_production_request_run_local.py b/src/LHCbDIRAC/ProductionManagementSystem/scripts/dirac_production_request_run_local.py index e4e8703f35..db1b4638df 100644 --- a/src/LHCbDIRAC/ProductionManagementSystem/scripts/dirac_production_request_run_local.py +++ b/src/LHCbDIRAC/ProductionManagementSystem/scripts/dirac_production_request_run_local.py @@ -312,6 +312,10 @@ def prepareProductionRequest( bkQueryDict = pr._getBKKQuery().copy() if "RunNumbers" in bkQueryDict: bkQueryDict["RunNumbers"] = bkQueryDict["RunNumbers"].split(";;;") + if "DataQualityFlag" in bkQueryDict: + bkQueryDict["DataQualityFlag"] = bkQueryDict["DataQualityFlag"].split(";;;") + if "ExtendedDQOK" in bkQueryDict: + bkQueryDict["ExtendedDQOK"] = bkQueryDict["ExtendedDQOK"].split(";;;") result = returnValueOrRaise(BookkeepingClient().getFilesWithMetadata(bkQueryDict)) if result["TotalRecords"] == 0: raise ValueError("No input files found in the bookkeeping") -- GitLab