diff --git a/setup.cfg b/setup.cfg index 554b9f55bd4f1c0e0eedcee2382a5821b0c537ed..2662ed95e8d8eeae2ce3f09e315f95298986ddad 100644 --- a/setup.cfg +++ b/setup.cfg @@ -88,6 +88,7 @@ console_scripts = dirac-bookkeeping-get-stats = LHCbDIRAC.BookkeepingSystem.scripts.dirac_bookkeeping_get_stats:main dirac-bookkeeping-get-tck = LHCbDIRAC.BookkeepingSystem.scripts.dirac_bookkeeping_get_tck:main dirac-bookkeeping-getdataquality-runs = LHCbDIRAC.BookkeepingSystem.scripts.dirac_bookkeeping_getdataquality_runs:main + dirac-bookkeeping-getextendeddqok-runs = LHCbDIRAC.BookkeepingSystem.scripts.dirac_bookkeeping_getextendeddqok_runs:main dirac-bookkeeping-job-info = LHCbDIRAC.BookkeepingSystem.scripts.dirac_bookkeeping_job_info:main dirac-bookkeeping-job-input-output = LHCbDIRAC.BookkeepingSystem.scripts.dirac_bookkeeping_job_input_output:main dirac-bookkeeping-prod4path = LHCbDIRAC.BookkeepingSystem.scripts.dirac_bookkeeping_prod4path:main @@ -102,6 +103,7 @@ console_scripts = dirac-bookkeeping-setdataquality = LHCbDIRAC.BookkeepingSystem.scripts.dirac_bookkeeping_setdataquality:main dirac-bookkeeping-setdataquality-files = LHCbDIRAC.BookkeepingSystem.scripts.dirac_bookkeeping_setdataquality_files:main dirac-bookkeeping-setdataquality-run = LHCbDIRAC.BookkeepingSystem.scripts.dirac_bookkeeping_setdataquality_run:main + dirac-bookkeeping-setextendeddqok-run = LHCbDIRAC.BookkeepingSystem.scripts.dirac_bookkeeping_setextendeddqok_run:main dirac-bookkeeping-setdataquality-run-processing-pass = LHCbDIRAC.BookkeepingSystem.scripts.dirac_bookkeeping_setdataquality_run_processing_pass:main dirac-bookkeeping-simulationconditions-insert = LHCbDIRAC.BookkeepingSystem.scripts.dirac_bookkeeping_simulationconditions_insert:main dirac-bookkeeping-simulationconditions-list = LHCbDIRAC.BookkeepingSystem.scripts.dirac_bookkeeping_simulationconditions_list:main diff --git a/src/LHCbDIRAC/BookkeepingSystem/Client/BookkeepingClient.py b/src/LHCbDIRAC/BookkeepingSystem/Client/BookkeepingClient.py index 2f4489b2c89f54ecf5f2e33956b44d6fdf88aa5f..3e212d547dd63671dac507911784048fa06aee90 100644 --- a/src/LHCbDIRAC/BookkeepingSystem/Client/BookkeepingClient.py +++ b/src/LHCbDIRAC/BookkeepingSystem/Client/BookkeepingClient.py @@ -251,6 +251,18 @@ class BookkeepingClient(Client): runs = [runs] return self._getRPC().getRunFilesDataQuality(runs) + ############################################################################# + def getRunExtendedDQOK(self, runnb): + """For retrieve ExtendedDQOK for run. + + :param (int, str) runnb: run number. + :return: list of system names + """ + try: + return self._getRPC().getRunExtendedDQOK(int(runnb)) + except (ValueError, TypeError) as e: + return S_ERROR(f"Invalid run number: {repr(e)}") + ############################################################################# def setFilesInvisible(self, lfns): """It is used to set the file(s) invisible in the database. diff --git a/src/LHCbDIRAC/BookkeepingSystem/DB/LegacyOracleBookkeepingDB.py b/src/LHCbDIRAC/BookkeepingSystem/DB/LegacyOracleBookkeepingDB.py index 3ce021541c297b0c176164fa2209b2b5e9687e3c..6c59bf1d9512ec395d16eeed4d43139eb061f6f9 100644 --- a/src/LHCbDIRAC/BookkeepingSystem/DB/LegacyOracleBookkeepingDB.py +++ b/src/LHCbDIRAC/BookkeepingSystem/DB/LegacyOracleBookkeepingDB.py @@ -1169,6 +1169,7 @@ WHERE prod.production=cont.production %s AND prod.filetypeId=ftypes.filetypeid % jobEnd=None, selection=None, smog2States=None, + dqok=None, ): """For retrieving files with meta data. @@ -1243,6 +1244,11 @@ WHERE prod.production=cont.production %s AND prod.filetypeId=ftypes.filetypeid % return retVal condition, tables = retVal["Value"] + retVal = self.__buildExtendedDQOK(quality, dqok, condition, tables) + if not retVal["OK"]: + return retVal + condition, tables = retVal["Value"] + retVal = self.__buildProcessingPass(processing, condition, tables) if not retVal["OK"]: return retVal @@ -3570,6 +3576,7 @@ prod.production>0 AND prod.production=cont.production AND cont.processingid=%d" jobStart=None, jobEnd=None, smog2States=None, + dqok=None, ): """returns a list of lfns. @@ -3638,6 +3645,11 @@ prod.production>0 AND prod.production=cont.production AND cont.processingid=%d" return retVal condition, tables = retVal["Value"] + retVal = self.__buildExtendedDQOK(flag, dqok, condition, tables) + if not retVal["OK"]: + return retVal + condition, tables = retVal["Value"] + condition = self.__buildReplicaflag(replicaFlag, condition) condition, tables = self.__buildVisibilityflag(visible, condition, tables) @@ -4019,6 +4031,63 @@ prod.production>0 AND prod.production=cont.production AND cont.processingid=%d" condition += " AND f.qualityid=" + str(quality) return S_OK((condition, tables)) + ############################################################################# + def __buildExtendedDQOK(self, quality, dqok, condition, tables): + """it adds the ExtendedDQOOK tables and conditions. + NOTE: Data quality must be (explicitly) 'OK'. + That can be changed, f.e. set 'OK' when not set at all, ignore ExtendedDQOK for no explicit DataQuality + or no 'OK' in it and making condition to use ExtendedDQOK restrictions to + files with 'OK' only (when several possible qualities are specified, including 'OK'). + + :param str|list[str] quality: data quality flag + :param str|list[str] dqok: the list of systems which must be ok + :param str condition: condition string + :param str tables: tables used by join + :return: condition and tables + """ + + if not dqok: + return S_OK((condition, tables)) # don't wake the sleeping bear... + + # require jobs and files tables. + # __buildDataquality() assumes files table, so that is not an extra requirement + # but I explicitly check (at least for now) + + if not tables or "jobs j" not in tables.lower() or "files f" not in tables.lower(): + raise RuntimeError("No jobs or files to join ExtendedDQOK, exiting...") + if condition is None: + condition = "" # all jobs... the list can be long... + + if quality not in [default, None]: + if not isinstance(quality, (list, tuple)): + quality = [quality] + if len(quality) != 1 or str(quality[0]) != "OK": + return S_ERROR("ExtendedDQOK can be specified with DataQuality=OK only") + else: + return S_ERROR("DataQuality=OK must be explicitly specified to use ExtendedDQOK" + str(quality)) + # I leave it here if we wabt change the logic (set 'OK' when not specified at all) + # mimic __buildDataquality("OK") when not specified + # hardcoded into DB Schema as 2. But current BK test wipe it and set to 1... + # command = "SELECT QualityId FROM dataquality WHERE dataqualityflag='OK'" + # res = self.dbR_.query(command) + # if not res["OK"]: + # return S_ERROR("Data quality problem:", res["Message"]) + # elif not res["Value"]: + # return S_ERROR("No file found! Dataquality is missing!") + # qualityid = res["Value"][0][0] + # condition += f" AND f.qualityid={qualityid}" + + if not isinstance(dqok, (list, tuple)): + if isinstance(dqok, str): + dqok = [str] + else: + return S_ERROR("incorrect ExtendedDQOK type, expected a string or a list of strings") + + for i, system in enumerate(dqok): + tables += f" , extendeddqok edqok{i}" + condition += f" AND edqok{i}.runnumber = j.runnumber AND edqok{i}.systemname = '{system}'" + return S_OK((condition, tables)) + ############################################################################# @staticmethod def __buildReplicaflag(replicaFlag, condition): @@ -4242,6 +4311,7 @@ j.runnumber, j.fillnumber, f.filesize, j.totalluminosity, f.luminosity, f.instLu jobStart=None, jobEnd=None, smog2States=None, + dqok=None, ): """File summary for a given data set. @@ -4320,6 +4390,11 @@ prod.eventtypeid=f.eventtypeid %s " return retVal condition, tables = retVal["Value"] + retVal = self.__buildExtendedDQOK(dataQuality, dqok, condition, tables) + if not retVal["OK"]: + return retVal + condition, tables = retVal["Value"] + retVal = self._buildSMOG2Conditions(smog2States, condition, tables) if not retVal["OK"]: return retVal diff --git a/src/LHCbDIRAC/BookkeepingSystem/DB/NewOracleBookkeepingDB.py b/src/LHCbDIRAC/BookkeepingSystem/DB/NewOracleBookkeepingDB.py index f90e51dd8ee42f80364b58bf0186b129d468590b..3ec7038fa4f2ef05de7111e45be89128bc69f537 100644 --- a/src/LHCbDIRAC/BookkeepingSystem/DB/NewOracleBookkeepingDB.py +++ b/src/LHCbDIRAC/BookkeepingSystem/DB/NewOracleBookkeepingDB.py @@ -9,7 +9,7 @@ # or submit itself to any jurisdiction. # ############################################################################### from DIRAC import gLogger -from DIRAC.Core.Utilities.ReturnValues import DReturnType, returnValueOrRaise, convertToReturnValue +from DIRAC.Core.Utilities.ReturnValues import S_OK, DReturnType, returnValueOrRaise, convertToReturnValue class NewOracleBookkeepingDB: @@ -42,7 +42,7 @@ class NewOracleBookkeepingDB: def getRunsForSMOG2(self, state: str) -> list[int]: """Retrieve all runs with specified SMOG2 state - :param sr state: required state + :param str state: required state """ query = "SELECT runs.runnumber FROM smog2 LEFT JOIN runs ON runs.smog2_id = smog2.id WHERE smog2.state = :state" result = returnValueOrRaise(self.dbR_.query(query, kwparams={"state": state})) @@ -58,3 +58,46 @@ class NewOracleBookkeepingDB: return self.dbW_.executeStoredProcedure( "BOOKKEEPINGORACLEDB.setSMOG2", parameters=[state, update], output=False, array=runs ) + + def setExtendedDQOK(self, run: int, update: bool, dqok: list[str]) -> DReturnType[None]: + """Set ExtendedDQOK for specified run and systems. In case update is allowed, + not specified systems are unset for the run. + + :param int run: run number for which systems are specified + :param bool update: when True, updates existing set, when False throw an error in such case + :param list[str] dqok: list of system names + """ + return self.dbW_.executeStoredProcedure( + "BOOKKEEPINGORACLEDB.setExtendedDQOK", parameters=[run, update, dqok], output=False + ) + + @convertToReturnValue + def getRunsWithExtendedDQOK(self, dqok: list[str]) -> list[int]: + """Retrieve all runs with specified systems in ExtendedDQOK + NOTE: it is NOT checking quality is set to OK, so it should NOT be used + for end user operations. + + :param list[str] dqok: systems + """ + if not dqok: + return [] + sql = ["SELECT ok.runnumber FROM extendeddqok ok"] + params = {"sysname0": dqok[0]} + for i, system in enumerate(dqok[1::]): + sql.append( + f"INNER JOIN extendeddqok ok{i} ON ok{i}.runnumber = ok.runnumber AND ok{i}.systemname = :sysname{i}" + ) + params[f"sysname{i}"] = system + sql.append("WHERE ok.systemname = :sysname0") + result = returnValueOrRaise(self.dbR_.query(" ".join(sql), kwparams=params)) + return [run for run, in result] + + @convertToReturnValue + def getRunExtendedDQOK(self, runnb: int) -> list[str]: + """Return the list of systems in ExtendedDQOK for given run + + :param int runnb: run number + """ + query = "SELECT systemname FROM extendeddqok WHERE runnumber = :run" + result = returnValueOrRaise(self.dbR_.query(query, kwparams={"run": runnb})) + return [sysname for sysname, in result] diff --git a/src/LHCbDIRAC/BookkeepingSystem/DB/database_schema.sql b/src/LHCbDIRAC/BookkeepingSystem/DB/database_schema.sql index 2933097056fc4aa1374347cd1a44b5d0f169c5f2..719d61fd7df307b3cf31749797f3a2de4f15abae 100644 --- a/src/LHCbDIRAC/BookkeepingSystem/DB/database_schema.sql +++ b/src/LHCbDIRAC/BookkeepingSystem/DB/database_schema.sql @@ -322,6 +322,14 @@ INSERT INTO dataquality (qualityid,dataqualityflag) SELECT 2,'OK' FROM dual WHER INSERT INTO dataquality (qualityid,dataqualityflag) SELECT 3,'BAD' FROM dual WHERE NOT EXISTS (SELECT * FROM dataquality WHERE (qualityid = 3 AND dataqualityflag = 'BAD')); COMMIT; +--------------------------------------------------------------------------------------- +CREATE TABLE extendeddqok( + runnumber NUMBER, + systemname varchar2(64), + CONSTRAINT pk_extendeddqok_p PRIMARY KEY (runnumber, systemname) +); +COMMIT; + --------------------------------------------------------------------------------------- CREATE TABLE smog2( id NUMBER, diff --git a/src/LHCbDIRAC/BookkeepingSystem/DB/oracle_schema_storedprocedures.sql b/src/LHCbDIRAC/BookkeepingSystem/DB/oracle_schema_storedprocedures.sql index cf70bd90a92c6459b073e45c5fb78f1b09f9398e..21243d1f877506b10344088d2b0eb75b92bd46e9 100644 --- a/src/LHCbDIRAC/BookkeepingSystem/DB/oracle_schema_storedprocedures.sql +++ b/src/LHCbDIRAC/BookkeepingSystem/DB/oracle_schema_storedprocedures.sql @@ -157,6 +157,7 @@ PROCEDURE insertproductionscontainer(v_prod NUMBER, v_processingid NUMBER, v_sim PROCEDURE geteventtypes(cname VARCHAR2, cversion VARCHAR2, a_cursor OUT udt_refcursor); FUNCTION getrunnumber(lfn VARCHAR2) RETURN NUMBER; PROCEDURE insertrunquality(run NUMBER, qid NUMBER,procid NUMBER); +PROCEDURE setextendeddqok(run NUMBER, overwrite BOOLEAN, dqok varchararray); PROCEDURE getrunnbandtck(lfn VARCHAR2, a_cursor OUT udt_refcursor); PROCEDURE deleteproductionscont(v_prod NUMBER); PROCEDURE getruns(c_name VARCHAR2, c_version VARCHAR2, a_cursor OUT udt_refcursor); @@ -1480,6 +1481,53 @@ EXCEPTION UPDATE newrunquality SET qualityid = qid WHERE processingid = procid AND runnumber = run; COMMIT; END; +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +PROCEDURE setextendeddqok(run NUMBER, overwrite BOOLEAN, dqok varchararray) IS + v_dqok varchararray := dqok; -- yet another Oracle "feature"... + v_to_del varchararray; + v_to_ins varchararray; +BEGIN + -- collect what we should do + BEGIN + SELECT systemname BULK COLLECT INTO v_to_del FROM extendeddqok + WHERE runnumber = run AND systemname NOT IN (SELECT * FROM TABLE(v_dqok)); + EXCEPTION WHEN NO_DATA_FOUND THEN + NULL; + WHEN OTHERS THEN + RAISE; + END; + + BEGIN + SELECT ok.column_value BULK COLLECT INTO v_to_ins FROM TABLE(v_dqok) ok + LEFT JOIN extendeddqok e ON e.runnumber = run AND e.systemname = ok.column_value + WHERE e.systemname IS NULL; + EXCEPTION WHEN NO_DATA_FOUND THEN + NULL; + WHEN OTHERS THEN + RAISE; + END; + + -- apply changed, if required and allowed + IF v_to_del.COUNT != 0 OR v_to_ins.COUNT != 0 THEN + IF NOT overwrite AND ( v_to_del.COUNT != 0 OR v_to_ins.COUNT != dqok.COUNT) THEN + raise_application_error(-20007, 'Run ' || run || ' has different ExtendedDQOK'); + END IF; + IF v_to_del.COUNT != 0 THEN + -- I wish that works: DELETE FROM extendeddqok WHERE runnumber = v_run AND systemname IN (SELECT * FROM TABLE(v_to_del)); + -- but it does not, at least in earlier Oracle versions + FOR i IN v_to_del.first .. v_to_del.last LOOP + -- dbms_output.put_line(v_to_del(i)); + DELETE FROM extendeddqok WHERE runnumber = run AND systemname = v_to_del(i); + END LOOP; + END IF; + IF v_to_ins.COUNT != 0 THEN + FOR i IN v_to_ins.first .. v_to_ins.last LOOP + INSERT INTO extendeddqok (runnumber, systemname) VALUES (run, v_to_ins(i)); + END LOOP; + END IF; + COMMIT; + END IF; +END; ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- PROCEDURE getrunnbandtck(lfn VARCHAR2, a_cursor OUT udt_refcursor) IS BEGIN diff --git a/src/LHCbDIRAC/BookkeepingSystem/Service/BookkeepingManagerHandler.py b/src/LHCbDIRAC/BookkeepingSystem/Service/BookkeepingManagerHandler.py index 0c24c21b8fce3c73b4648ad6950f0b02e3b4fccd..473e4dee47a33edd841696fd693825e3c1027982 100644 --- a/src/LHCbDIRAC/BookkeepingSystem/Service/BookkeepingManagerHandler.py +++ b/src/LHCbDIRAC/BookkeepingSystem/Service/BookkeepingManagerHandler.py @@ -504,6 +504,7 @@ class BookkeepingManagerHandlerMixin: jobStart = in_dict.get("JobStartDate", None) jobEnd = in_dict.get("JobEndDate", None) smog2States = in_dict.get("SMOG2", None) + extendedDQOK = in_dict.get("ExtendedDQOK", None) if "ProductionID" in in_dict: cls.log.verbose("ProductionID will be removed. It will changed to Production") @@ -537,6 +538,7 @@ class BookkeepingManagerHandlerMixin: jobStart, jobEnd, smog2States, + extendedDQOK, ) if not retVal["OK"]: return retVal @@ -567,6 +569,7 @@ class BookkeepingManagerHandlerMixin: jobStart = in_dict.get("JobStartDate", None) jobEnd = in_dict.get("JobEndDate", None) smog2States = in_dict.get("SMOG2", None) + extendedDQOK = in_dict.get("ExtendedDQOK", None) if "EventTypeId" in in_dict: cls.log.verbose("The EventTypeId has to be replaced by EventType!") @@ -595,6 +598,7 @@ class BookkeepingManagerHandlerMixin: jobEnd, None, smog2States, + extendedDQOK, ) if not retVal["OK"]: return retVal @@ -690,6 +694,7 @@ class BookkeepingManagerHandlerMixin: jobStart = in_dict.get("JobStartDate", None) jobEnd = in_dict.get("JobEndDate", None) smog2States = in_dict.get("SMOG2", None) + extendedDQOK = in_dict.get("ExtendedDQOK", None) if "EventTypeId" in in_dict: self.log.verbose("The EventTypeId has to be replaced by EventType!") @@ -717,6 +722,7 @@ class BookkeepingManagerHandlerMixin: jobStart=jobStart, jobEnd=jobEnd, smog2States=smog2States, + dqok=extendedDQOK, ) if not retVal["OK"]: return retVal @@ -1030,6 +1036,16 @@ class BookkeepingManagerHandlerMixin: """ return cls.bkkDB.setRunDataQuality(runNb, flag) + ############################################################################# + types_setExtendedDQOK = [int, bool, list] + + @classmethod + def export_setExtendedDQOK(cls, runNB, update, dqok): + """For given run insert or update the list of Systems which are OK + NOTE: this call is priviledged + """ + return cls.bkkDB.setExtendedDQOK(runNB, update, dqok) + ############################################################################# types_setProductionDataQuality = [int, str] @@ -1531,6 +1547,14 @@ class BookkeepingManagerHandlerMixin: """more info in the BookkeepingClient.py.""" return cls.bkkDB.getRunFilesDataQuality(runs) + ############################################################################# + types_getRunExtendedDQOK = [int] + + @classmethod + def export_getRunExtendedDQOK(cls, runNB): + """Return the list of systems in ExtendedDQOK for given run""" + return cls.bkkDB.getRunExtendedDQOK(runNB) + ############################################################################# types_setFilesInvisible = [list] diff --git a/src/LHCbDIRAC/BookkeepingSystem/scripts/dirac_bookkeeping_getextendeddqok_runs.py b/src/LHCbDIRAC/BookkeepingSystem/scripts/dirac_bookkeeping_getextendeddqok_runs.py new file mode 100755 index 0000000000000000000000000000000000000000..b3bb743cb20e6805605bdbb128c5acf4d91d89c1 --- /dev/null +++ b/src/LHCbDIRAC/BookkeepingSystem/scripts/dirac_bookkeeping_getextendeddqok_runs.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +############################################################################### +# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +"""Get Data Quality Flag for the given run.""" +import DIRAC + +from DIRAC import gLogger +from DIRAC.Core.Base.Script import Script + + +@Script() +def main(): + Script.setUsageMessage( + __doc__ + + "\n".join( + ["Usage:", f" {Script.scriptName} [option|cfgfile] ... Run ...", "Arguments:", " Run: Run number"] + ) + ) + Script.parseCommandLine(ignoreErrors=True) + runSet = {int(id) for arg in Script.getPositionalArgs() for id in arg.split(",")} + + if not runSet: + Script.showHelp(exitCode=1) + + gLogger.showHeaders(False) + from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient + + cl = BookkeepingClient() + + gLogger.notice("Run Number".ljust(15) + "ExtendedDQOK".ljust(15)) + + error = False + for runId in sorted(runSet): + retVal = cl.getRunExtendedDQOK(runId) + if retVal["OK"]: + systems = ",".join(retVal["Value"]) + gLogger.notice(str(runId).ljust(15) + str(systems).ljust(15)) + else: + gLogger.error(retVal["Message"]) + error = True + + if error: + DIRAC.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/LHCbDIRAC/BookkeepingSystem/scripts/dirac_bookkeeping_setextendeddqok_run.py b/src/LHCbDIRAC/BookkeepingSystem/scripts/dirac_bookkeeping_setextendeddqok_run.py new file mode 100755 index 0000000000000000000000000000000000000000..4dc1c180e6a263c221a9c7c9f7b25847bfb700b1 --- /dev/null +++ b/src/LHCbDIRAC/BookkeepingSystem/scripts/dirac_bookkeeping_setextendeddqok_run.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +############################################################################### +# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "LICENSE". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +"""Set ExtendedDQOK systems for the given run.""" +import DIRAC +from DIRAC.Core.Base.Script import Script + + +@Script() +def main(): + Script.registerSwitch("", "Overwrite", "Overwrite already registered list") + Script.setUsageMessage( + __doc__ + + "\n".join( + [ + "Usage:", + f" {Script.scriptName} [option|cfgfile] ... Run [Systems]", + "Arguments:", + " Run: Run number", + " Systems: Comma separated list of system names (unset all when not specified)", + ] + ) + ) + Script.parseCommandLine(ignoreErrors=True) + overwrite = False + for switch in Script.getUnprocessedSwitches(): + if switch[0] == "Overwrite": + overwrite = True + args = Script.getPositionalArgs() + + from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient + + bk = BookkeepingClient() + + if len(args) < 1: + Script.showHelp() + elif len(args) < 2: + systems = [] + else: + systems = args[1].split(",") + + exitCode = 0 + rnb = int(args[0]) + result = bk.setExtendedDQOK(rnb, overwrite, systems) + + if not result["OK"]: + if "ORA-20007" in result["Message"]: + print("ERROR: different list is already registered for the run (use --Overwrite when desired)") + else: + print(f"ERROR: {result['Message']}") + exitCode = 2 + else: + print("The list was updated") + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/LHCbDIRAC/DataManagementSystem/Client/DMScript.py b/src/LHCbDIRAC/DataManagementSystem/Client/DMScript.py index abebee9891ad76ba836061227bd902a1da6a708d..525af1fa69bc2a51c8516f2c278376ef8b57d395 100644 --- a/src/LHCbDIRAC/DataManagementSystem/Client/DMScript.py +++ b/src/LHCbDIRAC/DataManagementSystem/Client/DMScript.py @@ -254,6 +254,9 @@ class DMScript: Script.registerSwitch( "", "SMOG2=", " Required SMOG2 (comma separated list, may include 'Undefined')", self.setSMOG2 ) + Script.registerSwitch( + "", "ExtendedDQOK=", " Comma separated list of (extended) systems which must be ok", self.setExtendedDQOK + ) def registerNamespaceSwitches(self, action="search [ALL]"): """Register namespace switches.""" @@ -358,6 +361,12 @@ class DMScript: self.options["SMOG2"] = states return DIRAC.S_OK() + def setExtendedDQOK(self, arg): + """Setter.""" + states = arg.split(",") + self.options["ExtendedDQOK"] = states + return DIRAC.S_OK() + def setVisibility(self, arg): """Setter.""" if arg.lower() in ("yes", "no", "all"): @@ -571,6 +580,8 @@ class DMScript: self.bkClientQuery.setOption("TCK", self.options["TCK"]) if "SMOG2" in self.options: self.bkClientQuery.setOption("SMOG2", self.options["SMOG2"]) + if "ExtendedDQOK" in self.options: + self.bkClientQuery.setOption("ExtendedDQOK", self.options["ExtendedDQOK"]) return self.bkClientQuery def getRequestID(self, prod=None): diff --git a/tests/Integration/BookkeepingSystem/Test_BookkeepingDB.py b/tests/Integration/BookkeepingSystem/Test_BookkeepingDB.py index d08bc888358341d22b80a2699ecc74b915659326..899871aa0f5315af555bd83d23f29846d514a62c 100644 --- a/tests/Integration/BookkeepingSystem/Test_BookkeepingDB.py +++ b/tests/Integration/BookkeepingSystem/Test_BookkeepingDB.py @@ -97,3 +97,43 @@ def test_SMOG2State(bkkDB): assert "is unknown" in str(excinfo.value) # Check that data are fine assert set(returnValueOrRaise(bkkDB.getRunsForSMOG2("Oxygen"))) == {1001} + + +def test_ExtendedDQOK(bkkDB): + wipeOutDB(bkkDB) + # unset not existing run, should be fine (nothing to do) + assert returnValueOrRaise(bkkDB.setExtendedDQOK(5000, False, [])) is None + # set for new run + assert returnValueOrRaise(bkkDB.setExtendedDQOK(5000, False, ["SMOG2", "AA"])) is None + # extend for existing run, should fail (not allowed) + with pytest.raises(SErrorException) as excinfo: + assert returnValueOrRaise(bkkDB.setExtendedDQOK(5000, False, ["SMOG2", "AA", "BB"])) + # extend for existing run, should be fine (allowed) + assert returnValueOrRaise(bkkDB.setExtendedDQOK(5000, True, ["SMOG2", "AA", "BB"])) is None + # extend and remove for existing run, should fail (not allowed) + with pytest.raises(SErrorException) as excinfo: + assert returnValueOrRaise(bkkDB.setExtendedDQOK(5000, False, ["SMOG2", "BB", "CC"])) + # extend and remove for existing run, should be fine (allowed) + assert returnValueOrRaise(bkkDB.setExtendedDQOK(5000, True, ["SMOG2", "BB", "CC"])) is None + # set for yet another runs + assert returnValueOrRaise(bkkDB.setExtendedDQOK(5001, False, ["SMOG2", "BB", "DD"])) is None + assert ( + returnValueOrRaise( + bkkDB.setExtendedDQOK( + 5002, + False, + [ + "SMOG2", + "AA", + ], + ) + ) + is None + ) + # check that data are as expected + assert set(returnValueOrRaise(bkkDB.getRunsWithExtendedDQOK(["SMOG2"]))) == {5000, 5001, 5002} + assert set(returnValueOrRaise(bkkDB.getRunsWithExtendedDQOK(["AA"]))) == {5002} + assert set(returnValueOrRaise(bkkDB.getRunsWithExtendedDQOK(["SMOG2", "BB"]))) == {5000, 5001} + assert set(returnValueOrRaise(bkkDB.getRunsWithExtendedDQOK(["SMOG2", "BB", "DD"]))) == {5001} + assert set(returnValueOrRaise(bkkDB.getRunsWithExtendedDQOK(["EE"]))) == set() + assert set(returnValueOrRaise(bkkDB.getRunsWithExtendedDQOK([]))) == set() diff --git a/tests/Integration/BookkeepingSystem/Test_Bookkeeping_Files.py b/tests/Integration/BookkeepingSystem/Test_Bookkeeping_Files.py index 26ee7b68e96fab4e5d9b3213847c22d768dc0ced..7cd01ac976b14be5720fe4bb7325f2e935fd031c 100644 --- a/tests/Integration/BookkeepingSystem/Test_Bookkeeping_Files.py +++ b/tests/Integration/BookkeepingSystem/Test_Bookkeeping_Files.py @@ -25,6 +25,7 @@ from .Utilities import wipeOutDB, addBasicData, insertRAWFiles, forceScheduledJo # sut from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient from LHCbDIRAC.BookkeepingSystem.DB.OracleBookkeepingDB import OracleBookkeepingDB +from DIRAC.Core.Utilities.ReturnValues import SErrorException, returnValueOrRaise # What's used for the tests @@ -276,6 +277,19 @@ def test_getFiles_SMOG2(wipeout): assert len(res["Value"]) == 6 +def test_getFiles_ExtendedDQOK(wipeout): + """test getFiles method with ExtendedDQOK""" + bkQueryDict = {"ConfigName": "Test", "FileType": "RAW", "ExtendedDQOK": ["SMOG2", "AA"]} + with pytest.raises(SErrorException) as excinfo: + returnValueOrRaise(bk.getFiles(bkQueryDict)) + assert "DataQuality=OK" in str(excinfo.value) + + bkQueryDict = {"ConfigName": "Test", "FileType": "RAW", "DataQuality": "OK", "ExtendedDQOK": ["SMOG2", "AA"]} + res = bk.getFiles(bkQueryDict) + assert res["OK"], res["Message"] + assert len(res["Value"]) == 6 + + def test_getFilesWithMetadata_SMOG2(wipeout): """test getFilesWithMetadata works with SMOG2 state""" bkQueryDict = {"ConfigName": "Test", "FileType": "RAW", "SMOG2": ["Undefined"]} @@ -294,6 +308,14 @@ def test_getFilesWithMetadata_SMOG2(wipeout): assert res["Value"]["TotalRecords"] == 6 +def test_getFilesWithMetadata_ExtendedDQOK(wipeout): + """test getFilesWithMetadata method with ExtendedDQOK""" + bkQueryDict = {"ConfigName": "Test", "FileType": "RAW", "DataQuality": "OK", "ExtendedDQOK": ["SMOG2", "AA"]} + res = bk.getFilesWithMetadata(bkQueryDict) + assert res["OK"], res["Message"] + assert res["Value"]["TotalRecords"] == 6 + + def test_getFilesSummary_SMOG2(wipeout): """test getFilesSummary works with SMOG2 state""" bkQueryDict = {"ConfigName": "Test", "FileType": "RAW", "SMOG2": ["Undefined"]} @@ -308,6 +330,14 @@ def test_getFilesSummary_SMOG2(wipeout): assert res["Value"]["Records"][0][0] == 5 +def test_getFilesSummary_ExtendedDQOK(wipeout): + """test getFilesSummary method with ExtendedDQOK""" + bkQueryDict = {"ConfigName": "Test", "FileType": "RAW", "DataQuality": "OK", "ExtendedDQOK": ["SMOG2", "AA"]} + res = bk.getFilesSummary(bkQueryDict) + assert res["OK"], res["Message"] + assert res["Value"]["Records"][0][0] == 6 + + def test_addFiles(wipeout): """ add replica flag diff --git a/tests/Integration/BookkeepingSystem/Utilities.py b/tests/Integration/BookkeepingSystem/Utilities.py index b0203d897a9ca4e59c9e4abecf8a6e4f06383860..7eaf092fc240743d5850f81c2588d8d5dfd6eb9a 100644 --- a/tests/Integration/BookkeepingSystem/Utilities.py +++ b/tests/Integration/BookkeepingSystem/Utilities.py @@ -157,6 +157,7 @@ def wipeOutDB(bkDB): bkDB._legacydb.dbW_.query("DELETE FROM data_taking_conditions") bkDB._legacydb.dbW_.query("DELETE FROM newrunquality") bkDB._legacydb.dbW_.query("DELETE FROM runs") + bkDB._legacydb.dbW_.query("DELETE FROM extendeddqok") # still needed? bkDB._legacydb.dbW_.query("DELETE FROM applications") @@ -182,6 +183,8 @@ def insertRAWFiles(bk): assert res["OK"], res["Message"] res = bk.setRunAndProcessingPassDataQuality(runnb_2, "/Real Data", "OK") assert res["OK"], res["Message"] + res = bk.setExtendedDQOK(runnb_2, True, ["SMOG2", "AA"]) + assert res["OK"], res["Message"] currentTime = datetime.datetime.now()