diff --git a/src/LHCbDIRAC/DataManagementSystem/Agent/StorageHistoryAgent.py b/src/LHCbDIRAC/DataManagementSystem/Agent/StorageHistoryAgent.py index d2ef564638dd0d682394379ce8b984ee8b7deb43..67aa388c413941e1ca8d568265a81da4bec58d4b 100644 --- a/src/LHCbDIRAC/DataManagementSystem/Agent/StorageHistoryAgent.py +++ b/src/LHCbDIRAC/DataManagementSystem/Agent/StorageHistoryAgent.py @@ -185,7 +185,7 @@ class StorageHistoryAgent(AgentModule): # d, size, files = row topDirLogicalUsage[topDir]["Files"] += logicalUsage[row]["Files"] topDirLogicalUsage[topDir]["Size"] += logicalUsage[row]["Size"] - self.log.verbose( + self.log.debug( f"After scan of {directory}, total of {topDir}: ", "size: %.4f TB files: %d" % ( @@ -307,15 +307,15 @@ class StorageHistoryAgent(AgentModule): if dirLfn not in fullDirectory: self.log.error(f"ERROR: fullDirectory should include the dirname: {fullDirectory} {dirLfn} ") continue - self.log.verbose(f"Processing directory {dirLfn} ") + self.log.debug(f"Processing directory {dirLfn} ") if dirLfn not in self.pfnUsage: self.log.error(f"ERROR: directory does not have PFN usage {dirLfn} ") continue - self.log.verbose(f"PFN usage: {self.pfnUsage[dirLfn]} ") + self.log.debug(f"PFN usage: {self.pfnUsage[dirLfn]} ") if dirLfn not in self.lfnUsage: self.log.error(f"ERROR: directory does not have LFN usage {dirLfn} ") continue - self.log.verbose(f"LFN usage: {self.lfnUsage[dirLfn]} ") + self.log.debug(f"LFN usage: {self.lfnUsage[dirLfn]} ") # for DEBUGGING: for se in self.pfnUsage[dirLfn]: @@ -414,7 +414,7 @@ class StorageHistoryAgent(AgentModule): notInCache = [] if notFound: self.log.info(f"Memory metadata cache missed for {len(notFound)} directories") - self.log.verbose(f"call getDirMetadata for (first 10): {str(notFound[0:10])} ") + self.log.debug(f"call getDirMetadata for (first 10): {str(notFound[0:10])} ") for dirChunk in breakListIntoChunks(notFound, 10000): self.callsToDudbForMetadata += 1 res = self.__dataUsageClient.getDirMetadata( @@ -427,7 +427,7 @@ class StorageHistoryAgent(AgentModule): # 3 just a try ##############################################3 notInCache += dirChunk continue - self.log.verbose(f"getDirMetadata returned: {str(res['Value'])} ") + self.log.debug(f"getDirMetadata returned: {str(res['Value'])} ") for dirName in dirChunk: # Compatibility with old (list for single file) and new (dictionary) service if isinstance(res["Value"], type({})): @@ -480,7 +480,7 @@ class StorageHistoryAgent(AgentModule): ) # All is OK, directory found _fillMetadata(metaForDir, metadata) - self.log.verbose(f"Cache entry {dirName} in DirMetadata table..") + self.log.debug(f"Cache entry {dirName} in DirMetadata table..") resInsert = self.__dataUsageClient.insertToDirMetadata({dirName: metadata}) if not resInsert["OK"]: self.log.error( @@ -488,11 +488,11 @@ class StorageHistoryAgent(AgentModule): ) else: cachedFromBK.append(dirName) - self.log.verbose(f"Successfully cached metadata for {dirName} : {str(metadata)}") + self.log.debug(f"Successfully cached metadata for {dirName} : {str(metadata)}") self.log.debug(f"result: {str(resInsert)} ") else: # Directory not found - self.log.verbose(f"Directory {dirName} not registered in Bookkeeping!") + self.log.debug(f"Directory {dirName} not registered in Bookkeeping!") _fillMetadata(metaForDir, "notInBkk") failedBK.append(dirName) self.directoriesNotInBkk.append(dirName) @@ -559,7 +559,7 @@ class StorageHistoryAgent(AgentModule): continue secDir = splitDir[2] if secDir in ignoredDirectories: - self.log.verbose(f"Directory to be ignored, skipped: {dirItem} ") + self.log.debug(f"Directory to be ignored, skipped: {dirItem} ") ignoredDirectories[secDir] += 1 continue # for each type of directory (MC, reconstructed data and raw data) check the format, @@ -576,12 +576,12 @@ class StorageHistoryAgent(AgentModule): try: # RAW data directories have a special format, see above if splitDir[-6] == "RAW": - self.log.verbose(f"RAW DATA directory: {splitDir}") + self.log.debug(f"RAW DATA directory: {splitDir}") directory = dirItem fullDirectory = directory else: # These are production directories, see above for the format - self.log.verbose(f"MC or reconstructed data directory: {splitDir}") + self.log.debug(f"MC or reconstructed data directory: {splitDir}") # HIST directories do not have a "suffix" (i.e. all in the same directory)! if splitDir[-3] == "HIST": directory = dirItem @@ -592,7 +592,7 @@ class StorageHistoryAgent(AgentModule): fullDirectory = dirItem # Keep the link between the accounting directory and the full directory name self.dirDict.setdefault(directory, fullDirectory) - self.log.verbose(f"Directory contains production files: {directory} ") + self.log.debug(f"Directory contains production files: {directory} ") except Exception: self.log.warn(f"The directory has unexpected format: {dirItem} ") @@ -610,7 +610,7 @@ class StorageHistoryAgent(AgentModule): self.log.info("Getting the number of files and size from StorageUsage service") for directory in self.dirDict: - self.log.verbose(f"Get storage usage for directory {directory} ") + self.log.debug(f"Get storage usage for directory {directory} ") res = self.__stDB.getDirectorySummaryPerSE(directory) self.callsToDirectorySummary += 1 if not res["OK"]: @@ -619,7 +619,7 @@ class StorageHistoryAgent(AgentModule): # save the PFN usage per SE in a dictionary self.pfnUsage.setdefault(directory, res["Value"]) - self.log.verbose(f"Get logical usage for directory {directory} ") + self.log.debug(f"Get logical usage for directory {directory} ") # This returns the number of files and size for the directory and all its subdirectories res = self.__stDB.getSummary(directory) self.callsToGetSummary += 1 @@ -633,8 +633,8 @@ class StorageHistoryAgent(AgentModule): for dirInfo in res["Value"].values(): # can be an iterator self.lfnUsage[directory]["LfnSize"] += dirInfo["Size"] self.lfnUsage[directory]["LfnFiles"] += dirInfo["Files"] - self.log.verbose(f"PFN usage: {self.pfnUsage[directory]}") - self.log.verbose(f"LFN usage: {self.lfnUsage[directory]}") + self.log.debug(f"PFN usage: {self.pfnUsage[directory]}") + self.log.debug(f"LFN usage: {self.lfnUsage[directory]}") end = time.time() self.genTotalTime = end - start @@ -657,7 +657,7 @@ class StorageHistoryAgent(AgentModule): self.log.error("Error querying the Bkk:", res["Message"]) else: self.eventTypeDescription.update(dict(res["Value"])) - self.log.verbose(f"Updated self.eventTypeDescription dict: {str(self.eventTypeDescription)} ") + self.log.debug(f"Updated self.eventTypeDescription dict: {str(self.eventTypeDescription)} ") # If still not found, log it! if eventType not in self.eventTypeDescription: self.log.error(f"EventType {str(eventType)} is not in cached dictionary") @@ -676,18 +676,18 @@ class StorageHistoryAgent(AgentModule): dataRecord.setValueByKey("LogicalFiles", logicalFiles) for key in ("DataType", "Activity", "FileType", "Production", "ProcessingPass", "Conditions", "EventType"): dataRecord.setValueByKey(key, metadataDict.get(key, "na")) - self.log.verbose(">>> Send DataStorage record to accounting:") - self.log.verbose("\tlfnFiles: %d lfnSize: %d " % (logicalFiles, logicalSize)) + self.log.debug(">>> Send DataStorage record to accounting:") + self.log.debug("\tlfnFiles: %d lfnSize: %d " % (logicalFiles, logicalSize)) for se in self.pfnUsage[lfnDir]: - self.log.verbose(f"Filling accounting record for se {se}") + self.log.debug(f"Filling accounting record for se {se}") physicalSize = self.pfnUsage[lfnDir][se]["Size"] physicalFiles = self.pfnUsage[lfnDir][se]["Files"] dataRecord.setValueByKey("StorageElement", se) dataRecord.setValueByKey("PhysicalSize", physicalSize) dataRecord.setValueByKey("PhysicalFiles", physicalFiles) - self.log.verbose( + self.log.debug( "\t\tStorageElement: %s --> physFiles: %d physSize: %d " % (se, physicalFiles, physicalSize) )