diff --git a/Script/CastorScript.py b/Script/CastorScript.py
index cfd285affbeaec85c1e329d69c13d2024fce7882..3a46daaf892edb583f4c35a2ccf50b72f51af813 100644
--- a/Script/CastorScript.py
+++ b/Script/CastorScript.py
@@ -98,14 +98,6 @@ def main(conf):
logging.getLogger('').addHandler(email_logger)
# end if
- ##### Set root Logger for main thread #####
- mainLog = logging.FileHandler(os.path.join(conf.LogDir,'mainLog.out'),
- 'w')
- mainLog.setFormatter(formatter)
- logger = logging.getLogger('main')
- LogLevel = conf.LogLevel
- set_log_level(LogLevel, mainLog)
- logger.addHandler(mainLog)
# get logger to files, (Will also log to ERS, when enabled)
logger = enableFileLogging("main","mainLog.out", conf.LogDir, conf.LogLevel)
@@ -118,11 +110,9 @@ def main(conf):
from ispy import IPCPartition #pylint: disable=no-name-in-module
IPCPartition = IPCPartition()
#ers.addLoggingHandler("main")
- #TODO rename main_ERS_handler to ers_handler
- main_ERS_handler = ers.LoggingHandler()
- set_log_level(conf.ERSLogLevel, main_ERS_handler)
- #TODO attach ers handler to root logger instead of "main" logger
- logging.getLogger("main").addHandler(main_ERS_handler)
+ ERSHandler = ers.LoggingHandler()
+ set_log_level(conf.ERSLogLevel, ERSHandler)
+ logging.getLogger().addHandler(ERSHandler)
logger.info("CastorScript is now logging to partition '%s'",
conf.partition)
except Exception as ex:
@@ -131,16 +121,11 @@ def main(conf):
##### Start looking for the partition #####
if conf.ERSenabled:
- check = CheckThread(conf, manager, copy, delete, event, main_ERS_handler)
+ check = CheckThread(conf, event, ERSHandler)
##### Create db file logger #####
dblogger = enableFileLogging("Database","DatabaseLog.out",conf.LogDir,conf.LogLevel)
- 'w')
- dbLog.setFormatter(formatter)
- dblogger = logging.getLogger('Database')
- set_log_level(LogLevel,dbLog)
- dblogger.addHandler(dbLog)
#### Extend the PYTHON_PATH ####
diff --git a/Script/cs/Threads/CheckThread.py b/Script/cs/Threads/CheckThread.py
index cb94ff360a00d4762f12468ba11540f7e3b8fa38..c15452a7761c098ef81600b96c0394dba15acb4d 100644
--- a/Script/cs/Threads/CheckThread.py
+++ b/Script/cs/Threads/CheckThread.py
@@ -1,76 +1,40 @@
#!/bin/env python
##### Thread checking constantly the state of the partition #####
-import os
import threading
import logging
-from cs.Tools.utils import set_log_level,formatter,thread_id_string
+from cs.Tools.utils import thread_id_string, enableFileLogging
class CheckThread(threading.Thread):
- def __init__(self, conf, manager, copy, delete, event, mainERSHandler):
+ def __init__(self, conf, event, ERSHandler):
threading.Thread.__init__(self, name="CheckThread")
- import ers
from ispy import IPCPartition #pylint: disable=no-name-in-module
self.IPCPartition = IPCPartition
- self.conf = conf
self.event = event
- self.manager = manager
- self.copy = copy
- self.delete = delete
-
- ##### Main logger #####
- #TODO remove mainLogger (only root logger is needed and there is no need to store it locally)
- self.mainLogger = logging.getLogger('main')
- self.main_ERS_handler = mainERSHandler
+ self.ERSHandler = ERSHandler
##### Get configuration parameters #####
- self.CheckTimeout = self.conf.CheckTimeout
- self.ERSLogLevel = self.conf.ERSLogLevel
- self.LogDir = self.conf.LogDir
- self.ERSenabled = self.conf.ERSenabled
- self.partition = self.conf.partition
- self.LogLevel = self.conf.LogLevel
- self.ERSTimeout = self.conf.ERSTimeout
+ self.CheckTimeout = conf.CheckTimeout
+ self.partition = conf.partition
+ self.ERSTimeout = conf.ERSTimeout
##### Define Thread internal variables #####
self.exitFlag = False
self.partition_connected = True
- ##### Set Logger for CheckThread #####
- self.logFile = os.path.join(self.LogDir,'CheckLog.out')
- # TODO rename all handlers (they are not loggers)
- # TODO there is no need to store them in the object (double check) (make them local variables)
- self.CheckLog = logging.FileHandler(self.logFile,'w')
- self.CheckLog.setFormatter(formatter)
-
- ##### Set Logger Level #####
- set_log_level(self.LogLevel, self.CheckLog)
-
- self.logger = logging.getLogger('CheckThread')
- self.logger.addHandler(self.CheckLog)
-
- ##### Add ERS handler #####
- if self.ERSenabled:
- try:
- #TODO remove: we don't need another ers handler, just use the central one
- self.check_ERS_handler=ers.LoggingHandler()
- set_log_level(self.ERSLogLevel,self.check_ERS_handler)
- self.logger.addHandler( self.check_ERS_handler )
- except Exception,ex:
- self.logger.error("CheckThread could not create ERS, reason is '%s'", str(ex))
+ self.logger = enableFileLogging("CheckThread","CheckLog.out", conf.LogDir, conf.LogLevel)
+
# end def __init__()
def run(self):
- #TODO log thread id to local logger and not mainLogger
- self.mainLogger.info(thread_id_string())
-
+ self.logger.info(thread_id_string())
while True:
@@ -83,45 +47,20 @@ class CheckThread(threading.Thread):
self.event.wait(self.ERSTimeout)
self.logger.warning('Found partition %s, adding ERS', self.partition)
self.partition_connected = True
- ##### Add ERS handlers to all loggers #####
- self.change_state_ers(True)
-
+ ##### Add ERS handler to root logger #####
+ logging.getLogger().addHandler(self.ERSHandler)
##### Check for the disappearance of partition #####
elif self.partition_connected and not self.IPCPartition(self.partition).isValid():
self.logger.warning('Lost connection to partition %s, removing ERS handlers', self.partition)
self.partition_connected = False
- ##### Remove ERS handlers from all loggers #####
- self.change_state_ers(False)
+
+ ##### Remove ERS handler from root logger #####
+ logging.getLogger().removeHandler(self.ERSHandler)
self.event.wait(self.CheckTimeout)
# end while
# end def run()
-
- ##### Call add/remove ERS functions from threads #####
- def change_state_ers(self, create_ers_handler):
- if create_ers_handler:
- ##### Add ERS handler to all loggers #####
- #TODO this should be the root logger
- self.mainLogger.addHandler(self.main_ERS_handler)
- self.logger.addHandler(self.check_ERS_handler)
- self.copy.AddRemoveERS(True)
- self.manager.AddRemoveERS(True)
- if self.delete:
- self.delete.AddRemoveERS(True)
- else:
- ##### Remove ERS from all loggers #####
- #TODO this should be the root logger
- self.mainLogger.removeHandler(self.main_ERS_handler)
- self.logger.removeHandler(self.check_ERS_handler)
- self.copy.AddRemoveERS(False)
- self.manager.AddRemoveERS(False)
- if self.delete:
- self.delete.AddRemoveERS(False)
- # end if,else
- # end def change_state_ers()
-
-
##### Exit handler #####
def checkExit(self):
self.exitFlag = True
diff --git a/Script/cs/Threads/CopyThread.py b/Script/cs/Threads/CopyThread.py
index 62af5edc2223c1136273a99aae2c078d93a0677a..436ca77f7a22b21c0e84ab89b5995c2d49835fc3 100644
--- a/Script/cs/Threads/CopyThread.py
+++ b/Script/cs/Threads/CopyThread.py
@@ -3,12 +3,11 @@
import threading, glob, os
from time import time
import signal
-import logging
import Queue
import uuid
import errno
import cs.Tools.Constants as Constants
-from cs.Tools.utils import set_log_level,formatter,thread_id_string,adler32,get_bw
+from cs.Tools.utils import thread_id_string,adler32,get_bw, enableFileLogging
class CopyThread(threading.Thread):
@@ -34,11 +33,7 @@ class CopyThread(threading.Thread):
self.NoneTimeout = self.conf.NoneTimeout
self.CopyTimeout = self.conf.CopyTimeout
self.bwLimit = self.conf.bwLimit
- self.bwDevice = self.conf.bwDevice
- self.LogDir = self.conf.LogDir
- self.LogLevel = self.conf.LogLevel
- self.ERSLogLevel = self.conf.ERSLogLevel
- self.ERSenabled = self.conf.ERSenabled
+ self.bwDevice = self.conf.bwDevice
self.localChecksum = self.conf.computeChecksum
self.DdmMonitoringEnabled = conf.DdmMonitoringEnabled
@@ -48,25 +43,8 @@ class CopyThread(threading.Thread):
self.dbFlag = True
self.ddmQueueFullSince = None #timestamp indicating since when the queue is full
- ##### Set Logger for CopyThread #####
- self.logFile = os.path.join(self.LogDir,'CopyLog.out')
- self.CopyLog = logging.FileHandler(self.logFile,'w')
- self.CopyLog.setFormatter(formatter)
- set_log_level(self.LogLevel, self.CopyLog)
-
- self.logger = logging.getLogger('CopyThread')
- self.logger.addHandler(self.CopyLog)
-
- ##### Add ERS handler #####
- if self.ERSenabled:
- try:
- import ers
- #ers.addLoggingHandler('CopyThread')
- self.Copy_ERS_handler=ers.LoggingHandler()
- set_log_level(self.ERSLogLevel,self.Copy_ERS_handler)
- logging.getLogger( 'CopyThread' ).addHandler( self.Copy_ERS_handler )
- except Exception,ex:
- self.logger.error("CopyThread could not create ERS, reason is '%s'", str(ex))
+ # get logger to files, (Will also log to ERS)
+ self.logger = enableFileLogging("CopyThread","CopyLog.out", conf.LogDir, conf.LogLevel)
# end def __init__()
diff --git a/Script/cs/Threads/DdmMonitoringThread.py b/Script/cs/Threads/DdmMonitoringThread.py
index fddd376febd5591e0eb52f77fa9eeb92d43d545f..71cab22da7cf416e964ca6f4ea3fd628ff38333c 100644
--- a/Script/cs/Threads/DdmMonitoringThread.py
+++ b/Script/cs/Threads/DdmMonitoringThread.py
@@ -1,13 +1,11 @@
#!/bin/env python
import threading
-import logging
-import os
import urllib2
import json
import Queue
import time
-from cs.Tools.utils import set_log_level, formatter, thread_id_string
+from cs.Tools.utils import thread_id_string, enableFileLogging
class DdmMonitoringThread(threading.Thread):
@@ -22,24 +20,8 @@ class DdmMonitoringThread(threading.Thread):
self.requestErrorSince = None
self.requestErrorCriticalLogged = False
- ##### Set Logger #####
- self.logFile = os.path.join(self.conf.LogDir, 'DdmMonitoring.out')
- self.LogFileHandler = logging.FileHandler(self.logFile, 'w')
- self.LogFileHandler.setFormatter(formatter)
- set_log_level(self.conf.LogLevel, self.LogFileHandler)
-
- self.logger = logging.getLogger('DdmMonitoring')
- self.logger.addHandler(self.LogFileHandler)
-
- self.LogERSHandler = None
- if self.conf.ERSenabled:
- try:
- import ers
- self.LogERSHandler = ers.LoggingHandler()
- set_log_level(self.conf.ERSLogLevel, self.LogERSHandler)
- self.logger.addHandler(self.LogERSHandler)
- except Exception as ex:
- self.logger.error("DdmMonitoringThread could not create ERS, reason is '%s'", str(ex))
+ # get logger to files, (Will also log to ERS)
+ self.logger = enableFileLogging("DdmMonitoring", "DdmMonitoring.out", conf.LogDir, conf.LogLevel)
def run(self):
diff --git a/Script/cs/Threads/DeleteThread.py b/Script/cs/Threads/DeleteThread.py
index f944a115dea9d96d1c1400e31ccdbb9843332281..ef0806cf4ce4a0830ec5f3723f6d99d7de355996 100644
--- a/Script/cs/Threads/DeleteThread.py
+++ b/Script/cs/Threads/DeleteThread.py
@@ -6,10 +6,9 @@ import glob
import errno
from time import time
import operator
-import logging
from itertools import imap, chain
import cs.Tools.Constants as Constants
-from cs.Tools.utils import set_log_level,formatter,thread_id_string
+from cs.Tools.utils import thread_id_string, enableFileLogging
class DeleteThread(threading.Thread):
@@ -36,10 +35,6 @@ class DeleteThread(threading.Thread):
self.DirList = [os.path.normpath(k) for k in self.conf.DirList]
self.DeleteTimeout = self.conf.DeleteTimeout
self.MigDelay = self.conf.MigDelay
- self.LogDir = self.conf.LogDir
- self.LogLevel = self.conf.LogLevel
- self.ERSLogLevel = self.conf.ERSLogLevel
- self.ERSenabled = self.conf.ERSenabled
#### Import the function to resolve the merged file names #####
self.mergedChecker = None
@@ -53,27 +48,8 @@ class DeleteThread(threading.Thread):
self.dbFlag = True
self.deleteNow = False
- ##### Set Logger for DeleteThread #####
- self.logFile = os.path.join(self.LogDir,'DeleteLog.out')
- self.DeleteLog = logging.FileHandler(self.logFile,'w')
- self.DeleteLog.setFormatter(formatter)
- set_log_level(self.LogLevel, self.DeleteLog)
-
- self.logger = logging.getLogger('DeleteThread')
- self.logger.addHandler(self.DeleteLog)
-
- ##### Add ERS handler #########
- if self.ERSenabled:
- try:
- import ers
- #ers.addLoggingHandler('DeleteThread')
- self.Del_ERS_handler=ers.LoggingHandler()
- set_log_level(self.ERSLogLevel,self.Del_ERS_handler)
- logging.getLogger('DeleteThread').addHandler( self.Del_ERS_handler )
- except Exception,ex:
- self.logger.error("DeleteThread could not create ERS, reason is '%s'", str(ex))
-
-
+ # get logger to files, (Will also log to ERS)
+ self.logger = enableFileLogging("DeleteThread","DeleteLog.out", conf.LogDir, conf.LogLevel)
# end def __init__()
diff --git a/Script/cs/Threads/ManagerThread.py b/Script/cs/Threads/ManagerThread.py
index e2153c0f9c2582cb5391366389579f42be55a0fd..a9eef28355adb0bdae0d970f397f5ff7472b2159 100644
--- a/Script/cs/Threads/ManagerThread.py
+++ b/Script/cs/Threads/ManagerThread.py
@@ -7,14 +7,13 @@ It is in charge of ls on SFO disk.
import threading, os, glob
from time import time
-import logging
import datetime
import pprint
import math
import cs.Tools.Constants as Constants
import errno
import re
-from cs.Tools.utils import set_log_level,formatter,thread_id_string
+from cs.Tools.utils import thread_id_string, enableFileLogging
from itertools import chain
class ManagerThread(threading.Thread):
@@ -44,11 +43,7 @@ class ManagerThread(threading.Thread):
self.CopyDir = self.conf.CopyDir
self.Pool = self.conf.Pool
self.StageHost = self.conf.StageHost
- self.drivenPool = self.conf.drivenPool
- self.LogDir = self.conf.LogDir
- self.LogLevel = self.conf.LogLevel
- self.ERSLogLevel = self.conf.ERSLogLevel
- self.ERSenabled = self.conf.ERSenabled
+ self.drivenPool = self.conf.drivenPool
self.CopyFileList = [] # all files currently handled by CastorScript
self.ProblDict = {} # files declared problematic with extra info to manage retries
@@ -56,24 +51,8 @@ class ManagerThread(threading.Thread):
self.Year = ''
self.Month = ''
- # Set Logger for ManagerThread
- self.logFile = os.path.join(self.LogDir,'ManagerLog.out')
- self.ManagerLog = logging.FileHandler(self.logFile,'w')
- self.ManagerLog.setFormatter(formatter)
- set_log_level(self.LogLevel, self.ManagerLog)
-
- self.logger = logging.getLogger('ManagerThread')
- self.logger.addHandler(self.ManagerLog)
-
- if self.ERSenabled:
- try:
- import ers
- #ers.addLoggingHandler('ManagerThread')
- self.manage_ERS_handler=ers.LoggingHandler()
- set_log_level(self.ERSLogLevel,self.manage_ERS_handler)
- logging.getLogger('ManagerThread').addHandler( self.manage_ERS_handler )
- except Exception,ex:
- self.logger.error("ManagerThread could not create ERS, reason is '%s'", str(ex))
+ # get logger to files, (Will also log to ERS)
+ self.logger = enableFileLogging("ManagerThread","ManagerLog.out", conf.LogDir, conf.LogLevel)
def initialize(self):