diff --git a/Script/Conf.py b/Script/Conf.py index 7e5888181eee93ffcbc477ba0b55f0a1c72f13e6..8f4b337a431146f6c82d10e5e435d0ae97605b55 100755 --- a/Script/Conf.py +++ b/Script/Conf.py @@ -167,7 +167,7 @@ class Conf: self.MergedNameModule = cfg.MergedNameModule # Perform checksum check - self.checksumCheck = cfg.checksumCheck + self.deletionChecksumCheck = cfg.checksumCheck # Max total filesystem usage allowed while waiting for migration (in %) self.highCriticMark = cfg.highCriticMark diff --git a/Script/DeleteThread.py b/Script/DeleteThread.py index f1280542f508bd5869202fdab9442794b5571af3..c459bab6ee920718f5deb8b1f84d866c7731db8c 100755 --- a/Script/DeleteThread.py +++ b/Script/DeleteThread.py @@ -12,7 +12,7 @@ import logging from Constants import * from operator import itemgetter, add, contains from functools import partial -from utils import set_log_level +from utils import set_log_level, castorChecksum class DeleteThread(threading.Thread): @@ -33,7 +33,7 @@ class DeleteThread(threading.Thread): self.nFiles = self.conf.nFiles self.minFileTime = self.conf.minFileTime self.migFlag = self.conf.migFlag - self.checksumCheck = self.conf.checksumCheck + self.checksumCheck = self.conf.deletionChecksumCheck self.highCriticMark = self.conf.highCriticMark self.lowCriticMark = self.conf.lowCriticMark self.lowWaterMark = self.conf.lowWaterMark @@ -228,7 +228,8 @@ class DeleteThread(threading.Thread): checksumok = True if self.db and self.checksumCheck and (not isMerged): - checksum = self.castorChecksum(Castor_file, castorEnv) + checksum = castorChecksum(Castor_file, \ + castorEnv, self.logger) self.dbLock.acquire() dbchecksum, filehealth = self.db.CheckSum(deletefile[0]) @@ -334,34 +335,7 @@ class DeleteThread(threading.Thread): return isMig,isMerged - - def castorChecksum(self, castorfile, castorenv): - ##### Check if file has already been migrated ##### - self.logInfo = {'file':self.logger.findCaller()[0], - 'line':self.logger.findCaller()[1]} - self.logger.debug('Fetching checksum from castor for: ' - + castorfile, - extra = self.logInfo) - nsls = Popen(['nsls', '-T', '--checksum',castorfile], - stdout = PIPE, stderr = STDOUT, env= castorenv) - ret = nsls.wait() - nslsOut = nsls.stdout.read() - self.logInfo = {'file':self.logger.findCaller()[0], - 'line':self.logger.findCaller()[1]} - self.logger.debug(nslsOut,extra = self.logInfo) - - try: - id = nslsOut.split().index('adler32') - return nslsOut.split()[id+1] - except ValueError: - self.logInfo = {'file':self.logger.findCaller()[0], - 'line':self.logger.findCaller()[1]} - - self.logger.warning('Adler32 checksum not found for ' - + castorfile + '. nsls output: ' - + nslsOut ,extra = self.logInfo) - return None - + ##### Get Castor environment ##### def getCastorEnv(self,pool,stagehost): @@ -460,7 +434,7 @@ class DeleteThread(threading.Thread): self.nFiles = self.conf.nFiles self.minFileTime = self.conf.minFileTime self.migFlag = self.conf.migFlag - self.checksumCheck = self.conf.checksumCheck + self.checksumCheck = self.conf.deletionChecksumCheck self.highCriticMark = self.conf.highCriticMark self.lowCriticMark = self.conf.lowCriticMark self.lowWaterMark = self.conf.lowWaterMark diff --git a/Script/utils.py b/Script/utils.py index 2e181915893be50e863575fb6d3842b98cfa53b1..90d1b21a3e6093d88526868166ef786f20bbe85e 100644 --- a/Script/utils.py +++ b/Script/utils.py @@ -1,5 +1,5 @@ -__version__='$Revision:$' +__version__='$Revision$' import logging @@ -17,3 +17,30 @@ def set_log_level(level, logger): logger.setLevel(map[level]) + + +def castorChecksum(castorfile, castorenv, logger): + + logInfo = {'file':logger.findCaller()[0], + 'line':logger.findCaller()[1]} + logger.debug('Fetching checksum from castor for: ' + + castorfile, extra = logInfo) + nsls = Popen(['nsls', '-l', '--checksum',castorfile], + stdout=PIPE, stderr=STDOUT, env=castorenv) + ret = nsls.wait() + nslsOut = nsls.stdout.read() + logInfo = {'file':logger.findCaller()[0], + 'line':logger.findCaller()[1]} + logger.debug(nslsOut, extra=logInfo) + + try: + id = nslsOut.split().index('AD') + return nslsOut.split()[id+1] + except ValueError: + logInfo = {'file':logger.findCaller()[0], + 'line':logger.findCaller()[1]} + + logger.warning('Adler32 checksum not found for ' + + castorfile + '. nsls output: ' + + nslsOut ,extra=logInfo) + return None