From 5c584176be64b8ce51489641cf0ad8a80c1ac0b5 Mon Sep 17 00:00:00 2001
From: Wainer Vandelli <Wainer.Vandelli@cern.ch>
Date: Tue, 27 Apr 2010 17:14:50 +0000
Subject: [PATCH] Initial implementation of size && checksum checks during copy
and deletion
---
Script/CopyThread.py | 33 ++++++++++++---------
Script/Database.py | 7 ++---
Script/DeleteThread.py | 66 ++++++++++++++----------------------------
3 files changed, 45 insertions(+), 61 deletions(-)
diff --git a/Script/CopyThread.py b/Script/CopyThread.py
index 2591abe..80fac7e 100755
--- a/Script/CopyThread.py
+++ b/Script/CopyThread.py
@@ -11,7 +11,7 @@ from subprocess import *
import signal
import logging
from Constants import *
-from utils import set_log_level
+from utils import set_log_level,castorinfo
class CopyThread(threading.Thread):
@@ -180,19 +180,26 @@ class CopyThread(threading.Thread):
##### Compare size of original (SFO) and copied (Castor) files #####
SFO_filesize = os.stat(DataFile)[6]
+
+ Castor_filesize, checksum = \
+ castorinfo(CastorFile, castorEnv, \
+ self.logger)
+
+ success = (SFO_filesize == Castor_filesize)
+
- #SFO_filesize = os.path.getsize(DataFile)
- nsls = Popen(['nsls', '-l',CastorFile], stdout=PIPE, \
- stderr=STDOUT, env=castorEnv)
- w = nsls.wait()
- nslsOut = nsls.stdout.read()
- Castor_filesize = 0
- if w==0: Castor_filesize = long(filter(lambda x: x!='',nslsOut.split(' '))[4])
- self.logInfo = {'file':self.logger.findCaller()[0],'line':self.logger.findCaller()[1]}
- self.logger.debug(nslsOut,extra = self.logInfo)
- # end if
+ ### If DB is available, compare also checksum and db filesize
+ if self.db:
+ self.dbLock.acquire()
+ dbchecksum,dbfilesize,filehealth = \
+ self.db.FileInfo(DataFile)
+ self.dbLock.release()
+
+ if not filehealth == 'TRUNCATED':
+ success = success and (Castor_filesize == dbfilesize)
+ success = success and (dbchecksum.lower() == checksum.lower())
- if SFO_filesize == Castor_filesize:
+ if success:
self.logInfo = {'file':self.logger.findCaller()[0],'line':self.logger.findCaller()[1]}
self.logger.info('Copy of file ' + DataFile + ' was successful and file sizes match',extra = self.logInfo)
@@ -225,7 +232,7 @@ class CopyThread(threading.Thread):
else:
self.logInfo = {'file':self.logger.findCaller()[0],'line':self.logger.findCaller()[1]}
- self.logger.warning('Copy of file ' + DataFile + ' was successful BUT file sizes do NOT match',extra = self.logInfo)
+ self.logger.warning('Copy of file ' + DataFile + ' was successful. Either size or checksum check failed',extra = self.logInfo)
if self.exitFlag: self.CopyList.remove(element)
else: self.handleUnsuccCopy(element)
diff --git a/Script/Database.py b/Script/Database.py
index a5e8b9c..f92c62c 100755
--- a/Script/Database.py
+++ b/Script/Database.py
@@ -64,9 +64,9 @@ class Database:
logInfo = {'file':self.logger.findCaller()[0],'line':self.logger.findCaller()[1]}
self.logger.info("DB connection refreshed.",extra = logInfo)
- def CheckSum(self, sfofile):
+ def FileInfo(self, sfofile):
- sql="select checksum,filehealth from " + self.file_table + " "
+ sql="select checksum,filesize,filehealth from " + self.file_table + " "
sql+="where sfopfn = :ssfopfn"
args=[]
keys={}
@@ -76,7 +76,7 @@ class Database:
logInfo = {'file':self.logger.findCaller()[0],'line':self.logger.findCaller()[1]}
try:
self.db.execute(args,keys)
- result = self.db.getNextRow()[:2]
+ result = self.db.getNextRow()[:3]
except (cx_Oracle.InterfaceError,cx_Oracle.DatabaseError),ex:
ex_info = ex.message
self.logger.error(str(ex_info),extra = logInfo)
@@ -90,7 +90,6 @@ class Database:
return result
-
def Deletion(self,sfofile):
try:
diff --git a/Script/DeleteThread.py b/Script/DeleteThread.py
index c459bab..aa05e83 100755
--- a/Script/DeleteThread.py
+++ b/Script/DeleteThread.py
@@ -12,7 +12,7 @@ import logging
from Constants import *
from operator import itemgetter, add, contains
from functools import partial
-from utils import set_log_level, castorChecksum
+from utils import set_log_level, castorinfo
class DeleteThread(threading.Thread):
@@ -196,7 +196,6 @@ class DeleteThread(threading.Thread):
continue
# end if
- # TO BE CHANGED FOR T0 RAW MERGING
self.logInfo = {'file':self.logger.findCaller()[0],'line':self.logger.findCaller()[1]}
self.logger.debug('Migration required by the deletion policy',extra = self.logInfo)
@@ -225,56 +224,35 @@ class DeleteThread(threading.Thread):
### For merged file the checksum check must be skipped
### since original files are not migrated
- checksumok = True
+ success = True
- if self.db and self.checksumCheck and (not isMerged):
- checksum = castorChecksum(Castor_file, \
- castorEnv, self.logger)
-
+
+ castorfilesize, checksum = \
+ castorinfo(Castor_file, castorEnv, \
+ self.logger)
+
+ sfofilesize = os.stat(deletefile[0])[6]
+
+ success = (sfofilesize == castorfilesize)
+
+ if self.db and (not isMerged):
+
self.dbLock.acquire()
- dbchecksum, filehealth = self.db.CheckSum(deletefile[0])
+ dbchecksum,dbfilesize,filehealth = \
+ self.db.FileInfo(deletefile[0])
self.dbLock.release()
-
- if checksum and dbchecksum and \
- checksum.lower() == dbchecksum.lower():
- checksumok = True
- self.logInfo = {'file':self.logger.findCaller()[0],
- 'line':self.logger.findCaller()[1]}
-
- self.logger.debug('Checksum check is ok.',
- extra = self.logInfo)
- elif not dbchecksum and filehealth == 'TRUNCATED':
- #TRUNCATED files do not have the checksum
- checksumok = True
- self.logInfo = {'file':self.logger.findCaller()[0],
- 'line':self.logger.findCaller()[1]}
-
- self.logger.warning('DB Checksumk for file %s is \
- missing. The FILEHEALTH state is %s' \
- % (deletefile[0], str(filehealth)),
- extra = self.logInfo)
- else:
- self.logInfo = {'file':self.logger.findCaller()[0],
- 'line':self.logger.findCaller()[1]}
-
- self.logger.critical('Checksum check failed.'
- + 'File: ' + deletefile[0] + '.'
- + 'Castor checksum: '
- + str(checksum)
- + '.'
- + 'DB checksum: '
- + str(dbchecksum),
- extra = self.logInfo)
- checksumok = False
-
-
- if checksumok:
+
+ if not filehealth == 'TRUNCATED':
+ success = success and (castorfilesize == dbfilesize)
+ success = success and (dbchecksum.lower() == checksum.lower())
+
+ if success:
self.delete(deletefile)
else:
self.logInfo = {'file':self.logger.findCaller()[0],
'line':self.logger.findCaller()[1]}
- self.logger.warning('Checksum check failed for: '
+ self.logger.warning('Checksum check or size check failed for: '
+ deletefile[0] + '. Not deleting',
extra = self.logInfo)
--
GitLab