From 7926de93d38c9032e7a7ac5663e36c77f4c8f354 Mon Sep 17 00:00:00 2001
From: Fabrice Le Goff <fabrice.le.goff@cern.ch>
Date: Mon, 27 Aug 2018 14:01:42 +0200
Subject: [PATCH] check small files only if not already handled; might be being
deleted otherwise
---
Script/ManagerThread.py | 63 ++++++++++++++++++-----------------------
1 file changed, 28 insertions(+), 35 deletions(-)
diff --git a/Script/ManagerThread.py b/Script/ManagerThread.py
index a93ca1f..59e98c0 100644
--- a/Script/ManagerThread.py
+++ b/Script/ManagerThread.py
@@ -50,8 +50,8 @@ class ManagerThread(threading.Thread):
self.ERSLogLevel = self.conf.ERSLogLevel
self.ERSenabled = self.conf.ERSenabled
- self.CopyFileList = []
- self.ProblDict = {}
+ self.CopyFileList = [] # all files currently handled by CastorScript
+ self.ProblDict = {} # files declared problematic with extra info to manage retries
self.exitFlag = False
self.Year = ''
self.Month = ''
@@ -209,43 +209,40 @@ class ManagerThread(threading.Thread):
for filename in files:
- ### Protection against bad pattern definition in the
- ### previous glob
+ # Protection against bad pattern definition in the previous glob
if os.path.splitext(filename)[1] in (
Constants.tobecopied_ext, Constants.copied_ext,
Constants.copying_ext, Constants.problematic_ext):
continue
- # end if
-
- ### Check for minimal file size, if needed
- if self.MinSizekB:
- try:
- filesize = os.path.getsize(filename)
- except OSError as exc:
- if exc.errno == errno.ENOENT:
- # The file has been removed by an external cause.
- # This is bad but this is not a reason to crash.
- self.logger.critical('error getting file size for %s: no such file', filename)
- continue
- else:
- raise exc
-
- if filesize/1024.< self.MinSizekB:
- self.logger.debug('File %s is too small. Skipping.', filename)
- if self.RemoveSmallFiles:
- self.logger.debug('File %s is too small. Deleting.', filename)
- try:
- os.remove(filename)
- except OSError, ex:
- self.logger.warning('Cannot delete %s. ERROR: %s', filename, ex.__repr__())
-
- continue
-
- ##### Do not take already listed files #####
+ # Do not take already listed files
if not filename in self.CopyFileList:
self.logger.debug('New file: ' + filename)
+ # Check for minimal file size, if needed
+ if self.MinSizekB:
+ try:
+ filesize = os.path.getsize(filename)
+ except OSError as exc:
+ if exc.errno == errno.ENOENT:
+ # The file has been removed by an external cause.
+ # This is bad but this is not a reason to crash.
+ self.logger.critical('error getting file size for %s: no such file', filename)
+ continue
+ else:
+ raise exc
+
+ if filesize/1024.< self.MinSizekB:
+ self.logger.debug('File %s is too small. Skipping.', filename)
+ if self.RemoveSmallFiles:
+ self.logger.debug('File %s is too small. Deleting.', filename)
+ try:
+ os.remove(filename)
+ except OSError, ex:
+ self.logger.warning('Cannot delete %s. ERROR: %s', filename, ex.__repr__())
+
+ continue
+
##### Check if .PROBLEMATIC file is ready to be copied #####
if glob.glob(filename + Constants.problematic_ext) \
and not self.checkProbl(filename):
@@ -269,10 +266,6 @@ class ManagerThread(threading.Thread):
else:
self.logger.debug('File ' + filename + ' already in the internal copy list')
- # end if
- # end for
- # end if,else
- # end for
##### Sort the files with priority to express streams, then to oldest ones #####
self.logger.debug('Sort new files, with priority to express streams, then to oldest ones')
--
GitLab