Commit d7fe842f authored by Andrea Sciaba's avatar Andrea Sciaba
Browse files

New gfal2 plugin

parent ffa83188
......@@ -4,6 +4,9 @@ _GFAL2Impl_
Implementation of StageOutImpl interface for gfal-copy
"""
import os
import argparse
import logging
from WMCore.Storage.Registry import registerStageOutImpl
from WMCore.Storage.StageOutImpl import StageOutImpl
from WMCore.Storage.Execute import runCommandWithOutput as runCommand
......@@ -20,20 +23,43 @@ class GFAL2Impl(StageOutImpl):
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self.removeCommand = "env -i X509_USER_PROXY=$X509_USER_PROXY gfal-rm -vvv %s"
self.copyCommand = "env -i X509_USER_PROXY=$X509_USER_PROXY gfal-copy -t 2400 -T 2400 -p -vvv"
# If we want to execute commands in clean shell, we can`t separate them with ';'.
# Next commands after separation are executed without env -i and this leads us with
# mixed environment with COMP and system python.
# GFAL2 is not build under COMP environment and it had failures with mixed environment.
self.setups = "env -i X509_USER_PROXY=$X509_USER_PROXY JOBSTARTDIR=$JOBSTARTDIR bash -c '%s'"
self.removeCommand = self.setups % '. $JOBSTARTDIR/startup_environment.sh 2> /dev/null; date; gfal-rm -t 600 %s '
self.copyCommand = self.setups % '. $JOBSTARTDIR/startup_environment.sh 2> /dev/null; date; gfal-copy -t 2400 -T 2400 -p %(checksum)s %(options)s %(source)s %(destination)s'
def createSourceName(self, protocol, pfn):
def createFinalPFN(self, pfn):
"""
_createSourceName_
GFAL2 uses file:/// urls
_createFinalPFN_
GFAL2 requires file:/// for any direction transfers
"""
if pfn.startswith('file:'):
return pfn
elif os.path.isfile(pfn):
return "file://%s" % os.path.abspath(pfn)
else:
return pfn
elif pfn.startswith('/'):
return "file://%s" % os.path.abspath(pfn)
return pfn
def createSourceName(self, protocol, pfn):
"""
_createSourceName_
GFAL2 uses file:/// urls
"""
return self.createFinalPFN(pfn)
def createTargetName(self, protocol, pfn):
"""
_createTargetName_
GFAL2 uses file:// urls
"""
return self.createFinalPFN(pfn)
def createOutputDirectory(self, targetPFN):
......@@ -47,33 +73,56 @@ class GFAL2Impl(StageOutImpl):
def createRemoveFileCommand(self, pfn):
"""
_createRemoveFileCommand_
handle file remove using gfal-rm
gfal-rm options used:
-t global timeout for the execution of the command.
Command is interrupted if time expires before it finishes
"""
if pfn.startswith("file:"):
return self.removeCommand % pfn
elif os.path.isfile(pfn):
if os.path.isfile(pfn):
return "/bin/rm -f %s" % os.path.abspath(pfn)
else:
return self.removeCommand % pfn
return self.removeCommand % self.createFinalPFN(pfn)
def createStageOutCommand(self, sourcePFN, targetPFN, options=None, checksums=None):
"""
_createStageOutCommand_
Build an gfal-copy command
Build a gfal-copy command
gfal-copy options used:
-t maximum time for the operation to terminate
-T global timeout for the transfer operation
-p if the destination directory does not exist, create it
-K checksum algorithm to use, or algorithm:value
"""
result = "#!/bin/sh\n"
result = "#!/bin/bash\n"
copyCommandDict = {'checksum': '', 'options': '', 'source': '', 'destination': ''}
useChecksum = (checksums is not None and 'adler32' in checksums and not self.stageIn)
if not options:
options = ''
parser = argparse.ArgumentParser()
parser.add_argument('--nochecksum', action='store_true')
args, unknown = parser.parse_known_args(options.split())
if not args.nochecksum:
if useChecksum:
checksums['adler32'] = "%08x" % int(checksums['adler32'], 16)
copyCommandDict['checksum'] = "-K adler32:%s" % checksums['adler32']
else:
copyCommandDict['checksum'] = "-K adler32"
useChecksum = (checksums != None and 'adler32' in checksums and not self.stageIn)
copyCommandDict['options'] = ' '.join(unknown)
copyCommand = self.copyCommand
if useChecksum:
copyCommand += " -K adler32 "
if options != None:
copyCommand += " %s " % options
copyCommand += " %s " % sourcePFN
copyCommand += " %s \n" % targetPFN
copyCommandDict['source'] = self.createFinalPFN(sourcePFN)
copyCommandDict['destination'] = self.createFinalPFN(targetPFN)
copyCommand = self.copyCommand % copyCommandDict
result += copyCommand
if _CheckExitCodeOption:
......@@ -97,13 +146,10 @@ class GFAL2Impl(StageOutImpl):
_removeFile_
CleanUp pfn provided
"""
command = ""
if os.path.isfile(pfnToRemove):
command = "/bin/rm -f %s" % os.path.abspath(pfnToRemove)
if pfnToRemove.startswith("file:"):
command = self.removeCommand % pfnToRemove
else:
command = self.removeCommand % pfnToRemove
command = self.removeCommand % self.createFinalPFN(pfnToRemove)
self.executeCommand(command)
......
......@@ -60,8 +60,10 @@ fi
export PYTHONPATH=$SAME_SENSOR_HOME/cms-MC-test/lib/python2.6/site-packages:$PYTHONPATH
#echo "PYTHONPATH=$PYTHONPATH"
#echo
set > startup_environment.sh
sed -e 's/^/export /' startup_environment.sh > tmp_env.sh
mv -f tmp_env.sh startup_environment.sh
export JOBSTARTDIR=$PWD
cd $SAME_SENSOR_HOME/cms-MC-test/
if [ $? -ne 0 ]; then
......
......@@ -4,7 +4,7 @@
Summary: WLCG Compliant Probes from %{site}
Name: nagios-plugins-wlcg-org.cms
Version: 1.1.37
Version: 1.1.39
Release: 1%{?dist}
License: GPL
......@@ -49,6 +49,10 @@ install --directory %{buildroot}/etc/cron.d
/etc/cron.d/cms_glexec
%changelog
* Tue Feb 28 2017 Andrea Sciaba <Andrea.Sciaba@cern.ch> 1.1.39-1.
- included the latest version of the GFAL2 plugin for WMCore
* Tue Feb 14 2017 Andrea Sciaba <Andrea.Sciaba@cern.ch> 1.1.38-1.
- fixed spacetoken bug in SRM test
* Wed Feb 1 2017 Andrea Sciaba <Andrea.Sciaba@cern.ch> 1.1.37-1.
- new etf_plugin_cms.py to support new VO feed
* Thu Jan 26 2017 Andrea Sciaba <Andrea.Sciaba@cern.ch> 1.1.36-1.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment