New gfal2 plugin

SiteTests/MonteCarlo/cms-MC-test/lib/python2.6/site-packages/WMCore/Storage/Backends/GFAL2Impl.py

@@ -4,6 +4,9 @@ _GFAL2Impl_
 Implementation of StageOutImpl interface for gfal-copy
 """
 import os
+import argparse
+import logging
+
 from WMCore.Storage.Registry import registerStageOutImpl
 from WMCore.Storage.StageOutImpl import StageOutImpl
 from WMCore.Storage.Execute import runCommandWithOutput as runCommand
@@ -20,20 +23,43 @@ class GFAL2Impl(StageOutImpl):
 
     def __init__(self, stagein=False):
         StageOutImpl.__init__(self, stagein)
-        self.removeCommand = "env -i X509_USER_PROXY=$X509_USER_PROXY gfal-rm -vvv %s"
-        self.copyCommand = "env -i X509_USER_PROXY=$X509_USER_PROXY gfal-copy -t 2400 -T 2400 -p -vvv"
+        # If we want to execute commands in clean shell, we can`t separate them with ';'.
+        # Next commands after separation are executed without env -i and this leads us with 
+        # mixed environment with COMP and system python.
+        # GFAL2 is not build under COMP environment and it had failures with mixed environment.
+        self.setups = "env -i X509_USER_PROXY=$X509_USER_PROXY JOBSTARTDIR=$JOBSTARTDIR bash -c '%s'"
+        self.removeCommand = self.setups % '. $JOBSTARTDIR/startup_environment.sh 2> /dev/null; date; gfal-rm -t 600 %s '
+        self.copyCommand = self.setups % '. $JOBSTARTDIR/startup_environment.sh 2> /dev/null; date; gfal-copy -t 2400 -T 2400 -p %(checksum)s %(options)s %(source)s %(destination)s'
 
-    def createSourceName(self, protocol, pfn):
+
+    def createFinalPFN(self, pfn):
         """
-        _createSourceName_
-        GFAL2 uses file:/// urls
+        _createFinalPFN_
+        GFAL2 requires file:/// for any direction transfers
         """
         if pfn.startswith('file:'):
             return pfn
         elif os.path.isfile(pfn):
             return "file://%s" % os.path.abspath(pfn)
-        else:
-            return pfn
+        elif pfn.startswith('/'):
+            return "file://%s" % os.path.abspath(pfn)
+        return pfn
+
+
+    def createSourceName(self, protocol, pfn):
+        """
+        _createSourceName_
+        GFAL2 uses file:/// urls
+        """
+        return self.createFinalPFN(pfn)
+
+
+    def createTargetName(self, protocol, pfn):
+        """
+        _createTargetName_
+        GFAL2 uses file:// urls
+        """
+        return self.createFinalPFN(pfn)
 
 
     def createOutputDirectory(self, targetPFN):
@@ -47,33 +73,56 @@ class GFAL2Impl(StageOutImpl):
 
     def createRemoveFileCommand(self, pfn):
         """
+        _createRemoveFileCommand_
         handle file remove using gfal-rm
+
+        gfal-rm options used:
+          -t   global timeout for the execution of the command.
+               Command is interrupted if time expires before it finishes
         """
-        if pfn.startswith("file:"):
-            return self.removeCommand % pfn
-        elif os.path.isfile(pfn):
+        if os.path.isfile(pfn):
             return "/bin/rm -f %s" % os.path.abspath(pfn)
         else:
-            return self.removeCommand % pfn
+            return self.removeCommand % self.createFinalPFN(pfn)
 
 
     def createStageOutCommand(self, sourcePFN, targetPFN, options=None, checksums=None):
         """
         _createStageOutCommand_
-        Build an gfal-copy command
+        Build a gfal-copy command
+
+        gfal-copy options used:
+          -t   maximum time for the operation to terminate
+          -T   global timeout for the transfer operation
+          -p   if the destination directory does not exist, create it
+          -K   checksum algorithm to use, or algorithm:value
         """
-        result = "#!/bin/sh\n"
+        result = "#!/bin/bash\n"
+
+        copyCommandDict = {'checksum': '', 'options': '', 'source': '', 'destination': ''}
+
+        useChecksum = (checksums is not None and 'adler32' in checksums and not self.stageIn)
+
+        if not options:
+            options = ''
+
+        parser = argparse.ArgumentParser()
+        parser.add_argument('--nochecksum', action='store_true')
+        args, unknown = parser.parse_known_args(options.split())
+
+        if not args.nochecksum:
+            if useChecksum:
+                checksums['adler32'] = "%08x" % int(checksums['adler32'], 16)
+                copyCommandDict['checksum'] = "-K adler32:%s" % checksums['adler32']
+            else:
+                copyCommandDict['checksum'] = "-K adler32"
 
-        useChecksum = (checksums != None and 'adler32' in checksums and not self.stageIn)
+        copyCommandDict['options'] = ' '.join(unknown)
 
-        copyCommand = self.copyCommand
-        if useChecksum:
-            copyCommand += " -K adler32 "
-        if options != None:
-            copyCommand += " %s " % options
-        copyCommand += " %s " % sourcePFN
-        copyCommand += " %s \n" % targetPFN
+        copyCommandDict['source'] = self.createFinalPFN(sourcePFN)
+        copyCommandDict['destination'] = self.createFinalPFN(targetPFN)
 
+        copyCommand = self.copyCommand % copyCommandDict
         result += copyCommand
 
         if _CheckExitCodeOption:
@@ -97,13 +146,10 @@ class GFAL2Impl(StageOutImpl):
         _removeFile_
         CleanUp pfn provided
         """
-        command = ""
         if os.path.isfile(pfnToRemove):
             command = "/bin/rm -f %s" % os.path.abspath(pfnToRemove)
-        if pfnToRemove.startswith("file:"):
-            command = self.removeCommand % pfnToRemove
         else:
-            command = self.removeCommand % pfnToRemove
+            command = self.removeCommand % self.createFinalPFN(pfnToRemove)
         self.executeCommand(command)
 
 

SiteTests/MonteCarlo/tests/CE-cms-mc

@@ -60,8 +60,10 @@ fi
 
 export PYTHONPATH=$SAME_SENSOR_HOME/cms-MC-test/lib/python2.6/site-packages:$PYTHONPATH
 
-#echo "PYTHONPATH=$PYTHONPATH"
-#echo
+set > startup_environment.sh
+sed -e 's/^/export /' startup_environment.sh > tmp_env.sh
+mv -f tmp_env.sh startup_environment.sh
+export JOBSTARTDIR=$PWD
 
 cd $SAME_SENSOR_HOME/cms-MC-test/
 if [ $? -ne 0 ]; then

nagios/grid-monitoring-probes-org.cms-etf.spec

@@ -4,7 +4,7 @@
 
 Summary: WLCG Compliant Probes from %{site}
 Name: nagios-plugins-wlcg-org.cms
-Version: 1.1.37
+Version: 1.1.39
 Release: 1%{?dist}
 
 License: GPL
@@ -49,6 +49,10 @@ install --directory %{buildroot}/etc/cron.d
 /etc/cron.d/cms_glexec
 
 %changelog
+* Tue Feb 28 2017 Andrea Sciaba <Andrea.Sciaba@cern.ch> 1.1.39-1.
+- included the latest version of the GFAL2 plugin for WMCore
+* Tue Feb 14 2017 Andrea Sciaba <Andrea.Sciaba@cern.ch> 1.1.38-1.
+- fixed spacetoken bug in SRM test
 * Wed Feb 1 2017 Andrea Sciaba <Andrea.Sciaba@cern.ch> 1.1.37-1.
 - new etf_plugin_cms.py to support new VO feed
 * Thu Jan 26 2017 Andrea Sciaba <Andrea.Sciaba@cern.ch> 1.1.36-1.