Commit bd43349e authored by Nicolo Magini's avatar Nicolo Magini
Browse files

Modified SRMv2 tests to use SpaceToken published in PhEDEx TFC

parent 631fde04
#!/usr/bin/python
import sys
import os
import urllib2
import time
import re
import simplejson
# Read SAM environment
workdir = os.environ.get("SAME_SENSOR_WORK")
sensordir = os.environ.get("SAME_SENSOR_HOME")
sensorname = os.environ.get("SAME_SENSOR_NAME")
samehome = os.environ.get("SAME_HOME")
#URLs for PhEDEx DataService for TFC
tfcURL="http://cmsweb.cern.ch/phedex/datasvc/json/prod/lfn2pfn?node="
pfnMatchURL="&lfn="
pfnProtocolOption = "&protocol=srmv2"
destinationOption = "&destination="
custodialOption = "&custodial="
#Path of text file with list of SRM endpoints
seMapFileURL="http://cern.ch/magini/phedex-v2-endpoints.txt"
# get nodeName to test from environment
nodeName = sys.argv[1]
nodeWorkDir = workdir + "/nodes/" + nodeName
# Define LFN path for file to test transfers
# If LFN is not provided as second argument of the script, the default /store/unmerged/SAM/testSRM is used
try:
testLFN = sys.argv[2]
except IndexError:
testLFN = "/store/unmerged/SAM/testSRM"
print "LFN path of test file is " + testLFN
print
# Define filename for file used to store PFN for successive tests
# If filename is not provided as third argument of the script, the defaults pfnpath.txt & pfnlist.txt are used
try:
basePfnFilename= sys.argv[3]
basePfnListFilename= "list_"+sys.argv[3]
except:
basePfnFilename="pfnpath.txt"
basePfnListFilename="pfnlist.txt"
pfnFilename = nodeWorkDir + "/" + basePfnFilename
pfnListFile= nodeWorkDir + "/" + basePfnListFilename
print "Retrieving list of endpoints to test at URL:"
print seMapFileURL
print
try:
siteNameFile=urllib2.urlopen(seMapFileURL)
for siteLine in siteNameFile.readlines():
endpointName=siteLine.split()[0]
siteName=siteLine.split()[1]
if endpointName == nodeName:
print nodeName+" is listed as SRM for Site "+siteName
# Contact web service to get PFN and spacetoken for non-custodial transfers
endpointLFN = "/SAM-%s" % endpointName
# Testing only non-custodial area - don't want to clutter custodial area with small files at T1s
for custodiality in ("n"):
pfnUrl = tfcURL+siteName+pfnMatchURL+testLFN+endpointLFN+pfnProtocolOption+destinationOption+siteName+custodialOption+custodiality
print "Setting custodiality flag="+custodiality
print "Contacting webservice to perform LFN-to-PFN matching at URL:"
print pfnUrl
print
try:
pfnFile=urllib2.urlopen(pfnUrl)
pfnJSON = simplejson.load(pfnFile)
pfn = (((pfnJSON[u'phedex'])[u'mapping'])[0])[u'pfn']
spacetoken = (((pfnJSON[u'phedex'])[u'mapping'])[0])[u'space_token']
if pfn == None:
print "LFN did not match to any PFN - probably the TFC does not contain any rule for the protocol "+sensorname.lower()
else:
print "LFN was matched to PFN "+pfn
if spacetoken:
print "In space token "+spacetoken+" for custodiality="+custodiality
print
else:
print "No space token defined for custodiality="+custodiality
print
if not re.compile("^srm://.+:[0-9]+/.+\?SFN=.+$").match(pfn):
print "WARNING: Invalid matching to srm protocol"
print "Note: this test currently supports only PFNs in the full format:"
print "srm://hostname:port/srmendpointpath?SFN=sitefilename"
print
else:
if re.compile("^srm://.+srm/managerv1.+$").match(pfn):
print "WARNING: matching to srmv1 protocol"
elif re.compile("^srm://.+srm/managerv2.+$").match(pfn) or re.compile("^srm://.+srm/v2/server.+$").match(pfn):
pfntonode=re.sub(":.+$","",re.sub("^srm://","",pfn))
if pfntonode!=nodeName :
print "WARNING: the resulting PFN matches to SRM "+pfntonode+" instead of SRM "+nodeName
print
else:
fou=open(pfnListFile,"a")
if spacetoken:
fou.write(pfn+" 1 "+spacetoken + "\n")
else:
fou.write(pfn+" 0 notoken"+ "\n")
fou.close()
else:
print "WARNING: srm version not specified in PFN"
except urllib2.URLError:
print "WARNING: Unable to open TFC webservice URL to perform LFN-to-PFN matching for Site "+siteName
print
except ValueError:
print "WARNING: Site "+siteName+" is unknown to TFC webservice"
print
try:
fou=open(pfnListFile,"r")
firstPfn=fou.readline()
for line in fou.readlines():
if line!=firstPfn:
print "WARNING: PFN matching was not the same on all PhEDEx nodes associated to SRM "+nodeName
print "The PFN path used for testing will be:"
print firstPfn
fou.close()
print "The PFN path used for testing will be:"
print firstPfn
fou=open(pfnFilename,"w")
fou.write(firstPfn)
fou.close()
except IOError:
print "No PFN retrieved from TFC webservice"
print "Possible reasons are:"
print "1) The site is not running a FileExport agent for the Prod instance of PhEDEx"
print "2) The storage.xml catalog published by FileExport does not include a rule for protocol "+sensorname.lower()
print "3) The rule for protocol "+sensorname.lower()+" does not match to a valid PFN"
except urllib2.URLError:
print "WARNING: unable to open URL with SRM list"
......@@ -17,7 +17,7 @@ echo "</h5>"
nodeName=$1
cd $SAME_HOME/sensors/SRM/tests
cd $SAME_SENSOR_HOME/tests
if [ $? -ne 0 ]; then
echo "ERROR: Could not access the tests directory"
exit $SAME_ERROR
......@@ -56,10 +56,10 @@ else
fi
cat $SAME_SENSOR_WORK/nodes/$nodeName/pfnpath.txt | cut -d"?" -f1 > $SAME_SENSOR_WORK/nodes/$nodeName/endpoint.txt
storagepath=`cat $SAME_SENSOR_WORK/nodes/$nodeName/pfnpath.txt | cut -d= -f2-`
token=CMS_DEFAULT
storagepath=`cat $SAME_SENSOR_WORK/nodes/$nodeName/pfnpath.txt | cut -d= -f2- | cut -d " " -f1`
token=`cat $SAME_SENSOR_WORK/nodes/$nodeName/pfnpath.txt | cut -d " " -f3`
echo $storagepath > $SAME_SENSOR_WORK/nodes/$nodeName/SApath.txt
echo "GlueVOInfoPath: "$storagepath" GlueVOInfoTag: "$token" GlueChunkKey: cmsdefaultspacetoken" > $SAME_SENSOR_WORK/nodes/$nodeName/VOInfo.txt
echo "GlueVOInfoPath: "$storagepath" GlueVOInfoTag: "$token" GlueChunkKey: noncustodialfromtfc" > $SAME_SENSOR_WORK/nodes/$nodeName/VOInfo.txt
grep -w ERROR $SAME_SENSOR_WORK/nodes/$nodeName/LfnToPfnFromTfc.out > /dev/null
ERROR_RESULT=$?
......
......@@ -45,20 +45,30 @@ echo "</pre>"
VO=$SAME_VO
echo "<h2>"
echo "Testing space tokens"
echo "Checking space tokens"
echo "</h2>"
if [ -f $SAME_SENSOR_WORK/nodes/$nodeName/VOInfo.txt ] ; then
cat $SAME_SENSOR_WORK/nodes/$nodeName/VOInfo.txt | while read line
do
echo "<p>"
echo "Testing storage space with token description:"
echo "</p>"
SPACE=`echo $line | awk '{print $4}'`
echo "<pre>"
echo $SPACE
echo "</pre>"
tokenoption=" "
gettokenoption=" "
if [ $SPACE == "notoken" ] ; then
echo "<p>"
echo "No space token specified; testing default storage area without specifying space tokens"
echo "</p>"
else
echo "<p>"
echo "Testing storage space with token description:"
echo "</p>"
echo "<pre>"
echo $SPACE
echo "</pre>"
tokenoption="-S $SPACE"
gettokenoption="-s $SPACE "
fi
echo "<p>"
echo "Storage path will be:"
echo "</p>"
......@@ -83,124 +93,7 @@ if [ -f $SAME_SENSOR_WORK/nodes/$nodeName/VOInfo.txt ] ; then
echo "<p>"
echo "<pre>"
set -x
lcg-cp -b --vo ${VO} -D srmv2 -U srmv2 -S ${SPACE} -v file:$SAME_SENSOR_WORK/testFile.txt ${SURL}
result=$?
set +x
echo "</pre>"
if [ $result -ne 0 ] ; then
echo "<p>"
echo "WARNING: File ${SURL} not copied"
echo "</p>"
echo "<p>"
echo "This warning may be ignored if the "$SPACE" space token is not defined on "$nodeName
echo "</p>"
else
echo "<p>"
echo "File ${SURL} copied successfully"
echo "</p>"
echo "<p>"
echo "Sleeping for 30 seconds"
echo "</p>"
echo "<pre>"
set -x
sleep 30
set +x
echo "</pre>"
echo "<pre>"
set -x
lcg-cp -b --vo ${VO} -D srmv2 -T srmv2 -s ${SPACE} -v ${SURL} file:/tmp/test-${cdate}.txt
lresult=$?
set +x
echo "</pre>"
if [ $lresult != 0 ] ; then
echo "<p>"
echo "WARNING: File ${SURL} not copied back"
echo "</p>"
echo "<p>"
echo "Exit status: $lresult"
echo "</p>"
echo "<p>"
echo "This warning may be ignored if the "$SPACE" space token is not defined on "$nodeName
echo "</p>"
else
echo "<p>"
echo "File ${SURL} copied back successfully"
echo "</p>"
fi
echo "<pre>"
set -x
lcg-del -v -b -l -D srmv2 -T srmv2 --vo ${VO} ${SURL}
dresult=$?
set +x
echo "</pre>"
if [ $dresult != 0 ] ; then
echo "<p>"
echo "WARNING: File ${SURL} not deleted"
echo "</p>"
echo "<p>"
echo "Exit status: $dresult"
echo "</p>"
echo "<p>"
echo "This warning may be ignored if the "$SPACE" space token is not defined on "$nodeName
echo "</p>"
else
echo "<p>"
echo "File ${SURL} deleted successfully"
echo "</p>"
fi
fi
if [ $result != 0 ] || [ $lresult != 0 ] || [ $dresult != 0 ] ; then
echo 1 > $SAME_SENSOR_WORK/nodes/$nodeName/SRM-lcg-cp-token.error
else
echo "<p>"
echo "Space token "
echo $SPACE
echo " tested successfully"
echo "</p>"
fi
done
else
echo "<p>"
echo "WARNING: Space tokens not defined"
echo "</p>"
echo 1 > $SAME_SENSOR_WORK/nodes/$nodeName/SRM-lcg-cp-token.error
fi
if [ ! -f $SAME_SENSOR_WORK/nodes/$nodeName/SRM-lcg-cp-token.error ] ; then
exit $SAME_OK
fi
echo "<h2>"
echo "Testing storage area without specifying space tokens"
echo "</h2>"
if [ -f $SAME_SENSOR_WORK/nodes/$nodeName/SApath.txt ] ; then
cat $SAME_SENSOR_WORK/nodes/$nodeName/SApath.txt | while read line
do
echo "<p>"
echo "Testing storage space in path:"
echo "</p>"
echo "<pre>"
echo $line
echo "</pre>"
SURL=$srmv2?SFN=$line/lcg-util/testfile-cp-${cdate}.txt
echo "<p>"
echo "The SURL used for testing will be:"
echo "</p>"
echo "<pre>"
echo $SURL
echo "</pre>"
echo "<p>"
echo "<pre>"
set -x
lcg-cp -b --vo ${VO} -D srmv2 -U srmv2 -v file:$SAME_SENSOR_WORK/testFile.txt ${SURL} 1>$SAME_SENSOR_WORK/nodes/$nodeName/SRMv2-lcg-cp-put-output 2>$SAME_SENSOR_WORK/nodes/$nodeName/SRMv2-lcg-cp-put-stderr
lcg-cp -b --vo ${VO} -D srmv2 -U srmv2 ${tokenoption} -v file:$SAME_SENSOR_WORK/testFile.txt ${SURL} 1>$SAME_SENSOR_WORK/nodes/$nodeName/SRMv2-lcg-cp-put-output 2>$SAME_SENSOR_WORK/nodes/$nodeName/SRMv2-lcg-cp-put-stderr
#copyresult=${PIPESTATUS[0]}
copyresult=$?
set +x
......@@ -250,7 +143,7 @@ if [ -f $SAME_SENSOR_WORK/nodes/$nodeName/SApath.txt ] ; then
echo "<pre>"
set -x
lcg-cp -b --vo ${VO} -D srmv2 -T srmv2 -v ${SURL} file:/tmp/test-${cdate}.txt 1>$SAME_SENSOR_WORK/nodes/$nodeName/SRMv2-lcg-cp-get-output 2>$SAME_SENSOR_WORK/nodes/$nodeName/SRMv2-lcg-cp-get-stderr
lcg-cp -b --vo ${VO} -D srmv2 -T srmv2 ${gettokenoption} -v ${SURL} file:/tmp/test-${cdate}.txt 1>$SAME_SENSOR_WORK/nodes/$nodeName/SRMv2-lcg-cp-get-output 2>$SAME_SENSOR_WORK/nodes/$nodeName/SRMv2-lcg-cp-get-stderr
lresult=$?
set +x
cat $SAME_SENSOR_WORK/nodes/$nodeName/SRMv2-lcg-cp-get-output | sed 's/\&/\&amp;/g' | sed 's/</\&lt;/g' | sed 's/>/\&gt;/g'
......@@ -308,16 +201,18 @@ if [ -f $SAME_SENSOR_WORK/nodes/$nodeName/SApath.txt ] ; then
echo 1 > $SAME_SENSOR_WORK/nodes/$nodeName/SRM-lcg-cp.error
else
echo "<p>"
echo "Default path "
echo $line
echo "Path "
echo $srmPath
echo " with token "
echo $SPACE
echo " tested successfully"
echo "</p>"
echo "</p>"
fi
done
else
echo "<p>"
echo "WARNING: Default path not defined"
echo "ERROR: path/token to test not specified"
echo "</p>"
echo 1 > $SAME_SENSOR_WORK/nodes/$nodeName/SRM-lcg-cp.error
fi
......
......@@ -51,7 +51,7 @@ if [ $VO == "dteam" ]; then
fi
echo "<h2>"
echo "Testing space tokens"
echo "Checking space tokens"
echo "</h2>"
......@@ -59,13 +59,23 @@ if [ -f $SAME_SENSOR_WORK/nodes/$nodeName/VOInfo.txt ] ; then
cat $SAME_SENSOR_WORK/nodes/$nodeName/VOInfo.txt | while read line
do
echo "<p>"
echo "Testing storage space with token description:"
echo "</p>"
SPACE=`echo $line | awk '{print $4}'`
echo "<pre>"
echo $SPACE
echo "</pre>"
tokenoption=" "
gttokenoption=" "
if [ $SPACE == "notoken" ] ; then
echo "<p>"
echo "No space token specified; testing default storage area without specifying space tokens"
echo "</p>"
else
echo "<p>"
echo "Testing storage space with token description:"
echo "</p>"
echo "<pre>"
echo $SPACE
echo "</pre>"
tokenoption="-S $SPACE"
gttokenoption="-s $SPACE "
fi
echo "<p>"
echo "Storage path will be:"
srmPath=`echo $line | awk '{print $2}'`
......@@ -89,130 +99,7 @@ if [ -f $SAME_SENSOR_WORK/nodes/$nodeName/VOInfo.txt ] ; then
echo "<p>"
echo "<pre>"
set -x
lcg-cp -b --vo ${VO} -D srmv2 -U srmv2 -S ${SPACE} -v file:$SAME_SENSOR_WORK/testFile.txt ${SURL}
result=$?
set +x
echo "</pre>"
if [ $result -ne 0 ] ; then
echo "<p>"
echo "WARNING: File ${SURL} not copied"
echo "</p>"
echo "<p>"
echo "Exit status: $result"
echo "</p>"
echo "<p>"
echo "This warning may be ignored if the "$SPACE" space token is not defined on "$nodeName
echo "</p>"
else
echo "<p>"
echo "File ${SURL} copied successfully"
echo "</p>"
echo "<p>"
echo "Sleeping for 30 seconds"
echo "</p>"
echo "<pre>"
set -x
sleep 30
set +x
echo "</pre>"
tresult=2
for prot in ${var} ; do
echo "<pre>"
set -x
lcg-gt -b -D srmv2 -T srmv2 -s ${SPACE} ${SURL} ${prot}
gtresult=$?
set +x
echo "</pre>"
if [ $gtresult -ne 0 ] ; then
echo "<p>"
echo "WARNING: TURL for file ${SURL} with protocol ${prot} not provided"
echo "</p>"
else
tresult=0
echo "<p>"
echo "TURL for file ${fil} with protocol ${prot} provided correctly"
echo "</p>"
fi
done
if [ $tresult -ne 0 ] ; then
echo "<p>"
echo "WARNING: No TURLs provided for any of the requested protocols"
echo "</p>"
echo "<p>"
echo "This warning may be ignored if the "$SPACE" space token is not defined on "$nodeName
echo "</p>"
fi
#
echo "<pre>"
set -x
lcg-del -v -b -l -D srmv2 -T srmv2 --vo ${VO} ${SURL}
dresult=$?
set +x
echo "</pre>"
if [ $dresult -ne 0 ] ; then
echo "<p>"
echo "WARNING: File ${SURL} not deleted"
echo "</p>"
echo "<p>"
echo "Exit status: $dresult"
echo "</p>"
echo "<p>"
echo "This warning may be ignored if the "$SPACE" space token is not defined on "$nodeName
echo "</p>"
else
echo "<p>"
echo "File ${fil} deleted successfully"
echo "</p>"
fi
fi
if [ $result != 0 ] || [ $tresult != 0 ] || [ $dresult != 0 ] ; then
echo 1 > $SAME_SENSOR_WORK/nodes/$nodeName/SRM-lcg-gt-token.error
else
echo "<p>"
echo "Space token "
echo $SPACE
echo " tested successfully"
echo "</p>"
fi
done
else
echo "<p>"
echo "WARNING: Space tokens not defined"
echo "</p>"
echo 1 > $SAME_SENSOR_WORK/nodes/$nodeName/SRM-lcg-gt-token.error
fi
if [ ! -f $SAME_SENSOR_WORK/nodes/$nodeName/SRM-lcg-gt-token.error ] ; then
exit $SAME_OK
fi
echo "<h2>"
echo "Testing storage area without specifying space tokens"
echo "</h2>"
cat $SAME_SENSOR_WORK/nodes/$nodeName/SApath.txt | while read line
do
echo "<p>"
echo "Testing storage space in path:"
echo "</p>"
echo "<pre>"
echo $line
echo "</pre>"
SURL=$srmv2?SFN=$line/lcg-util/testfile-gt-${cdate}.txt
echo "<p>"
echo "The SURL used for testing will be:"
echo "</p>"
echo "<pre>"
echo $SURL
echo "</pre>"
echo "<p>"
echo "<pre>"
set -x
lcg-cp -b --vo ${VO} -D srmv2 -U srmv2 -v file:$SAME_SENSOR_WORK/testFile.txt ${SURL} 1>$SAME_SENSOR_WORK/nodes/$nodeName/SRMv2-lcg-gt-put-output 2>$SAME_SENSOR_WORK/nodes/$nodeName/SRMv2-lcg-gt-put-stderr
lcg-cp -b --vo ${VO} -D srmv2 -U srmv2 ${tokenoption} -v file:$SAME_SENSOR_WORK/testFile.txt ${SURL} 1>$SAME_SENSOR_WORK/nodes/$nodeName/SRMv2-lcg-gt-put-output 2>$SAME_SENSOR_WORK/nodes/$nodeName/SRMv2-lcg-gt-put-stderr
#copyresult=${PIPESTATUS[0]}
copyresult=$?
set +x
......@@ -265,7 +152,7 @@ cat $SAME_SENSOR_WORK/nodes/$nodeName/SApath.txt | while read line
for prot in ${var} ; do
echo "<pre>"
set -x
lcg-gt -b -D srmv2 -T srmv2 ${SURL} ${prot}
lcg-gt -b -D srmv2 -T srmv2 ${gttokenoption} ${SURL} ${prot}
gtresult=$?
set +x
echo "</pre>"
......@@ -316,12 +203,20 @@ cat $SAME_SENSOR_WORK/nodes/$nodeName/SApath.txt | while read line
echo 1 > $SAME_SENSOR_WORK/nodes/$nodeName/SRM-lcg-gt.error
else
echo "<p>"
echo "Default path "
echo $line
echo "Path "
echo $srmPath
echo " with token "
echo $SPACE
echo " tested successfully"
echo "</p>"
fi
done
else
echo "<p>"
echo "ERROR: path/token to test not specified"
echo "</p>"
echo 1 > $SAME_SENSOR_WORK/nodes/$nodeName/SRM-lcg-gt.error
fi
if [ -f $SAME_SENSOR_WORK/nodes/$nodeName/SRM-lcg-gt.error ] ; then
......
......@@ -44,7 +44,7 @@ set +x
echo "</pre>"
echo "<h2>"
echo "Testing space tokens"
echo "Checking space tokens"
echo "</h2>"
VO="$SAME_VO"
......@@ -54,13 +54,23 @@ if [ -f $SAME_SENSOR_WORK/nodes/$nodeName/VOInfo.txt ] ; then
cat $SAME_SENSOR_WORK/nodes/$nodeName/VOInfo.txt | while read line
do
echo "<p>"
echo "Testing storage space with token description:"
echo "</p>"
SPACE=`echo $line | awk '{print $4}'`
echo "<pre>"
echo $SPACE
echo "</pre>"
tokenoption=" "
gttokenoption=" "
if [ $SPACE == "notoken" ] ; then
echo "<p>"
echo "No space token specified; testing default storage area without specifying space tokens"
echo "</p>"
else
echo "<p>"
echo "Testing storage space with token description:"