Commit 8d8466b1 authored by Andrea Sciaba's avatar Andrea Sciaba
Browse files

*** empty log message ***

parent 52014b53
testName: CE-cms-frontier
testTitle: Frontier test
testAbbr: frontier
testHelp: https://twiki.cern.ch/twiki/bin/view/CMS/SAMFrontier
testName: CE-cms-squid
testTitle: SQUID test
testAbbr: squid
testHelp: https://twiki.cern.ch/twiki/bin/view/CMS/SAMSquid
testName: CE-cms-mc
testTitle: Monte Carlo
testAbbr: mc
testHelp: https://twiki.cern.ch/twiki/bin/view/CMS/SAMMonteCarlo
#!/bin/bash
siteName=$1
nodeName=$2
inMaintenance=$3
envName=`cat $SAME_SENSOR_WORK/envName`
if [ "x$1" == "x--publish" ] ; then
shift
echo "$nodeName: published" >&2
else
echo "$nodeName: started" >&2
mkdir -p $SAME_SENSOR_WORK/nodes/$nodeName
cd $SAME_SENSOR_WORK/nodes/$nodeName
echo $envName > envName
echo $siteName > siteName
rm -f *.result
rm -f pfnlist.txt
rm -f pfn.txt
# Removing cached PFN path if older than one day
if [ -f pfnpath.txt ] ; then
now=`date +%s`
modificationtime=`stat -c %Y pfnpath.txt`
if ((now-modificationtime>604800)) ; then
rm pfnpath.txt
fi
fi
for test in `cat $SAME_SENSOR_HOME/test-sequence.lst` ; do
same-run-test -v $SAME_VO -t $test -n $nodeName -e $envName -d $SAME_SENSOR_HOME/tests $nodeName > $test.result
done
echo "$nodeName: publishing" >&2
same-publish-tuples TestData *.result >&2
echo "$nodeName: finished" >&2
fi
#!/bin/bash
voms-proxy-info -exists -valid 06:00 || ( echo "No valid VOMS proxy found" >&2 && false ) || exit 1
if [ "x$1" == "x--publish" ] ; then
echo "This sensor does not support publish operation!" >&2
else
echo "prepare-SRM script started." >&2
echo -n "got nodes:" >&2
wc -l nodes.map >&2
cd $SAME_SENSOR_WORK
echo -n "0123456789" > t1
cat t1 t1 > t2
cat t2 t2 > t1
cat t1 t1 > t2
echo "" >> t2
cat t2 t2 > t1
cat t1 t1 > t2
cat t2 t2 > t1
cat t1 t1 > t2
cat t2 t2 > t1
cat t1 t1 > t2
cat t2 t2 > t1
cat t1 t1 > t2
cat t2 t2 > t1
mv t1 testFile_put.txt
rm t2
cksum testFile_put.txt > testFile_put.cksum
adler32 testFile_put.txt | grep -v WARNING > testFile_put.adler32sum
md5sum testFile_put.txt > testFile_put.md5sum
echo -n "publishing test definitions... " >&2
same-publish-tuples TestDef $SAME_SENSOR_HOME/tests/*.def >&2
envName=SRMv1-`date +%s`
submitterDN=`grid-proxy-info -identity`
cat <<EOF | same-publish-tuples TestEnvVars
envName: $envName
name: submitterDN
value: $submitterDN
EOF
echo $envName > $SAME_SENSOR_WORK/envName
echo "DONE" >&2
fi
SRM-v1-get-pfn-from-tfc
SRM-v1-put
SRM-v1-get-metadata
SRM-v1-get
SRM-v1-advisory-delete
#!/usr/bin/python
import sys
import os
import urllib2
import time
import re
import simplejson
# Read SAM environment
workdir = os.environ.get("SAME_SENSOR_WORK")
sensordir = os.environ.get("SAME_SENSOR_HOME")
sensorname = os.environ.get("SAME_SENSOR_NAME")
samehome = os.environ.get("SAME_HOME")
#URLs for PhEDEx DataService for TFC
tfcURL="http://cmsweb.cern.ch/phedex/datasvc/json/prod/lfn2pfn?node="
pfnMatchURL="&lfn="
pfnProtocolOption = "&protocol=srmv2"
#Path of text file with list of SRM endpoints
seMapFileURL="http://cern.ch/magini/phedex-endpoints.txt"
if sensorname == "SRM":
seMapFileURL="http://cern.ch/magini/phedex-v1-endpoints.txt"
elif sensorname == "SRMv2":
seMapFileURL="http://cern.ch/magini/phedex-v2-endpoints.txt"
if sensorname == "SRM":
pfnProtocolOption="&protocol=srm"
# get nodeName to test from environment
nodeName = sys.argv[1]
nodeWorkDir = workdir + "/nodes/" + nodeName
# Define LFN path for file to test transfers
# If LFN is not provided as second argument of the script, the default /store/unmerged/SAM/testSRM is used
try:
testLFN = sys.argv[2]
except IndexError:
testLFN = "/store/unmerged/SAM/testSRM"
print "LFN path of test file is " + testLFN
print
# Define filename for file used to store PFN for successive tests
# If filename is not provided as third argument of the script, the defaults pfnpath.txt & pfnlist.txt are used
try:
basePfnFilename= sys.argv[3]
basePfnListFilename= "list_"+sys.argv[3]
except:
basePfnFilename="pfnpath.txt"
basePfnListFilename="pfnlist.txt"
pfnFilename = nodeWorkDir + "/" + basePfnFilename
pfnListFile= nodeWorkDir + "/" + basePfnListFilename
print "Retrieving list of endpoints to test at URL:"
print seMapFileURL
print
try:
siteNameFile=urllib2.urlopen(seMapFileURL)
for siteLine in siteNameFile.readlines():
endpointName=siteLine.split()[0]
siteName=siteLine.split()[1]
if endpointName == nodeName:
print nodeName+" is listed as SRM for Site "+siteName
# Contact web service to get LFN
endpointLFN = "/SAM-%s" % endpointName
pfnUrl = tfcURL+siteName+pfnMatchURL+testLFN+endpointLFN+pfnProtocolOption
print "Contacting webservice to perform LFN-to-PFN matching at URL:"
print pfnUrl
print
try:
pfnFile=urllib2.urlopen(pfnUrl)
pfnJSON = simplejson.load(pfnFile)
pfn = (((pfnJSON[u'phedex'])[u'mapping'])[0])[u'pfn']
if pfn == None:
if sensorname=="SRM":
print "LFN did not match to any PFN - probably the TFC does not contain any rule for the srm protocol"
elif sensorname=="SRMv2":
print "LFN did not match to any PFN - probably the TFC does not contain any rule for the srmv2 protocol"
else:
print "LFN was matched to PFN "+pfn
print
if not re.compile("^srm://.+:[0-9]+/.+\?SFN=.+$").match(pfn):
print "WARNING: Invalid matching to srm protocol"
print "Note: this test currently supports only PFNs in the full format:"
print "srm://hostname:port/srmendpointpath?SFN=sitefilename"
print
else:
if re.compile("^srm://.+srm/managerv1.+$").match(pfn):
if sensorname=="SRMv2":
print "WARNING: matching to srmv1 protocol"
else:
pfntonode=re.sub(":.+$","",re.sub("^srm://","",pfn))
if pfntonode!=nodeName :
print "WARNING: the resulting PFN matches to SRM "+pfntonode+" instead of SRM "+nodeName
print
else:
fou=open(pfnListFile,"a")
fou.write(pfn + "\n")
fou.close()
elif re.compile("^srm://.+srm/managerv2.+$").match(pfn) or re.compile("^srm://.+srm/v2/server.+$").match(pfn):
if sensorname=="SRM":
print "WARNING: matching to srmv2 protocol"
else:
pfntonode=re.sub(":.+$","",re.sub("^srm://","",pfn))
if pfntonode!=nodeName :
print "WARNING: the resulting PFN matches to SRM "+pfntonode+" instead of SRM "+nodeName
print
else:
fou=open(pfnListFile,"a")
fou.write(pfn + "\n")
fou.close()
else:
print "INFO: srm version not specified in PFN"
pfntonode=re.sub(":.+$","",re.sub("^srm://","",pfn))
if pfntonode!=nodeName :
print "WARNING: the resulting PFN matches to SRM "+pfntonode+" instead of SRM "+nodeName
print
else:
fou=open(pfnListFile,"a")
fou.write(pfn + "\n")
fou.close()
except urllib2.URLError:
print "WARNING: Unable to open TFC webservice URL to perform LFN-to-PFN matching for Site "+siteName
print
except ValueError:
print "WARNING: Site "+siteName+" is unknown to TFC webservice"
print
try:
fou=open(pfnListFile,"r")
firstPfn=fou.readline()
for line in fou.readlines():
if line!=firstPfn:
print "WARNING: PFN matching was not the same on all PhEDEx nodes associated to SRM "+nodeName
print "The PFN path used for testing will be:"
print firstPfn
fou.close()
print "The PFN path used for testing will be:"
print firstPfn
fou=open(pfnFilename,"w")
fou.write(firstPfn)
fou.close()
except IOError:
print "No PFN retrieved from TFC webservice"
print "Possible reasons are:"
print "1) The site is not running a FileExport agent for the Prod instance of PhEDEx"
print "2) The storage.xml catalog published by FileExport does not include a rule for protocol "+sensorname.lower()
print "3) The rule for protocol "+sensorname.lower()+" does not match to a valid PFN"
except urllib2.URLError:
print "WARNING: unable to open URL with SRM list"
#!/bin/bash
echo "Test OK"
exit $SAME_OK
testName: SRM-cms-dummy
testTitle: Dummy test
testAbbr: dummy
testHelp: none
#!/bin/bash
#****** SRM/SRM-v1-advisory-delete
# NAME
# SRM-v1-advisory-delete - Delete a file from the SRM using srm-advisory-delete
#
# AUTHOR
#
# SAM Nicolo.Magini[at]cern.ch
#
# LAST UPDATED
#
# 2007-04-18
#
# LANGUAGE
#
# bash
#
# SOURCE
nodeName=$1
siteName=`cat siteName`
pfn=`cat $SAME_SENSOR_WORK/nodes/$nodeName/pfn.txt`
echo "<br>"
echo "Checking if file $pfn can be removed from SRM $nodeName at site $siteName"
if [ x$pfn == "x" ] ; then
echo "<p>"
echo "PFN not specified"
exit $SAME_WARNING
echo "</p>"
fi
echo "<pre>"
set -x
srm-advisory-delete -debug=true -conf=$SAME_SENSOR_HOME/tests/samsrmconfig.xml $pfn
result=$?
set +x
echo "</pre>"
if [ $result == 0 ] ; then
exit $SAME_OK
else
exit $SAME_ERROR
fi
testName: SRM-v1-advisory-delete
testAbbr: v1-advisory-delete
testTitle: Delete a file from the SRM using srm-advisory-delete
testHelp: https://twiki.cern.ch/twiki/bin/view/CMS/SAMSRMv1AdvisoryDelete
#!/bin/bash
#****** SRM/SRM-v1-get
# NAME
# SRM-v1-get - Copy a file back from the SRM (get) using srmcp
#
# AUTHOR
#
# SAM Nicolo.Magini[at]cern.ch
#
# LAST UPDATED
#
# 2007-04-18
#
# LANGUAGE
#
# bash
#
# SOURCE
nodeName=$1
siteName=`cat siteName`
pfn=`cat $SAME_SENSOR_WORK/nodes/$nodeName/pfn.txt`
maxRetries=3
retryDelay=30
echo "<br>"
echo "Checking if file $pfn can be copied back from SRM $nodeName at site $siteName with srmcp"
if [ x$pfn == "x" ] ; then
echo "<p>"
echo "PFN not specified"
exit $SAME_WARNING
echo "</p>"
fi
srmgetsuccess=0
srmwarning=0
sleep $retryDelay
for (( nRetries=0 ; nRetries <= maxRetries ; nRetries++))
do
if [ $nRetries != 0 ] ; then
echo "<p>"
echo "Sleeping for "$retryDelay " seconds before retrying."
echo "</p>"
sleep $retryDelay
srmwarning=1
fi
if [ -f $SAME_SENSOR_WORK/nodes/$nodeName/testFile_get.txt ] ; then
rm $SAME_SENSOR_WORK/nodes/$nodeName/testFile_get.txt
fi
echo "<p>"
echo "Trying srmcp Get: n="$nRetries
echo "</p>"
echo "<pre>"
set -x
srmcp -conf=$SAME_SENSOR_HOME/tests/samsrmconfig-get.xml -debug=true -protocols=gsiftp -retry_num=0 $pfn file:///$SAME_SENSOR_WORK/nodes/$nodeName/testFile_get.txt >& $SAME_SENSOR_WORK/nodes/$nodeName/get_$nRetries.log
result=$?
cat $SAME_SENSOR_WORK/nodes/$nodeName/get_$nRetries.log
set +x
echo "</pre>"
if [ $result == 0 ] ; then
echo "<p>"
echo "srmcp Get successful"
echo "</p>"
srmgetsuccess=1
break
else
# Verifying if non-zero exit status caused by 0-filesize CASTOR bug
grep -q "we wrote more then file size" $SAME_SENSOR_WORK/nodes/$nodeName/get_$nRetries.log
grep_filesize=$?
if [ $grep_filesize == 0 ] ; then
echo "<p>"
echo "Verifying actual file size"
echo "</p>"
truesrmfilesize=`ls -l $SAME_SENSOR_WORK/nodes/$nodeName/testFile_get.txt | awk '{print $5}'`
filesize=`cat $SAME_SENSOR_WORK/testFile_put.cksum | awk '{print $2}'`
if [ $truesrmfilesize == $filesize ] ; then
echo "<p>"
echo "File size is correct"
echo "</p>"
echo "<p>"
echo "srmcp Get successful"
echo "</p>"
srmgetsuccess=1
break
fi
fi
fi
done
if [ $srmgetsuccess == 0 ] ; then
echo "srmcp (get) failed "$maxRetries" times."
exit $SAME_ERROR
fi
echo "Checking if files are identical"
echo "<pre>"
set -x
diff $SAME_SENSOR_WORK/testFile_put.txt $SAME_SENSOR_WORK/nodes/$nodeName/testFile_get.txt
diffresult=$?
set +x
echo "</pre>"
if [ $diffresult == 0 ] ; then
if [ $srmwarning == 0 ] ; then
exit $SAME_OK
else
echo "<p>"
echo "WARNING: srm Get succeeded, but not on first attempt"
echo "</p>"
exit $SAME_WARNING
fi
else
echo "File retrieved is different from original file!"
exit $SAME_ERROR
fi
#!/bin/bash
#****** SRM/SRM-v1-get-metadata
# NAME
# SRM-v1-get-metadata - Verify file stored in SRMv1 using srm-get-metadata
#
# AUTHOR
#
# SAM Nicolo.Magini[at]cern.ch
#
# LAST UPDATED
#
# 2007-04-18
#
# LANGUAGE
#
# bash
#
# SOURCE
nodeName=$1
siteName=`cat siteName`
pfn=`cat $SAME_SENSOR_WORK/nodes/$nodeName/pfn.txt`
echo "<br>"
echo "Checking file $pfn stored on SRM $nodeName at site $siteName"
if [ x$pfn == "x" ] ; then
echo "<p>"
echo "PFN not specified"
exit $SAME_WARNING
echo "</p>"
fi
echo "<pre>"
set -x
srm-get-metadata -conf=$SAME_SENSOR_HOME/tests/samsrmconfig.xml -retry_num=3 $pfn | tee $SAME_SENSOR_WORK/nodes/$nodeName/put_metadata.txt
grep FileMetaData $SAME_SENSOR_WORK/nodes/$nodeName/put_metadata.txt
result=$?
set +x
echo "</pre>"
if [ $result != 0 ] ; then
echo "srm-get-metadata failed"
exit $SAME_ERROR
fi
echo "Checking size and checksum of file $pfn"
echo "<pre>"
set -x
cksumtype=`grep "checksumType" $SAME_SENSOR_WORK/nodes/$nodeName/put_metadata.txt | cut -d ":" -f2`
srmsize=`grep "size" $SAME_SENSOR_WORK/nodes/$nodeName/put_metadata.txt | cut -d ":" -f2`
srmsum=`grep "checksumValue" $SAME_SENSOR_WORK/nodes/$nodeName/put_metadata.txt | cut -d ":" -f2`
filesize=`cat $SAME_SENSOR_WORK/testFile_put.cksum | awk '{print $2}'`
filesum=`cat $SAME_SENSOR_WORK/testFile_put.adler32sum`
set +x
echo "</pre>"
if [ $srmsize != $filesize ] ; then
echo "<p>"
echo "File size on srm differs from local file"
echo "</p>"
if [ $srmsize == "0" ] ; then
echo "<p>"
echo "WARNING: File size is 0 on srm. Real file size will be checked after get test"
echo "</p>"
exit $SAME_WARNING
else
exit $SAME_ERROR
fi
fi
if [ $cksumtype == "null" ] ; then
echo "File checksum not found in SRM metadata"
exit $SAME_OK
fi
if [ $srmsum != $filesum ] ; then
echo "File checksum on srm differs from local file"
exit $SAME_ERROR
else
exit $SAME_OK
fi
testName: SRM-v1-get-metadata
testAbbr: v1-get-metadata
testTitle: Verify file stored in SRMv1 using srm-get-metadata
testHelp: https://twiki.cern.ch/twiki/bin/view/CMS/SAMSRMv1GetMetadata
#!/bin/bash
echo "<h2>Checking if TFC is available for SRMv1 endpoint and using it to perform lfn to pfn matching</h2>"
echo "<h3>Checking PYTHONPATH</h3>"
echo "<pre>"
echo "PYTHONPATH=$PYTHONPATH"
echo "</pre>"
echo "<h3>Invoking site test</h3>"
echo "<pre>"
echo "<h5>"
echo "USER/WHOAMI?:" $USER/`whoami`
echo "USER ID:" $uid
voms-proxy-info -all
echo "</h5>"
nodeName=$1
cd $SAME_SENSOR_HOME/tests
if [ $? -ne 0 ]; then
echo "ERROR: Could not access the tests directory"
exit $SAME_ERROR
else
chmod u+x LfnToPfnFromTfc.py
./LfnToPfnFromTfc.py $nodeName | tee $SAME_SENSOR_WORK/nodes/$nodeName/LfnToPfnFromTfc.out
EXIT_STATUS=$?
fi
echo "</pre>"
if [ $EXIT_STATUS -ne 0 ]; then
echo "ERROR: LfnToPfnFromTfc.py script failed: $EXIT_STATUS"
exit $SAME_ERROR
else
if [ -f $SAME_SENSOR_WORK/nodes/$nodeName/pfnpath.txt ]; then
if [ ! -f $SAME_SENSOR_WORK/nodes/$nodeName/pfnlist.txt ]; then
echo "<pre>"
echo "WARNING: Retrieval of PFN from TFC webservice failed; cached PFN will be used"
echo "</pre>"
fi
echo "<pre>"
echo "TFC Test successful"
echo "</pre>"
grep -w $nodeName $SAME_SENSOR_WORK/nodes/$nodeName/pfnpath.txt > /dev/null
MATCH_RESULT=$?
if [ $MATCH_RESULT -ne 0 ]; then
echo "<pre>"
echo "SRM endpoint in TFC does not match $nodeName"
echo "</pre>"
fi
grep -w ERROR $SAME_SENSOR_WORK/nodes/$nodeName/LfnToPfnFromTfc.out > /dev/null
ERROR_RESULT=$?
grep -w WARNING $SAME_SENSOR_WORK/nodes/$nodeName/LfnToPfnFromTfc.out > /dev/null
WARNING_RESULT=$?
grep -w INFO $SAME_SENSOR_WORK/nodes/$nodeName/LfnToPfnFromTfc.out > /dev/null