Added storage waypoint and new basic test

SiteTests/SE/srmvometrics.py

@@ -593,6 +593,7 @@ class SRMVOMetrics(probe.MetricGatherer) :
                      'lcgse01.phy.bris.ac.uk': "gsiftp://lcgse01.phy.bris.ac.uk/dpm/phy.bris.ac.uk/home/cms/store/unmerged/SAM/testSRM/SAM-lcgse01.phy.bris.ac.uk/lcg-util",
                      'heplnx204.pp.rl.ac.uk': "srm://heplnx204.pp.rl.ac.uk:8443/srm/managerv2?SFN=/pnfs/pp.rl.ac.uk/data/cms/store/unmerged/SAM/testSRM/SAM-heplnx204.pp.rl.ac.uk/lcg-util",
                      'transfer.ultralight.org': "gsiftp://transfer.ultralight.org:2811//mnt/hadoop//store/unmerged/SAM/testSRM/SAM-transfer.ultralight.org/lcg-util",
+                     'transfer-10.ultralight.org': "gsiftp://transfer-10.ultralight.org:2811/storage/cms",
                      'cmsio.rc.ufl.edu': "gsiftp://cmsio.rc.ufl.edu//cmsuf/data/store/unmerged/SAM/testSRM/SAM-cmsio.rc.ufl.edu/lcg-util",
                      'se01.cmsaf.mit.edu': "gsiftp://se01.cmsaf.mit.edu:2811/cms/store/unmerged/SAM/testSRM/SAM-se01.cmsaf.mit.edu/lcg-util",
                      'red-gridftp.unl.edu': "gsiftp://red-gridftp.unl.edu//mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/unmerged/SAM/testSRM/SAM-red-gridftp.unl.edu/lcg-util",

SiteTests/testjob/tests/CE-cms-basic

@@ -95,13 +95,16 @@ if [ ! -d $CMS_PATH ] ; then
     exit $SAME_ERROR
 fi
 
-if [ ! -d ${CMS_PATH}/SITECONF/local/JobConfig ] ; then
-    echo "ERROR: JobConfig directory $CMS_PATH/SITECONF/local/JobConfig not existing"
+if [ -n "${SITECONFIG_PATH}" -a -d ${SITECONFIG_PATH}/JobConfig ] ; then
+    echo "SITECONFIG_PATH: ${SITECONFIG_PATH}"
+    ConfigFile=${SITECONFIG_PATH}/JobConfig/site-local-config.xml
+elif [ -d ${CMS_PATH}/SITECONF/local/JobConfig ] ; then
+    ConfigFile=${CMS_PATH}/SITECONF/local/JobConfig/site-local-config.xml
+else
+    echo "ERROR: no JobConfig directory at ${SITECONFIG_PATH}/JobConfig nor $CMS_PATH/SITECONF/local/JobConfig"
     echo "summary: NO_JOBCONFIG_DIR"
     exit $SAME_ERROR
 fi
-
-ConfigFile=${CMS_PATH}/SITECONF/local/JobConfig/site-local-config.xml
 echo "SiteLocalConfig: $ConfigFile"
 
 if [ ! -f $ConfigFile ] ; then
@@ -128,21 +131,42 @@ if [ $status != 0 ] ; then
     error=1
 fi
 
+# obtain site/sub-site information from SITECONF path:
+SiteSubsite=`echo "${ConfigFile}" | /usr/bin/awk -F/ '{if(NF<=2){print "/"}else{if((NF==3)||($(NF-3)=="SITECONF")){printf "%s/\n",$(NF-2)}else{printf "%s/%s\n",$(NF-3),$(NF-2)}}}'`
+#
+CMSSite_Name=`echo ${SiteSubsite} | /usr/bin/awk -F/ '{print $1}'`
+if [ "${CMSSite_Name}" == "local" ]; then
+   LocalDir=`echo "${ConfigFile}" | /usr/bin/awk -F/ '{n=length($NF)+length($(NF-1))+2;for(i=NF-2;i>0;i--){if($i=="local"){print substr($0,1,length($0)-n);exit}else{n+=1+length($i)}}}'`
+   if [ -h "${LocalDir}" ]; then
+      LocalDir=`readlink -e "${LocalDir}"`
+      CMSSite_Name=`basename "${LocalDir}"`
+   fi
+fi
+CMSSite_Name=`expr match "${CMSSite_Name}" '^\(T[0-9]_[A-Z][A-Z]_[A-Za-z0-9_][A-Za-z0-9_]*\)$'`
+#
+CMSSubsite_Name=`echo ${SiteSubsite} | /usr/bin/awk -F/ '{print $2}'`
+CMSSubsite_Name=`expr match "${CMSSubsite_Name}" '^\([A-Za-z0-9][-_A-Za-z0-9]*\)$'`
+
+# override site/sub-site information in case of SITECONF tags:
 SiteName=`sed -nr '/<site name/s/^.*\"(.*)\".*$/\1/p' $ConfigFile`
-status=$?
-if [ $status != 0 ] ; then
+if [ $? != 0 -o -z "${SiteName}" ] ;then
+   SiteName="${CMSSite_Name}"
+fi
+if [ -z "${SiteName}" ]; then
     echo "ERROR: site name string missing"
     errorSummary="summary: NO_SITENAME"
     error=1
 else
-    echo "SiteName: $SiteName"
+    echo "CMSSite_Name: $SiteName"
 fi
 
 subSiteName=""
 subSiteName=`grep "subsite name" $ConfigFile | cut -d'"' -f2`
+if [ -z "$subSiteName" ]; then
+   subSiteName="${CMSSubsite_Name}"
+fi
 if [ -n "$subSiteName" ] ; then
-    echo "SubSiteName: $subSiteName"
-    subSiteName=_$subSiteName
+    echo "CMSSubsite_Name: $subSiteName"
 fi
 
 grep -q "frontier-connect" $ConfigFile
@@ -158,23 +182,55 @@ if [ $error == 1 ] ; then
     echo $errorSummary
     exit $SAME_ERROR
 fi
+echo ""
 
 let seconds=`date +%s`-`stat -c%Y $ConfigFile`
 let localConfigFileAgeInHours=$seconds/3600
 echo "SiteLocalConfigAge: ${localConfigFileAgeInHours} hours"
 
 asCvs=1
-cvsUrl="https://gitlab.cern.ch/api/v4/projects/siteconf%2F${SiteName}/repository/files/JobConfig%2Fsite-local-config${subSiteName}.xml/raw?ref=master"
 ConfigFileFromCVS=`mktemp /tmp/site-local-config-from-CVS.xml.XXX`
-echo "GITCopy: $cvsUrl"
-./fetch-from-web-gitlab \"$cvsUrl\" $ConfigFileFromCVS
-rc=$?
+if [ -n "${subSiteName}" ] ; then
+   echo "checking old sub-site GitLab location:"
+   baseURL="https://gitlab.cern.ch/api/v4/projects/siteconf%2F${SiteName}/repository/files/JobConfig%2Fsite-local-config_${subSiteName}.xml"
+   cvsUrl="${baseURL}/raw?ref=master"
+   echo "GITCopy: $cvsUrl"
+   ./fetch-from-web-gitlab \"${cvsUrl}\" $ConfigFileFromCVS
+   rc=$?
+   if [ $rc != 0 ] ; then
+      /bin/rm -f ${ConfigFileFromCVS} 1>/dev/null 2>&1
+      echo ""
+      echo "checking new sub-site GitLab location:"
+      baseURL="https://gitlab.cern.ch/api/v4/projects/siteconf%2F${SiteName}/repository/files/${subSiteName}/JobConfig%2Fsite-local-config.xml"
+      cvsUrl="${baseURL}/raw?ref=master"
+      echo "GITCopy: $cvsUrl"
+      ./fetch-from-web-gitlab \"${cvsUrl}\" $ConfigFileFromCVS
+      rc=$?
+      if [ $rc != 0 ] ; then
+         /bin/rm -f ${ConfigFileFromCVS} 1>/dev/null 2>&1
+         echo ""
+         echo "checking site GitLab location:"
+         baseURL="https://gitlab.cern.ch/api/v4/projects/siteconf%2F${SiteName}/repository/files/JobConfig%2Fsite-local-config.xml"
+         cvsUrl="${baseURL}/raw?ref=master"
+         echo "GITCopy: $cvsUrl"
+         ./fetch-from-web-gitlab \"${cvsUrl}\" $ConfigFileFromCVS
+         rc=$?
+      fi
+   fi
+else
+   # check site GitLab location:
+   baseURL="https://gitlab.cern.ch/api/v4/projects/siteconf%2F${SiteName}/repository/files/JobConfig%2Fsite-local-config.xml"
+   cvsUrl="${baseURL}/raw?ref=master"
+   echo "GITCopy: $cvsUrl"
+   ./fetch-from-web-gitlab \"${cvsUrl}\" $ConfigFileFromCVS
+   rc=$?
+fi
 if [ $rc == 0 ] ; then
   diff -q -w -B $ConfigFile $ConfigFileFromCVS
   if [ $? == 1 ] ; then asCvs=0; fi
   if [ $asCvs == 0 ] ; then
     noConfigDump=1
-    FileInfoUrl="https://gitlab.cern.ch/api/v4/projects/siteconf%2F${SiteName}/repository/files/JobConfig%2Fsite-local-config${subSiteName}.xml?ref=master"
+    FileInfoUrl="${baseURL}?ref=master"
     FileInfoTmp=`mktemp /tmp/site-local-config-from-CVS-Markup.xml.XXX`
     ./fetch-from-web-gitlab \"$FileInfoUrl\" $FileInfoTmp
     last_commit=$(grep -Po '"last_commit_id":.*?[^\\]",' ${FileInfoTmp} | perl -pe 's/"last_commit_id"://; s/^"//; s/",$//')
@@ -206,6 +262,7 @@ else
   errorSummary="summary: GIT_NO_ACCESS"
   note=1
 fi
+echo ""
 
 let seconds=`date +%s`-`stat -c%Y $TFCPath`
 let localTFCfileAgeInHours=$seconds/3600