Fixes to git access and to xrootd fallback

SiteTests/testjob/tests/CE-cms-basic

@@ -164,7 +164,7 @@ let localConfigFileAgeInHours=$seconds/3600
 echo "SiteLocalConfigAge: ${localConfigFileAgeInHours} hours"
 
 asCvs=1
-cvsUrl="https://gitlab.cern.ch/SITECONF/${SiteName}/raw/master/JobConfig/site-local-config${subSiteName}.xml"
+cvsUrl="https://gitlab.cern.ch/api/v4/projects/siteconf%2F${SiteName}/repository/files/JobConfig%2Fsite-local-config${subSiteName}.xml/raw?ref=master"
 ConfigFileFromCVS=`mktemp /tmp/site-local-config-from-CVS.xml.XXX`
 echo "GITCopy: $cvsUrl"
 ./fetch-from-web-gitlab \"$cvsUrl\" $ConfigFileFromCVS
@@ -174,11 +174,14 @@ if [ $rc == 0 ] ; then
   if [ $? == 1 ] ; then asCvs=0; fi
   if [ $asCvs == 0 ] ; then
     noConfigDump=1
-    cvsMarkupUrl="https://gitlab.cern.ch/SITECONF/${SiteName}/blob/master/JobConfig/site-local-config${subSiteName}.xml"
-    ConfigFileMarkup=`mktemp /tmp/site-local-config-from-CVS-Markup.xml.XXX`
-    ./fetch-from-web-gitlab \"$cvsMarkupUrl\" $ConfigFileMarkup
-    cvsFileDate=`/bin/awk '{i=index($0,"datetime=");if(i>0){s=substr($0,i+10,4) "-" substr($0,i+15,2) "-" substr($0,i+18,2) " " substr($0,i+21,2) ":" substr($0,i+24,2) ":" substr($0,i+27,2) "Z";print s}}' $ConfigFileMarkup`
-    let seconds=`date +%s`-`date +%s -d "$cvsFileDate"`
+    FileInfoUrl="https://gitlab.cern.ch/api/v4/projects/siteconf%2F${SiteName}/repository/files/JobConfig%2Fsite-local-config${subSiteName}.xml?ref=master"
+    FileInfoTmp=`mktemp /tmp/site-local-config-from-CVS-Markup.xml.XXX`
+    ./fetch-from-web-gitlab \"$FileInfoUrl\" $FileInfoTmp
+    last_commit=`jq -r .last_commit_id $FileInfoTmp`
+    CommitInfoUrl="https://gitlab.cern.ch/api/v4/projects/siteconf%2F${SiteName}/repository/commits/${last_commit}"
+    ./fetch-from-web-gitlab \"$CommitInfoUrl\" $FileInfoTmp
+    FileDate=`jq -r .authored_date $FileInfoTmp`
+    let seconds=`date +%s`-`date +%s -d "$FileDate"`
     let cvsFileAgeInHours=$seconds/3600
     echo "GITConfigFileAge: ${cvsFileAgeInHours} hours"
     if [ $localConfigFileAgeInHours -gt 120 ] && [ $cvsFileAgeInHours -gt 120 ] ; then
@@ -210,7 +213,7 @@ echo "LocalTFCAge: ${localTFCfileAgeInHours} hours"
 
 asCvs=1
 TFCfileName=`echo $TFCPath | awk -F'/' '{print $NF}'`
-cvsUrl="https://gitlab.cern.ch/SITECONF/${SiteName}/raw/master/PhEDEx/${TFCfileName}"
+cvsUrl="https://gitlab.cern.ch/api/v4/projects/siteconf%2F${SiteName}/repository/files/PhEDEx%2F${TFCfileName}/raw?ref=master"
 TfcFileFromCVS=`mktemp /tmp/storage-from-CVS.xml.XXXX`
 echo "TFCGITCopy: $cvsUrl"
 ./fetch-from-web-gitlab \"$cvsUrl\" $TfcFileFromCVS
@@ -221,12 +224,15 @@ if [ $rc == 0 ] ; then
   if [ $? == 1 ] ; then asCvs=0; fi
   if [ $asCvs == 0 ] ; then
     noTFCDump=1
-    cvsMarkupUrl="https://gitlab.cern.ch/SITECONF/${SiteName}/blob/master/PhEDEx/${TFCfileName}"
-    TfcFileMarkup=`mktemp /tmp/TFC-from-CVS-Markup.xml.XXX`
+    FileInfoUrl="https://gitlab.cern.ch/api/v4/projects/siteconf%2F${SiteName}/repository/files/PhEDEx%2F${TFCfileName}?ref=master"
+    FileInfoTmp=`mktemp /tmp/TFC-from-CVS-Markup.xml.XXX`
     echo "Fetch GIT Markup copy of TFC from $cvsMarkupUrl"
-    ./fetch-from-web-gitlab \"$cvsMarkupUrl\" $TfcFileMarkup
-    cvsFileDate=`/bin/awk '{i=index($0,"datetime=");if(i>0){s=substr($0,i+10,4) "-" substr($0,i+15,2) "-" substr($0,i+18,2) " " substr($0,i+21,2) ":" substr($0,i+24,2) ":" substr($0,i+27,2) "Z";print s}}' $TfcFileMarkup`
-    let seconds=`date +%s`-`date +%s -d "$cvsFileDate"`
+    ./fetch-from-web-gitlab \"$FileInfoUrl\" $FileInfoTmp
+    last_commit=`jq -r .last_commit_id $FileInfoTmp`
+    CommitInfoUrl="https://gitlab.cern.ch/api/v4/projects/siteconf%2F${SiteName}/repository/commits/${last_commit}"
+    ./fetch-from-web-gitlab \"$CommitInfoUrl\" $FileInfoTmp
+    FileDate=`jq -r .authored_date $FileInfoTmp`
+    let seconds=`date +%s`-`date +%s -d "$FileDate"`
     let cvsFileAgeInHours=$seconds/3600
     echo "GITTFCAge: ${cvsFileAgeInHours} hours"
     if [ $localTFCfileAgeInHours -gt 120 ] && [ $cvsFileAgeInHours -gt 120 ] ; then

SiteTests/testjob/tests/CE-cms-xrootd-fallback

@@ -340,9 +340,7 @@ def main():
         rndm_site = (rndm_site + 1) % len(CSWNFB_SITES)
     log.info("Fallback site: %s" % CSWNFB_SITES[rndm_site])
 
-    xrootd_file = "/store/test/xrootd/" + CSWNFB_SITES[rndm_site] + CSWNFB_FILES[rndm_file]
-
-    log.info("Xrootd file we will test: %s" % xrootd_file)
+    log.info("Xrootd file we will test: %s" % CSWNFB_FILES[rndm_file])
 
     shutil.rmtree("xrootd-fallback", 1)
     try:
@@ -366,16 +364,19 @@ def main():
     if opts.verbose:
         level = 2
 
-    fd = open("test_xrootd.py", "w")
-    fd.write(cms_file % (xrootd_file, level))
-    fd.close()
-
     exitCode = evalCommand("scramv1", ["runtime", "-sh"], opts)
     if exitCode:
         return print_summary("Failure when setting up scramv1 runtime (CMSSW %s, SCRAM_ARCH=%s)" % (cmssw_version, scram_arch), NAG_CRITICAL)
 
     no_trial = 0
     while ( no_trial < 3 ):
+        xrootd_file = "/store/test/xrootd/" + CSWNFB_SITES[rndm_site] + CSWNFB_FILES[rndm_file]
+        log.info("Xrootd fullpath: %s" % xrootd_file)
+
+        fd = open("test_xrootd.py", "w")
+        fd.write(cms_file % (xrootd_file, level))
+        fd.close()
+
         stdout, exitCode = runCommand("cmsRun", ["test_xrootd.py"], opts, combineStd=True)
         no_trial += 1
         for line in stdout.split('\n'):

SiteTests/testjob/tests/fetch-from-web-gitlab

@@ -49,7 +49,7 @@ if [ $useSquid == 1 ] ; then
 fi
 
 wgetOutput=`mktemp`
-cmd="wget --header=\"PRIVATE-TOKEN: 6STzvBkTLS7fqevzuehi\" -O $file $url"
+cmd="curl --request GET --header \"PRIVATE-TOKEN: kuD9s3WrDGNXZ8BvxFV2\" $url -o $file"
 
 cmdFile=`mktemp`
 echo "$cmd > $wgetOutput 2>&1" > $cmdFile