Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
etf
cmssam
Commits
8f60afa0
Commit
8f60afa0
authored
Jan 24, 2019
by
Andrea Sciaba
Browse files
Fixes to git access and to xrootd fallback
parent
81bdb124
Changes
3
Hide whitespace changes
Inline
Side-by-side
SiteTests/testjob/tests/CE-cms-basic
View file @
8f60afa0
...
...
@@ -164,7 +164,7 @@ let localConfigFileAgeInHours=$seconds/3600
echo
"SiteLocalConfigAge:
${
localConfigFileAgeInHours
}
hours"
asCvs
=
1
cvsUrl
=
"https://gitlab.cern.ch/
SITECONF/
${
SiteName
}
/raw/master
/JobConfig
/
site-local-config
${
subSiteName
}
.xml"
cvsUrl
=
"https://gitlab.cern.ch/
api/v4/projects/siteconf%2F
${
SiteName
}
/repository/files
/JobConfig
%2F
site-local-config
${
subSiteName
}
.xml
/raw?ref=master
"
ConfigFileFromCVS
=
`
mktemp
/tmp/site-local-config-from-CVS.xml.XXX
`
echo
"GITCopy:
$cvsUrl
"
./fetch-from-web-gitlab
\"
$cvsUrl
\"
$ConfigFileFromCVS
...
...
@@ -174,11 +174,14 @@ if [ $rc == 0 ] ; then
if
[
$?
==
1
]
;
then
asCvs
=
0
;
fi
if
[
$asCvs
==
0
]
;
then
noConfigDump
=
1
cvsMarkupUrl
=
"https://gitlab.cern.ch/SITECONF/
${
SiteName
}
/blob/master/JobConfig/site-local-config
${
subSiteName
}
.xml"
ConfigFileMarkup
=
`
mktemp
/tmp/site-local-config-from-CVS-Markup.xml.XXX
`
./fetch-from-web-gitlab
\"
$cvsMarkupUrl
\"
$ConfigFileMarkup
cvsFileDate
=
`
/bin/awk
'{i=index($0,"datetime=");if(i>0){s=substr($0,i+10,4) "-" substr($0,i+15,2) "-" substr($0,i+18,2) " " substr($0,i+21,2) ":" substr($0,i+24,2) ":" substr($0,i+27,2) "Z";print s}}'
$ConfigFileMarkup
`
let
seconds
=
`
date
+%s
`
-
`
date
+%s
-d
"
$cvsFileDate
"
`
FileInfoUrl
=
"https://gitlab.cern.ch/api/v4/projects/siteconf%2F
${
SiteName
}
/repository/files/JobConfig%2Fsite-local-config
${
subSiteName
}
.xml?ref=master"
FileInfoTmp
=
`
mktemp
/tmp/site-local-config-from-CVS-Markup.xml.XXX
`
./fetch-from-web-gitlab
\"
$FileInfoUrl
\"
$FileInfoTmp
last_commit
=
`
jq
-r
.last_commit_id
$FileInfoTmp
`
CommitInfoUrl
=
"https://gitlab.cern.ch/api/v4/projects/siteconf%2F
${
SiteName
}
/repository/commits/
${
last_commit
}
"
./fetch-from-web-gitlab
\"
$CommitInfoUrl
\"
$FileInfoTmp
FileDate
=
`
jq
-r
.authored_date
$FileInfoTmp
`
let
seconds
=
`
date
+%s
`
-
`
date
+%s
-d
"
$FileDate
"
`
let
cvsFileAgeInHours
=
$seconds
/3600
echo
"GITConfigFileAge:
${
cvsFileAgeInHours
}
hours"
if
[
$localConfigFileAgeInHours
-gt
120
]
&&
[
$cvsFileAgeInHours
-gt
120
]
;
then
...
...
@@ -210,7 +213,7 @@ echo "LocalTFCAge: ${localTFCfileAgeInHours} hours"
asCvs
=
1
TFCfileName
=
`
echo
$TFCPath
|
awk
-F
'/'
'{print $NF}'
`
cvsUrl
=
"https://gitlab.cern.ch/
SITECONF/
${
SiteName
}
/raw/master
/PhEDEx
/
${
TFCfileName
}
"
cvsUrl
=
"https://gitlab.cern.ch/
api/v4/projects/siteconf%2F
${
SiteName
}
/repository/files
/PhEDEx
%2F
${
TFCfileName
}
/raw?ref=master
"
TfcFileFromCVS
=
`
mktemp
/tmp/storage-from-CVS.xml.XXXX
`
echo
"TFCGITCopy:
$cvsUrl
"
./fetch-from-web-gitlab
\"
$cvsUrl
\"
$TfcFileFromCVS
...
...
@@ -221,12 +224,15 @@ if [ $rc == 0 ] ; then
if
[
$?
==
1
]
;
then
asCvs
=
0
;
fi
if
[
$asCvs
==
0
]
;
then
noTFCDump
=
1
cvsMarkup
Url
=
"https://gitlab.cern.ch/
SITECONF/
${
SiteName
}
/blob/master
/PhEDEx
/
${
TFCfileName
}
"
Tfc
File
Marku
p
=
`
mktemp
/tmp/TFC-from-CVS-Markup.xml.XXX
`
FileInfo
Url
=
"https://gitlab.cern.ch/
api/v4/projects/siteconf%2F
${
SiteName
}
/repository/files
/PhEDEx
%2F
${
TFCfileName
}
?ref=master
"
File
InfoTm
p
=
`
mktemp
/tmp/TFC-from-CVS-Markup.xml.XXX
`
echo
"Fetch GIT Markup copy of TFC from
$cvsMarkupUrl
"
./fetch-from-web-gitlab
\"
$cvsMarkupUrl
\"
$TfcFileMarkup
cvsFileDate
=
`
/bin/awk
'{i=index($0,"datetime=");if(i>0){s=substr($0,i+10,4) "-" substr($0,i+15,2) "-" substr($0,i+18,2) " " substr($0,i+21,2) ":" substr($0,i+24,2) ":" substr($0,i+27,2) "Z";print s}}'
$TfcFileMarkup
`
let
seconds
=
`
date
+%s
`
-
`
date
+%s
-d
"
$cvsFileDate
"
`
./fetch-from-web-gitlab
\"
$FileInfoUrl
\"
$FileInfoTmp
last_commit
=
`
jq
-r
.last_commit_id
$FileInfoTmp
`
CommitInfoUrl
=
"https://gitlab.cern.ch/api/v4/projects/siteconf%2F
${
SiteName
}
/repository/commits/
${
last_commit
}
"
./fetch-from-web-gitlab
\"
$CommitInfoUrl
\"
$FileInfoTmp
FileDate
=
`
jq
-r
.authored_date
$FileInfoTmp
`
let
seconds
=
`
date
+%s
`
-
`
date
+%s
-d
"
$FileDate
"
`
let
cvsFileAgeInHours
=
$seconds
/3600
echo
"GITTFCAge:
${
cvsFileAgeInHours
}
hours"
if
[
$localTFCfileAgeInHours
-gt
120
]
&&
[
$cvsFileAgeInHours
-gt
120
]
;
then
...
...
SiteTests/testjob/tests/CE-cms-xrootd-fallback
View file @
8f60afa0
...
...
@@ -340,9 +340,7 @@ def main():
rndm_site
=
(
rndm_site
+
1
)
%
len
(
CSWNFB_SITES
)
log
.
info
(
"Fallback site: %s"
%
CSWNFB_SITES
[
rndm_site
])
xrootd_file
=
"/store/test/xrootd/"
+
CSWNFB_SITES
[
rndm_site
]
+
CSWNFB_FILES
[
rndm_file
]
log
.
info
(
"Xrootd file we will test: %s"
%
xrootd_file
)
log
.
info
(
"Xrootd file we will test: %s"
%
CSWNFB_FILES
[
rndm_file
])
shutil
.
rmtree
(
"xrootd-fallback"
,
1
)
try
:
...
...
@@ -366,16 +364,19 @@ def main():
if
opts
.
verbose
:
level
=
2
fd
=
open
(
"test_xrootd.py"
,
"w"
)
fd
.
write
(
cms_file
%
(
xrootd_file
,
level
))
fd
.
close
()
exitCode
=
evalCommand
(
"scramv1"
,
[
"runtime"
,
"-sh"
],
opts
)
if
exitCode
:
return
print_summary
(
"Failure when setting up scramv1 runtime (CMSSW %s, SCRAM_ARCH=%s)"
%
(
cmssw_version
,
scram_arch
),
NAG_CRITICAL
)
no_trial
=
0
while
(
no_trial
<
3
):
xrootd_file
=
"/store/test/xrootd/"
+
CSWNFB_SITES
[
rndm_site
]
+
CSWNFB_FILES
[
rndm_file
]
log
.
info
(
"Xrootd fullpath: %s"
%
xrootd_file
)
fd
=
open
(
"test_xrootd.py"
,
"w"
)
fd
.
write
(
cms_file
%
(
xrootd_file
,
level
))
fd
.
close
()
stdout
,
exitCode
=
runCommand
(
"cmsRun"
,
[
"test_xrootd.py"
],
opts
,
combineStd
=
True
)
no_trial
+=
1
for
line
in
stdout
.
split
(
'
\n
'
):
...
...
SiteTests/testjob/tests/fetch-from-web-gitlab
View file @
8f60afa0
...
...
@@ -49,7 +49,7 @@ if [ $useSquid == 1 ] ; then
fi
wgetOutput
=
`
mktemp
`
cmd
=
"
wget
--header
=
\"
PRIVATE-TOKEN:
6STzvBkTLS7fqevzuehi
\"
-O
$file
$url
"
cmd
=
"
curl --request GET
--header
\"
PRIVATE-TOKEN:
kuD9s3WrDGNXZ8BvxFV2
\"
$url
-o
$file
"
cmdFile
=
`
mktemp
`
echo
"
$cmd
>
$wgetOutput
2>&1"
>
$cmdFile
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment