Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
etf
cmssam
Commits
d7fe842f
Commit
d7fe842f
authored
Feb 28, 2017
by
Andrea Sciaba
Browse files
New gfal2 plugin
parent
ffa83188
Changes
3
Hide whitespace changes
Inline
Side-by-side
SiteTests/MonteCarlo/cms-MC-test/lib/python2.6/site-packages/WMCore/Storage/Backends/GFAL2Impl.py
View file @
d7fe842f
...
...
@@ -4,6 +4,9 @@ _GFAL2Impl_
Implementation of StageOutImpl interface for gfal-copy
"""
import
os
import
argparse
import
logging
from
WMCore.Storage.Registry
import
registerStageOutImpl
from
WMCore.Storage.StageOutImpl
import
StageOutImpl
from
WMCore.Storage.Execute
import
runCommandWithOutput
as
runCommand
...
...
@@ -20,20 +23,43 @@ class GFAL2Impl(StageOutImpl):
def
__init__
(
self
,
stagein
=
False
):
StageOutImpl
.
__init__
(
self
,
stagein
)
self
.
removeCommand
=
"env -i X509_USER_PROXY=$X509_USER_PROXY gfal-rm -vvv %s"
self
.
copyCommand
=
"env -i X509_USER_PROXY=$X509_USER_PROXY gfal-copy -t 2400 -T 2400 -p -vvv"
# If we want to execute commands in clean shell, we can`t separate them with ';'.
# Next commands after separation are executed without env -i and this leads us with
# mixed environment with COMP and system python.
# GFAL2 is not build under COMP environment and it had failures with mixed environment.
self
.
setups
=
"env -i X509_USER_PROXY=$X509_USER_PROXY JOBSTARTDIR=$JOBSTARTDIR bash -c '%s'"
self
.
removeCommand
=
self
.
setups
%
'. $JOBSTARTDIR/startup_environment.sh 2> /dev/null; date; gfal-rm -t 600 %s '
self
.
copyCommand
=
self
.
setups
%
'. $JOBSTARTDIR/startup_environment.sh 2> /dev/null; date; gfal-copy -t 2400 -T 2400 -p %(checksum)s %(options)s %(source)s %(destination)s'
def
createSourceName
(
self
,
protocol
,
pfn
):
def
createFinalPFN
(
self
,
pfn
):
"""
_create
SourceName
_
GFAL2
us
es file:///
url
s
_create
FinalPFN
_
GFAL2
requir
es file:///
for any direction transfer
s
"""
if
pfn
.
startswith
(
'file:'
):
return
pfn
elif
os
.
path
.
isfile
(
pfn
):
return
"file://%s"
%
os
.
path
.
abspath
(
pfn
)
else
:
return
pfn
elif
pfn
.
startswith
(
'/'
):
return
"file://%s"
%
os
.
path
.
abspath
(
pfn
)
return
pfn
def
createSourceName
(
self
,
protocol
,
pfn
):
"""
_createSourceName_
GFAL2 uses file:/// urls
"""
return
self
.
createFinalPFN
(
pfn
)
def
createTargetName
(
self
,
protocol
,
pfn
):
"""
_createTargetName_
GFAL2 uses file:// urls
"""
return
self
.
createFinalPFN
(
pfn
)
def
createOutputDirectory
(
self
,
targetPFN
):
...
...
@@ -47,33 +73,56 @@ class GFAL2Impl(StageOutImpl):
def
createRemoveFileCommand
(
self
,
pfn
):
"""
_createRemoveFileCommand_
handle file remove using gfal-rm
gfal-rm options used:
-t global timeout for the execution of the command.
Command is interrupted if time expires before it finishes
"""
if
pfn
.
startswith
(
"file:"
):
return
self
.
removeCommand
%
pfn
elif
os
.
path
.
isfile
(
pfn
):
if
os
.
path
.
isfile
(
pfn
):
return
"/bin/rm -f %s"
%
os
.
path
.
abspath
(
pfn
)
else
:
return
self
.
removeCommand
%
pfn
return
self
.
removeCommand
%
self
.
createFinalPFN
(
pfn
)
def
createStageOutCommand
(
self
,
sourcePFN
,
targetPFN
,
options
=
None
,
checksums
=
None
):
"""
_createStageOutCommand_
Build an gfal-copy command
Build a gfal-copy command
gfal-copy options used:
-t maximum time for the operation to terminate
-T global timeout for the transfer operation
-p if the destination directory does not exist, create it
-K checksum algorithm to use, or algorithm:value
"""
result
=
"#!/bin/sh
\n
"
result
=
"#!/bin/bash
\n
"
copyCommandDict
=
{
'checksum'
:
''
,
'options'
:
''
,
'source'
:
''
,
'destination'
:
''
}
useChecksum
=
(
checksums
is
not
None
and
'adler32'
in
checksums
and
not
self
.
stageIn
)
if
not
options
:
options
=
''
parser
=
argparse
.
ArgumentParser
()
parser
.
add_argument
(
'--nochecksum'
,
action
=
'store_true'
)
args
,
unknown
=
parser
.
parse_known_args
(
options
.
split
())
if
not
args
.
nochecksum
:
if
useChecksum
:
checksums
[
'adler32'
]
=
"%08x"
%
int
(
checksums
[
'adler32'
],
16
)
copyCommandDict
[
'checksum'
]
=
"-K adler32:%s"
%
checksums
[
'adler32'
]
else
:
copyCommandDict
[
'checksum'
]
=
"-K adler32"
useChecksum
=
(
checksums
!=
None
and
'adler32'
in
checksums
and
not
self
.
stageI
n
)
copyCommandDict
[
'options'
]
=
' '
.
join
(
unknow
n
)
copyCommand
=
self
.
copyCommand
if
useChecksum
:
copyCommand
+=
" -K adler32 "
if
options
!=
None
:
copyCommand
+=
" %s "
%
options
copyCommand
+=
" %s "
%
sourcePFN
copyCommand
+=
" %s
\n
"
%
targetPFN
copyCommandDict
[
'source'
]
=
self
.
createFinalPFN
(
sourcePFN
)
copyCommandDict
[
'destination'
]
=
self
.
createFinalPFN
(
targetPFN
)
copyCommand
=
self
.
copyCommand
%
copyCommandDict
result
+=
copyCommand
if
_CheckExitCodeOption
:
...
...
@@ -97,13 +146,10 @@ class GFAL2Impl(StageOutImpl):
_removeFile_
CleanUp pfn provided
"""
command
=
""
if
os
.
path
.
isfile
(
pfnToRemove
):
command
=
"/bin/rm -f %s"
%
os
.
path
.
abspath
(
pfnToRemove
)
if
pfnToRemove
.
startswith
(
"file:"
):
command
=
self
.
removeCommand
%
pfnToRemove
else
:
command
=
self
.
removeCommand
%
pfnToRemove
command
=
self
.
removeCommand
%
self
.
createFinalPFN
(
pfnToRemove
)
self
.
executeCommand
(
command
)
...
...
SiteTests/MonteCarlo/tests/CE-cms-mc
View file @
d7fe842f
...
...
@@ -60,8 +60,10 @@ fi
export
PYTHONPATH
=
$SAME_SENSOR_HOME
/cms-MC-test/lib/python2.6/site-packages:
$PYTHONPATH
#echo "PYTHONPATH=$PYTHONPATH"
#echo
set
>
startup_environment.sh
sed
-e
's/^/export /'
startup_environment.sh
>
tmp_env.sh
mv
-f
tmp_env.sh startup_environment.sh
export
JOBSTARTDIR
=
$PWD
cd
$SAME_SENSOR_HOME
/cms-MC-test/
if
[
$?
-ne
0
]
;
then
...
...
nagios/grid-monitoring-probes-org.cms-etf.spec
View file @
d7fe842f
...
...
@@ -4,7 +4,7 @@
Summary: WLCG Compliant Probes from %{site}
Name: nagios-plugins-wlcg-org.cms
Version: 1.1.3
7
Version: 1.1.3
9
Release: 1%{?dist}
License: GPL
...
...
@@ -49,6 +49,10 @@ install --directory %{buildroot}/etc/cron.d
/etc/cron.d/cms_glexec
%changelog
* Tue Feb 28 2017 Andrea Sciaba <Andrea.Sciaba@cern.ch> 1.1.39-1.
- included the latest version of the GFAL2 plugin for WMCore
* Tue Feb 14 2017 Andrea Sciaba <Andrea.Sciaba@cern.ch> 1.1.38-1.
- fixed spacetoken bug in SRM test
* Wed Feb 1 2017 Andrea Sciaba <Andrea.Sciaba@cern.ch> 1.1.37-1.
- new etf_plugin_cms.py to support new VO feed
* Thu Jan 26 2017 Andrea Sciaba <Andrea.Sciaba@cern.ch> 1.1.36-1.
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment