Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
machinejobfeatures
mjf-scripts
Commits
7c693324
Commit
7c693324
authored
May 03, 2016
by
Andrew McNab
Browse files
Catch up
parent
e11fbdac
Changes
6
Hide whitespace changes
Inline
Side-by-side
VERSION
View file @
7c693324
VERSION=00.0
8
VERSION=00.
1
0
WN-mjf.py
View file @
7c693324
...
...
@@ -22,7 +22,7 @@ class mjf:
'job_id'
:
self
.
ErrorCode
,
'wall_limit_secs'
:
self
.
ErrorCode
,
'cpu_limit_secs'
:
self
.
ErrorCode
,
'max_rss_bytes'
:
self
.
Error
Code
,
'max_rss_bytes'
:
self
.
Warning
Code
,
'max_swap_bytes'
:
self
.
WarningCode
,
'scratch_limit_bytes'
:
self
.
WarningCode
}
}
...
...
epilogue.user
View file @
7c693324
...
...
@@ -32,8 +32,18 @@
# POSSIBILITY OF SUCH DAMAGE.
# Remove $JOBFEATURES directory for this job where the directory
# is at
/tmp
/jobfeatures-$PBS_JOBID
($1 is $PBS_JOBID)
# is at
$mjf_tmp_dir/mjf-$USER
/jobfeatures-$PBS_JOBID ($1 is $PBS_JOBID)
if
[
"
$1
"
!=
""
-a
-d
"/tmp/jobfeatures-
$1
"
-a
-O
"/tmp/jobfeatures-
$1
"
]
;
then
rm
-Rf
"/tmp/jobfeatures-
$1
"
mjf_tmp_dir
=
/tmp
if
[
-r
/etc/sysconfig/mjf
]
;
then
.
/etc/sysconfig/mjf
fi
if
[
-r
/var/run/mjf
]
;
then
.
/etc/sysconfig/mjf
fi
if
[
"
$1
"
!=
""
-a
-d
"
$mjf_tmp_dir
/mjf-
$USER
/jobfeatures-
$1
"
]
;
then
rm
-Rf
"
$mjf_tmp_dir
/mjf-
$USER
/jobfeatures-
$1
"
fi
mjf-torque.csh
View file @
7c693324
mjf_tmp_dir=/tmp
if ( -r /etc/sysconfig/mjf ) then
source /var/sysconfig/mjf
endif
if ( -r /var/run/mjf ) then
source /var/run/mjf
endif
if ( -d /etc/machinefeatures ) then
setenv MACHINEFEATURES /etc/machinefeatures
endif
test
if ( "$PBS_JOBID" != "" && -d "
/tmp
/jobfeatures-$PBS_JOBID" ) then
setenv JOBFEATURES "
/tmp
/jobfeatures-$PBS_JOBID"
if ( "$PBS_JOBID" != "" && -d "
$mjf_tmp_dir/mjf-$USER
/jobfeatures-$PBS_JOBID" ) then
setenv JOBFEATURES "
$mjf_tmp_dir/mjf_$USER
/jobfeatures-$PBS_JOBID"
endif
mjf-torque.sh
View file @
7c693324
mjf_tmp_dir
=
/tmp
if
[
-r
/etc/sysconfig/mjf
]
;
then
.
/etc/sysconfig/mjf
fi
if
[
-r
/var/run/mjf
]
;
then
.
/etc/sysconfig/mjf
fi
if
[
-d
/etc/machinefeatures
]
;
then
export
MACHINEFEATURES
=
/etc/machinefeatures
fi
if
[
"
$PBS_JOBID
"
!=
""
-a
-d
"
/tmp
/jobfeatures-
$PBS_JOBID
"
]
;
then
export
JOBFEATURES
=
/tmp
/jobfeatures-
$PBS_JOBID
if
[
"
$PBS_JOBID
"
!=
""
-a
-d
"
$mjf_tmp_dir
/mjf-
$USER
/jobfeatures-
$PBS_JOBID
"
]
;
then
export
JOBFEATURES
=
"
$mjf_tmp_dir
/mjf-
$USER
/jobfeatures-
$PBS_JOBID
"
fi
\ No newline at end of file
prologue.user
View file @
7c693324
...
...
@@ -35,20 +35,26 @@
# specification in HSF-TN-2016-02
#
# This script creates the $JOBFEATURES directory at
# /tmp/jobfeatures-$PBS_JOBID and attempts to populate it
# /tmp/
mjf-$USER/
jobfeatures-$PBS_JOBID and attempts to populate it
# from Torque/PBS information and from
# $MACHINEFEATURES=/etc/machinefeatures
#
# If the following variables are present in /var/run/mjf or
# /etc/sysconfig/mjf then they are preferred:
# /etc/sysconfig/mjf then they are used as defaults if not given
# by Torque/PBS:
#
# - hs06_job
# - allocated_cpu
# - wall_limit_secs
# - cpu_limit_secs
# - max_rss_bytes
# - max_swap_bytes
# - scratch_limit_bytes
# - hs06_job
#
# Values in /var/run/mjf are preferred over /etc/sysconfig/mjf.
#
# Additionally you can set mjf_tmp_dir in those files to use a
# directory other than /tmp for the mjf-$USER directories.
#
import
os
...
...
@@ -57,21 +63,67 @@ import sys
import
stat
import
time
os
.
umask
(
0022
)
try
:
job_id
=
sys
.
argv
[
1
]
except
:
print
'JOB ID not given on command line'
sys
.
exit
(
1
)
sys
.
exit
(
1
1
)
jobfeaturesDir
=
'/tmp/jobfeatures-'
+
job_id
# Values in /var/run/mjf take precedence
try
:
fromRun
=
open
(
'/var/run/mjf'
,
'r'
).
read
()
except
:
fromRun
=
''
# Also look in persistent /etc/sysconfig/mjf
try
:
fromSysconfig
=
open
(
'/etc/sysconfig/mjf'
,
'r'
).
read
()
except
:
fromSysconfig
=
''
mjfTmpDir
=
'/tmp'
if
'mjf_tmp_dir='
in
fromRun
+
fromSysconfig
:
try
:
matchObject
=
re
.
search
(
"^mjf_tmp_dir=(.*)"
,
fromRun
+
'
\n
'
+
fromSysconfig
)
mjfTmpDir
=
matchObject
.
group
(
1
)
except
:
pass
# First make sure the mjfTmpDir/mjf-$USER directory exists
try
:
os
.
mkdir
(
mjfTmpDir
+
'/mjf-'
+
sys
.
argv
[
2
])
except
:
# Ok if exists already
pass
try
:
userDirStat
=
os
.
stat
(
mjfTmpDir
+
'/mjf-'
+
sys
.
argv
[
2
])
except
:
print
mjfTmpDir
+
'/mjf-'
+
sys
.
argv
[
2
]
+
' could not be created'
sys
.
exit
(
12
)
# Check it is owned by us
if
userDirStat
.
st_uid
!=
os
.
getuid
()
or
userDirStat
.
st_gid
!=
os
.
getgid
():
print
mjfTmpDir
+
'/mjf-'
+
sys
.
argv
[
2
]
+
' has the wrong UID/GID'
sys
.
exit
(
13
)
# Ensure the right permissions
os
.
chmod
(
mjfTmpDir
+
'/mjf-'
+
sys
.
argv
[
2
],
stat
.
S_IRWXU
+
stat
.
S_IRGRP
+
stat
.
S_IXGRP
+
stat
.
S_IROTH
+
stat
.
S_IXOTH
)
# Now the directory for this job
jobfeaturesDir
=
mjfTmpDir
+
'/mjf-'
+
sys
.
argv
[
2
]
+
'/jobfeatures-'
+
job_id
try
:
os
.
mkdir
(
jobfeaturesDir
)
except
:
print
jobfeaturesDir
+
' already exists? - not overwriting'
sys
.
exit
(
0
)
else
:
os
.
chmod
(
jobfeaturesDir
,
stat
.
S_IRWXU
+
stat
.
S_IRGRP
+
stat
.
S_IXGRP
+
stat
.
S_IROTH
+
stat
.
S_IXOTH
)
print
'Failed to create '
+
jobfeaturesDir
sys
.
exit
(
14
)
#
else:
#
os.chmod(jobfeaturesDir, stat.S_IRWXU + stat.S_IRGRP + stat.S_IXGRP + stat.S_IROTH + stat.S_IXOTH)
open
(
jobfeaturesDir
+
'/job_id'
,
'w'
).
write
(
job_id
)
...
...
@@ -81,6 +133,17 @@ jobfeatures['allocated_cpu'] = 1
jobstart_secs
=
int
(
time
.
time
())
open
(
jobfeaturesDir
+
'/jobstart_secs'
,
'w'
).
write
(
str
(
jobstart_secs
))
# Get any defaults for these (integer) values
for
key
in
[
'allocated_cpu'
,
'wall_limit_secs'
,
'cpu_limit_secs'
,
'max_rss_bytes'
,
'max_swap_bytes'
,
'scratch_limit_bytes'
]:
if
key
+
'='
in
fromRun
+
fromSysconfig
:
try
:
matchObject
=
re
.
search
(
key
+
"=([0-9]*)"
,
fromRun
+
'
\n
'
+
fromSysconfig
)
jobfeatures
[
key
]
=
int
(
matchObject
.
group
(
1
))
except
:
pass
# Examine the 5th argument, for resource limits
try
:
# if just 1 processor-per-node then "nodes=1", but if 8, say, then "nodes=1:ppn=8"
...
...
@@ -119,29 +182,10 @@ else:
# Is this the right name??
jobfeatures
[
'max_rss_bytes'
]
=
rssInt
# Values in /var/run/mjf take precedence
try
:
fromRun
=
open
(
'/var/run/mjf'
,
'r'
).
read
()
except
:
fromRun
=
''
# Also look in persistent /etc/sysconfig/mjf
try
:
fromSysconfig
=
open
(
'/etc/sysconfig/mjf'
,
'r'
).
read
()
except
:
fromSysconfig
=
''
# These are all integers so handle with a list
# Write out if these have been set from files or prologue.user arguments
for
key
in
[
'allocated_cpu'
,
'wall_limit_secs'
,
'cpu_limit_secs'
,
'max_rss_bytes'
,
'max_swap_bytes'
,
'scratch_limit_bytes'
]:
if
key
+
'='
in
fromRun
+
fromSysconfig
:
try
:
matchObject
=
re
.
search
(
key
+
"=([0-9]*)"
,
fromSysconfig
+
fromRun
)
jobfeatures
[
key
]
=
int
(
matchObject
.
group
(
1
))
except
:
pass
if
key
in
jobfeatures
:
open
(
jobfeaturesDir
+
'/'
+
key
,
'w'
).
write
(
str
(
jobfeatures
[
key
]))
...
...
@@ -150,7 +194,7 @@ hs06_job = None
if
'hs06_job='
in
fromRun
+
fromSysconfig
:
try
:
matchObject
=
re
.
search
(
"hs06_job=([0-9.]*)"
,
from
Sysconfig
+
fromRun
)
matchObject
=
re
.
search
(
"hs06_job=([0-9.]*)"
,
from
Run
+
'
\n
'
+
fromSysconfig
)
hs06_job
=
float
(
matchObject
.
group
(
1
))
except
:
pass
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment