Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Corryvreckan
Corryvreckan
Commits
4d9de7a4
Commit
4d9de7a4
authored
Apr 06, 2021
by
Jens Kroeger
Browse files
start porting jobsub to python3
parent
388589ee
Changes
1
Hide whitespace changes
Inline
Side-by-side
jobsub/jobsub.py
View file @
4d9de7a4
#!/usr/bin/env python
2
#!/usr/bin/env python
3
"""
jobsub: a tool for Corryvreckan job submission
...
...
@@ -99,10 +99,10 @@ def loadparamsfromcsv(csvfilename, runs):
self
.
f
.
seek
(
0
)
self
.
linecount
=
0
def
next
(
self
):
line
=
self
.
f
.
next
(
)
line
=
next
(
self
.
f
)
self
.
linecount
+=
1
while
line
.
startswith
(
self
.
commentstring
)
or
not
line
.
strip
():
# test if line commented or empty
line
=
self
.
f
.
next
(
)
while
line
.
startswith
(
bytes
(
self
.
commentstring
,
encoding
=
'utf-8'
)
)
or
not
line
.
strip
():
# test if line commented or empty
line
=
next
(
self
.
f
)
self
.
linecount
+=
1
return
line
def
__iter__
(
self
):
...
...
@@ -119,20 +119,25 @@ def loadparamsfromcsv(csvfilename, runs):
log
.
debug
(
"Opening csv file '"
+
csvfilename
+
"'."
)
csvfile
=
open
(
csvfilename
,
'rb'
)
filteredfile
=
CommentedFile
(
csvfile
)
try
:
# construct a sample for the csv format sniffer:
sample
=
""
try
:
while
(
len
(
sample
)
<
1024
):
sample
+=
filteredfile
.
next
(
)
sample
+=
str
(
next
(
filteredfile
)
)
except
StopIteration
:
log
.
debug
(
"End of csv file reached, sample limited to "
+
str
(
len
(
sample
))
+
" bytes"
)
dialect
=
csv
.
Sniffer
().
sniff
(
sample
)
# test csv file format details
dialect
.
escapechar
=
"
\\
"
log
.
debug
(
"Determined the CSV dialect as follows: delimiter=%s, doublequote=%s, escapechar=%s, lineterminator=%s, quotechar=%s , quoting=%s, skipinitialspace=%s"
,
dialect
.
delimiter
,
dialect
.
doublequote
,
dialect
.
escapechar
,
list
(
ord
(
c
)
for
c
in
dialect
.
lineterminator
),
dialect
.
quotechar
,
dialect
.
quoting
,
dialect
.
skipinitialspace
)
filteredfile
.
rewind
()
# back to beginning of file
reader
=
csv
.
DictReader
(
filteredfile
,
dialect
=
dialect
)
# now process CSV file contents here and load them into memory
reader
.
next
()
# python < 2.6 requires an actual read access before filling 'DictReader.fieldnames'
reader
=
csv
.
DictReader
(
csvfile
,
dialect
=
dialect
)
# now process CSV file contents here and load them into memory
#next(reader) # python < 2.6 requires an actual read access before filling 'DictReader.fieldnames'
for
row
in
reader
:
print
(
row
)
log
.
debug
(
"CSV file contains the header info: %s"
,
reader
.
fieldnames
)
try
:
reader
.
fieldnames
=
[
field
.
lower
()
for
field
in
reader
.
fieldnames
]
# convert to lower case keys to avoid confusion
...
...
@@ -172,8 +177,8 @@ def loadparamsfromcsv(csvfilename, runs):
log
.
error
(
"Could not find an entry for the following run numbers in '"
+
csvfilename
+
"': "
+
', '
.
join
(
map
(
str
,
missingRuns
)))
finally
:
csvfile
.
close
()
except
csv
.
Error
,
e
:
log
.
error
(
"Problem loading the csv file '"
+
csvfilename
+
"'
(%s)
: %s"
%
(
e
.
errno
,
e
.
strerror
)
)
except
csv
.
Error
as
e
:
log
.
error
(
"Problem loading the csv file '"
+
csvfilename
+
"': %s"
%
e
)
exit
(
1
)
return
parameters_csv
...
...
@@ -206,7 +211,7 @@ def runCorryvreckan(filenamebase, jobtask, silent):
log
.
debug
(
"Found Corryvreckan executable: "
+
cmd
)
else
:
log
.
error
(
"Corryvreckan executable not found in PATH!"
)
log
.
error
(
os
.
getcwd
())
log
.
error
(
os
.
getcwd
())
exit
(
1
)
# search for stdbuf command: adjust stdout buffering
...
...
@@ -312,7 +317,7 @@ def runCorryvreckan(filenamebase, jobtask, silent):
finally
:
log_file
.
close
()
rcode
=
p
.
returncode
# get the return code
except
OSError
,
e
:
except
OSError
as
e
:
log
.
critical
(
"Problem with Corryvreckan execution: Command '%s' resulted in error #%s, %s"
,
cmd
,
e
.
errno
,
e
.
strerror
)
exit
(
1
)
return
rcode
...
...
@@ -355,7 +360,7 @@ def submitCondor(filenamebase, subfile, runnr):
log
.
info
(
"Now submitting Corryvreckan job: "
+
filenamebase
+
".conf to HTCondor"
)
log
.
debug
(
"Executing: "
+
cmd
)
os
.
popen
(
cmd
)
except
OSError
,
e
:
except
OSError
as
e
:
log
.
critical
(
"Problem with HTCondor submission: Command '%s' resulted in error #%s, %s"
,
cmd
,
e
.
errno
,
e
.
strerror
)
exit
(
1
)
return
0
...
...
@@ -406,7 +411,7 @@ def main(argv=None):
log
.
error
=
callcounted
(
log
.
error
)
import
os.path
import
C
onfig
P
arser
import
c
onfig
p
arser
try
:
import
argparse
except
ImportError
:
...
...
Jens Kroeger
@jekroege
mentioned in commit
62db832a
·
Apr 23, 2021
mentioned in commit
62db832a
mentioned in commit 62db832a775f9ee79a810cd2c2ddd4b24e10d9d2
Toggle commit list
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment