Commit 65b49a59 authored by Simon Spannagel's avatar Simon Spannagel
Browse files

Merge branch 'port_jobsub_python3' into 'master'

Port jobsub python3

See merge request !406
parents 388589ee 0829a7fa
Pipeline #2507777 failed with stages
in 17 minutes and 5 seconds
#!/usr/bin/env python2
#!/usr/bin/env python3
"""
jobsub: a tool for Corryvreckan job submission
......@@ -98,13 +98,13 @@ def loadparamsfromcsv(csvfilename, runs):
def rewind(self):
self.f.seek(0)
self.linecount = 0
def next(self):
line = self.f.next()
def __next__(self):
line = self.f.__next__()
self.linecount += 1
while line.startswith(self.commentstring) or not line.strip(): # test if line commented or empty
line = self.f.next()
line = self.f.__next__()
self.linecount += 1
return line
return str(line)
def __iter__(self):
return self
......@@ -117,14 +117,15 @@ def loadparamsfromcsv(csvfilename, runs):
exit(1)
try:
log.debug("Opening csv file '"+csvfilename+"'.")
csvfile = open(csvfilename, 'rb')
csvfile = open(csvfilename, 'rt')
filteredfile = CommentedFile(csvfile)
try:
# construct a sample for the csv format sniffer:
sample = ""
try:
while (len(sample)<1024):
sample += filteredfile.next()
sample += filteredfile.__next__()
except StopIteration:
log.debug("End of csv file reached, sample limited to " + str(len(sample))+ " bytes")
dialect = csv.Sniffer().sniff(sample) # test csv file format details
......@@ -132,7 +133,8 @@ def loadparamsfromcsv(csvfilename, runs):
log.debug("Determined the CSV dialect as follows: delimiter=%s, doublequote=%s, escapechar=%s, lineterminator=%s, quotechar=%s , quoting=%s, skipinitialspace=%s", dialect.delimiter, dialect.doublequote, dialect.escapechar, list(ord(c) for c in dialect.lineterminator), dialect.quotechar, dialect.quoting, dialect.skipinitialspace)
filteredfile.rewind() # back to beginning of file
reader = csv.DictReader(filteredfile, dialect=dialect) # now process CSV file contents here and load them into memory
reader.next() # python < 2.6 requires an actual read access before filling 'DictReader.fieldnames'
reader.__next__() # python requires an actual read access before filling 'DictReader.fieldnames'
log.debug("CSV file contains the header info: %s", reader.fieldnames)
try:
reader.fieldnames = [field.lower() for field in reader.fieldnames] # convert to lower case keys to avoid confusion
......@@ -148,7 +150,7 @@ def loadparamsfromcsv(csvfilename, runs):
log.info("Successfully loaded csv file'"+csvfilename+"'.")
# first: search through csv file to find corresponding runnumber entry line for every run
filteredfile.rewind() # back to beginning of file
reader.next() # .. and skip the header line
reader.__next__() # .. and skip the header line
missingRuns = list(runs) # list of runs to look for in csv file
cnt = 0
......@@ -172,8 +174,8 @@ def loadparamsfromcsv(csvfilename, runs):
log.error("Could not find an entry for the following run numbers in '"+csvfilename+"': "+', '.join(map(str, missingRuns)))
finally:
csvfile.close()
except csv.Error, e:
log.error("Problem loading the csv file '"+csvfilename+"'(%s): %s"%(e.errno, e.strerror))
except csv.Error as e:
log.error("Problem loading the csv file '"+csvfilename+"': %s"%e)
exit(1)
return parameters_csv
......@@ -206,7 +208,7 @@ def runCorryvreckan(filenamebase, jobtask, silent):
log.debug("Found Corryvreckan executable: " + cmd)
else:
log.error("Corryvreckan executable not found in PATH!")
log.error(os.getcwd())
log.error(os.getcwd())
exit(1)
# search for stdbuf command: adjust stdout buffering
......@@ -312,8 +314,8 @@ def runCorryvreckan(filenamebase, jobtask, silent):
finally:
log_file.close()
rcode = p.returncode # get the return code
except OSError, e:
log.critical("Problem with Corryvreckan execution: Command '%s' resulted in error #%s, %s", cmd, e.errno, e.strerror)
except OSError as e:
log.critical("Problem with Corryvreckan execution: Command '%s' resulted in error %s", cmd, e)
exit(1)
return rcode
......@@ -355,8 +357,8 @@ def submitCondor(filenamebase, subfile, runnr):
log.info ("Now submitting Corryvreckan job: "+filenamebase+".conf to HTCondor")
log.debug ("Executing: "+cmd)
os.popen(cmd)
except OSError, e:
log.critical("Problem with HTCondor submission: Command '%s' resulted in error #%s, %s", cmd, e.errno, e.strerror)
except OSError as e:
log.critical("Problem with HTCondor submission: Command '%s' resulted in error %s", cmd, e)
exit(1)
return 0
......@@ -406,7 +408,7 @@ def main(argv=None):
log.error=callcounted(log.error)
import os.path
import ConfigParser
import configparser
try:
import argparse
except ImportError:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment