From 74fe805f5eef52864c059162c6a7f76612b3da07 Mon Sep 17 00:00:00 2001
From: James Beacham <j.beacham@cern.ch>
Date: Mon, 14 May 2018 19:44:53 +0000
Subject: [PATCH] Merge branch '21.0-use-python-zip' into '21.0'
Removed dependency on zip binary from Archiving transform (ATLASJT-296)
See merge request atlas/athena!11230
(cherry picked from commit 04559265e8b0bfd294e450d571318101de87c931 [formerly 86ce1f3252312edadcf3ba33009439148ad5a010])
6d177b69 Removed dependency on zip binary from Archiving transform (ATLASJT-296)
Former-commit-id: d689c169c07d88e4c6512a1806b5ddbad81866c1
---
Tools/PyJobTransforms/python/trfExe.py | 28 ++++++++++++++-------
Tools/PyJobTransforms/scripts/Archive_tf.py | 4 ---
2 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/Tools/PyJobTransforms/python/trfExe.py b/Tools/PyJobTransforms/python/trfExe.py
index 77735112572..83ac6e4c734 100755
--- a/Tools/PyJobTransforms/python/trfExe.py
+++ b/Tools/PyJobTransforms/python/trfExe.py
@@ -658,7 +658,6 @@ class scriptExecutor(transformExecutor):
try:
p = subprocess.Popen(self._cmd, shell = False, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, bufsize = 1)
-
if self._memMonitor:
try:
self._memSummaryFile = 'mem.summary.' + self._name + '.json'
@@ -1888,12 +1887,23 @@ class archiveExecutor(scriptExecutor):
elif self.conf.argdict['compressionType'] == 'none':
pass
elif self._exe == 'zip':
- self._cmd = [self._exe]
- if 'compressionLevel' in self.conf.argdict:
- self._cmd.append(self.conf.argdict['compressionLevel'])
- self._cmd.extend([self.conf.argdict['outputArchFile'].value[0]])
- if '.' not in self.conf.argdict['outputArchFile'].value[0]:
- errmsg = 'Output filename must end in ".", ".zip" or ".anyname" '
- raise trfExceptions.TransformExecutionException(trfExit.nameToCode('TRF_OUTPUT_FILE_ERROR'), errmsg)
- self._cmd.extend(self.conf.argdict['inputDataFile'].value)
+ self._cmd = ['python']
+ try:
+ with open('zip_wrapper.py', 'w') as zip_wrapper:
+ print >> zip_wrapper, "import zipfile"
+ print >> zip_wrapper, "zf = zipfile.ZipFile('{}', mode='w', allowZip64=True)".format(self.conf.argdict['outputArchFile'].value[0])
+ print >> zip_wrapper, "for f in {}:".format(self.conf.argdict['inputDataFile'].value)
+ print >> zip_wrapper, " print 'Zipping file {}'.format(f)"
+ print >> zip_wrapper, " zf.write(f, compress_type=zipfile.ZIP_STORED)"
+ print >> zip_wrapper, "zf.close()"
+ os.chmod('zip_wrapper.py', 0755)
+ except (IOError, OSError) as e:
+ errMsg = 'error writing zip wrapper {fileName}: {error}'.format(fileName = 'zip_wrapper.py',
+ error = e
+ )
+ msg.error(errMsg)
+ raise trfExceptions.TransformExecutionException(trfExit.nameToCode('TRF_EXEC_SETUP_WRAPPER'),
+ errMsg
+ )
+ self._cmd.append('zip_wrapper.py')
super(archiveExecutor, self).preExecute(input=input, output=output)
diff --git a/Tools/PyJobTransforms/scripts/Archive_tf.py b/Tools/PyJobTransforms/scripts/Archive_tf.py
index d8afd52958d..d943496b1ce 100755
--- a/Tools/PyJobTransforms/scripts/Archive_tf.py
+++ b/Tools/PyJobTransforms/scripts/Archive_tf.py
@@ -46,7 +46,6 @@ def addMyArgs(parser):
# Use arggroup to get these arguments in their own sub-section (of --help)
parser.defineArgGroup('Archive_tf', 'Archive transform options')
parser.defineArgGroup('Tar archiver', 'Options')
- parser.defineArgGroup('Zip archiver', 'Options')
parser.add_argument('--exe', group='Archive_tf',
help='Archiving command. Default is zip', choices=['zip', 'tar'],
default='zip')
@@ -59,9 +58,6 @@ def addMyArgs(parser):
parser.add_argument('--compressionType', group='Tar archiver',
help='Underlying compression type of tar. Default is none', choices=['gzip', 'bzip2', 'none'],
default='none')
- parser.add_argument('--compressionLevel', group='Zip archiver',
- help='Compression level of zip. Default is -0', choices=['-0', '-1', '-2', '-3', '-4', '-5', '-6', '-7', '-8', '-9'],
- default='-0')
if __name__ == '__main__':
main()
--
GitLab