diff --git a/Tools/PyJobTransforms/python/trfExe.py b/Tools/PyJobTransforms/python/trfExe.py
index f8920425c94fb3ef56e4fa9c2cc69d70594d061e..311211946fb4440ec3825530599dc3ef78a58b3a 100755
--- a/Tools/PyJobTransforms/python/trfExe.py
+++ b/Tools/PyJobTransforms/python/trfExe.py
@@ -1869,11 +1869,8 @@ class archiveExecutor(scriptExecutor):
self.setPreExeStart()
self._memMonitor = False
- if 'exe' in self.conf.argdict:
- self._exe = self.conf.argdict['exe']
-
#unpack archived inputs
- import tarfile, zipfile
+ import zipfile
if 'inputDataFile' in self.conf.argdict:
for f in self.conf.argdict['inputDataFile'].value:
if zipfile.is_zipfile(f):
@@ -1881,57 +1878,9 @@ class archiveExecutor(scriptExecutor):
print 'Extracting input zip file {0} to temporary directory {1}'.format(f,'tmp')
archive.extractall('tmp')
archive.close()
- elif tarfile.is_tarfile(f):
- archive = tarfile.open(f, 'r:*')
- print 'Extracting input tar file {0} to temporary directory {1}'.format(f,'tmp')
- archive.extractall('tmp')
- archive.close()
-
- #proceed to archive
- if self._exe == 'tar':
- #this is needed to keep the transform from scheduling two sub-steps
- if 'outputArchFile' not in self.conf.argdict:
- raise trfExceptions.TransformExecutionException(trfExit.nameToCode('TRF_ARG_MISSING'), 'Missing output file name')
-
- self._cmd = ['python']
- try:
- with open('tar_wrapper.py', 'w') as tar_wrapper:
- print >> tar_wrapper, "import zipfile, tarfile, os, shutil"
- if os.path.exists(self.conf.argdict['outputArchFile'].value[0]):
- #appending input file(s) to existing archive. Compressed writing in append mode is not possible
- print >> tar_wrapper, "tar = tarfile.open('{}', 'a:')".format(self.conf.argdict['outputArchFile'].value[0])
- else:
- #creating new archive
- if 'compressionType' in self.conf.argdict:
- if self.conf.argdict['compressionType'] == 'gzip':
- print >> tar_wrapper, "tar = tarfile.open('{}', 'w:gz')".format(self.conf.argdict['outputArchFile'].value[0])
- elif self.conf.argdict['compressionType'] == 'bzip2':
- print >> tar_wrapper, "tar = tarfile.open('{}', 'w:bz2')".format(self.conf.argdict['outputArchFile'].value[0])
- elif self.conf.argdict['compressionType'] == 'none':
- print >> tar_wrapper, "tar = tarfile.open('{}', 'w:')".format(self.conf.argdict['outputArchFile'].value[0])
- print >> tar_wrapper, "for f in {}:".format(self.conf.argdict['inputDataFile'].value)
- print >> tar_wrapper, " if not zipfile.is_zipfile(f) and not tarfile.is_tarfile(f):"
- print >> tar_wrapper, " print 'Tarring {}'.format(os.path.basename(f))"
- print >> tar_wrapper, " tar.add(f)"
- print >> tar_wrapper, "if os.path.isdir('tmp'):"
- print >> tar_wrapper, " for root, dirs, files in os.walk('tmp'):"
- print >> tar_wrapper, " for name in files:"
- print >> tar_wrapper, " print 'Tarring {}'.format(name)"
- print >> tar_wrapper, " tar.add(os.path.join(root, name),name)"
- print >> tar_wrapper, " shutil.rmtree('tmp')"
- print >> tar_wrapper, "tar.close()"
- os.chmod('tar_wrapper.py', 0755)
- except (IOError, OSError) as e:
- errMsg = 'error writing tar wrapper {fileName}: {error}'.format(fileName = 'tar_wrapper.py',
- error = e
- )
- msg.error(errMsg)
- raise trfExceptions.TransformExecutionException(trfExit.nameToCode('TRF_EXEC_SETUP_WRAPPER'),
- errMsg
- )
- self._cmd.append('tar_wrapper.py')
- elif self._exe == 'zip':
+ #archiving
+ if self._exe == 'zip':
if 'outputArchFile' not in self.conf.argdict:
raise trfExceptions.TransformExecutionException(trfExit.nameToCode('TRF_ARG_MISSING'), 'Missing output file name')
@@ -1946,9 +1895,12 @@ class archiveExecutor(scriptExecutor):
#creating new archive
print >> zip_wrapper, "zf = zipfile.ZipFile('{}', mode='w', allowZip64=True)".format(self.conf.argdict['outputArchFile'].value[0])
print >> zip_wrapper, "for f in {}:".format(self.conf.argdict['inputDataFile'].value)
- print >> zip_wrapper, " if not zipfile.is_zipfile(f) and not tarfile.is_tarfile(f):"
+ print >> zip_wrapper, " if not zipfile.is_zipfile(f):"
print >> zip_wrapper, " print 'Zipping {}'.format(os.path.basename(f))"
print >> zip_wrapper, " zf.write(f, arcname=os.path.basename(f), compress_type=zipfile.ZIP_STORED)"
+ print >> zip_wrapper, " if os.access(f, os.F_OK):"
+ print >> zip_wrapper, " print 'Removing input file {}'.format(f)"
+ print >> zip_wrapper, " os.unlink(f)"
print >> zip_wrapper, "if os.path.isdir('tmp'):"
print >> zip_wrapper, " for root, dirs, files in os.walk('tmp'):"
print >> zip_wrapper, " for name in files:"
@@ -1967,19 +1919,18 @@ class archiveExecutor(scriptExecutor):
)
self._cmd.append('zip_wrapper.py')
+ #unarchiving
elif self._exe == 'unarchive':
- if not zipfile.is_zipfile(self.conf.argdict['inputArchFile'].value[0]) and not tarfile.is_tarfile(self.conf.argdict['inputArchFile'].value[0]):
- raise trfExceptions.TransformExecutionException(trfExit.nameToCode('TRF_INPUT_FILE_ERROR'),
- 'The input file is not a zip or tar archive - aborting unpacking')
+ for infile in self.conf.argdict['inputArchFile'].value:
+ if not zipfile.is_zipfile(infile):
+ raise trfExceptions.TransformExecutionException(trfExit.nameToCode('TRF_INPUT_FILE_ERROR'),
+ 'An input file is not a zip archive - aborting unpacking')
self._cmd = ['python']
try:
with open('unarchive_wrapper.py', 'w') as unarchive_wrapper:
- print >> unarchive_wrapper, "import zipfile, tarfile"
+ print >> unarchive_wrapper, "import zipfile"
print >> unarchive_wrapper, "for f in {}:".format(self.conf.argdict['inputArchFile'].value)
- print >> unarchive_wrapper, " if zipfile.is_zipfile(f):"
- print >> unarchive_wrapper, " archive = zipfile.ZipFile(f, mode='r')"
- print >> unarchive_wrapper, " elif tarfile.is_tarfile(f):"
- print >> unarchive_wrapper, " archive = tarfile.open(f, 'r:*')"
+ print >> unarchive_wrapper, " archive = zipfile.ZipFile(f, mode='r')"
print >> unarchive_wrapper, " path = '{}'".format(self.conf.argdict['path'])
print >> unarchive_wrapper, " print 'Extracting archive {0} to {1}'.format(f,path)"
print >> unarchive_wrapper, " archive.extractall(path)"
diff --git a/Tools/PyJobTransforms/scripts/Archive_tf.py b/Tools/PyJobTransforms/scripts/Archive_tf.py
index 2798ca5be0863301dc24b4fc24754a442662124e..83753708fc520e3bfd8145a1f736edd4453ad54d 100755
--- a/Tools/PyJobTransforms/scripts/Archive_tf.py
+++ b/Tools/PyJobTransforms/scripts/Archive_tf.py
@@ -49,9 +49,6 @@ def addMyArgs(parser):
# Use arggroup to get these arguments in their own sub-section (of --help)
parser.defineArgGroup('Archiver', 'Options')
parser.defineArgGroup('Unarchiver', 'Options')
- parser.defineArgGroup('Tar archiver', 'Options')
- parser.add_argument('--exe', group='Archiver',
- help='Archiving command, default is zip', choices=['zip', 'tar'])
parser.add_argument('--inputDataFile', '--inputFile', nargs='+',
type=trfArgClasses.argFactory(trfArgClasses.argFile, io='input', type='misc'),
help='Input file(s)', group='Archiver')
@@ -63,9 +60,6 @@ def addMyArgs(parser):
parser.add_argument('--outputArchFile', '--outputFile',
type=trfArgClasses.argFactory(trfArgClasses.argFile, io='output', type='misc'),
help='Output archive file', group='Archiver')
- parser.add_argument('--compressionType', group='Tar archiver',
- help='Underlying compression type of tar. The default is none', choices=['gzip', 'bzip2', 'none'],
- default='none')
if __name__ == '__main__':
main()