diff --git a/Tools/PyJobTransforms/python/trfExe.py b/Tools/PyJobTransforms/python/trfExe.py
index 8e2a5cea6c897981ba7222af70495f3a3b918c6c..f8920425c94fb3ef56e4fa9c2cc69d70594d061e 100755
--- a/Tools/PyJobTransforms/python/trfExe.py
+++ b/Tools/PyJobTransforms/python/trfExe.py
@@ -1862,7 +1862,7 @@ class tagMergeExecutor(scriptExecutor):
msg.debug('valStop time is {0}'.format(self._valStop))
-## @brief Archive transform - use tar
+## @brief Archive transform
class archiveExecutor(scriptExecutor):
def preExecute(self, input = set(), output = set()):
@@ -1872,26 +1872,89 @@ class archiveExecutor(scriptExecutor):
if 'exe' in self.conf.argdict:
self._exe = self.conf.argdict['exe']
+ #unpack archived inputs
+ import tarfile, zipfile
+ if 'inputDataFile' in self.conf.argdict:
+ for f in self.conf.argdict['inputDataFile'].value:
+ if zipfile.is_zipfile(f):
+ archive = zipfile.ZipFile(f, mode='r')
+ print 'Extracting input zip file {0} to temporary directory {1}'.format(f,'tmp')
+ archive.extractall('tmp')
+ archive.close()
+ elif tarfile.is_tarfile(f):
+ archive = tarfile.open(f, 'r:*')
+ print 'Extracting input tar file {0} to temporary directory {1}'.format(f,'tmp')
+ archive.extractall('tmp')
+ archive.close()
+
+ #proceed to archive
if self._exe == 'tar':
- self._cmd = [self._exe, '-c', '-v',]
- if 'compressionType' in self.conf.argdict:
- if self.conf.argdict['compressionType'] == 'gzip':
- self._cmd.append('-z')
- elif self.conf.argdict['compressionType'] == 'bzip2':
- self._cmd.append('-j')
- elif self.conf.argdict['compressionType'] == 'none':
- pass
- self._cmd.extend(['-f', self.conf.argdict['outputArchFile'].value[0]])
- self._cmd.extend(self.conf.argdict['inputDataFile'].value)
+ #this is needed to keep the transform from scheduling two sub-steps
+ if 'outputArchFile' not in self.conf.argdict:
+ raise trfExceptions.TransformExecutionException(trfExit.nameToCode('TRF_ARG_MISSING'), 'Missing output file name')
+
+ self._cmd = ['python']
+ try:
+ with open('tar_wrapper.py', 'w') as tar_wrapper:
+ print >> tar_wrapper, "import zipfile, tarfile, os, shutil"
+ if os.path.exists(self.conf.argdict['outputArchFile'].value[0]):
+ #appending input file(s) to existing archive. Compressed writing in append mode is not possible
+ print >> tar_wrapper, "tar = tarfile.open('{}', 'a:')".format(self.conf.argdict['outputArchFile'].value[0])
+ else:
+ #creating new archive
+ if 'compressionType' in self.conf.argdict:
+ if self.conf.argdict['compressionType'] == 'gzip':
+ print >> tar_wrapper, "tar = tarfile.open('{}', 'w:gz')".format(self.conf.argdict['outputArchFile'].value[0])
+ elif self.conf.argdict['compressionType'] == 'bzip2':
+ print >> tar_wrapper, "tar = tarfile.open('{}', 'w:bz2')".format(self.conf.argdict['outputArchFile'].value[0])
+ elif self.conf.argdict['compressionType'] == 'none':
+ print >> tar_wrapper, "tar = tarfile.open('{}', 'w:')".format(self.conf.argdict['outputArchFile'].value[0])
+ print >> tar_wrapper, "for f in {}:".format(self.conf.argdict['inputDataFile'].value)
+ print >> tar_wrapper, " if not zipfile.is_zipfile(f) and not tarfile.is_tarfile(f):"
+ print >> tar_wrapper, " print 'Tarring {}'.format(os.path.basename(f))"
+ print >> tar_wrapper, " tar.add(f)"
+ print >> tar_wrapper, "if os.path.isdir('tmp'):"
+ print >> tar_wrapper, " for root, dirs, files in os.walk('tmp'):"
+ print >> tar_wrapper, " for name in files:"
+ print >> tar_wrapper, " print 'Tarring {}'.format(name)"
+ print >> tar_wrapper, " tar.add(os.path.join(root, name),name)"
+ print >> tar_wrapper, " shutil.rmtree('tmp')"
+ print >> tar_wrapper, "tar.close()"
+ os.chmod('tar_wrapper.py', 0755)
+ except (IOError, OSError) as e:
+ errMsg = 'error writing tar wrapper {fileName}: {error}'.format(fileName = 'tar_wrapper.py',
+ error = e
+ )
+ msg.error(errMsg)
+ raise trfExceptions.TransformExecutionException(trfExit.nameToCode('TRF_EXEC_SETUP_WRAPPER'),
+ errMsg
+ )
+ self._cmd.append('tar_wrapper.py')
+
elif self._exe == 'zip':
+ if 'outputArchFile' not in self.conf.argdict:
+ raise trfExceptions.TransformExecutionException(trfExit.nameToCode('TRF_ARG_MISSING'), 'Missing output file name')
+
self._cmd = ['python']
try:
with open('zip_wrapper.py', 'w') as zip_wrapper:
- print >> zip_wrapper, "import zipfile"
- print >> zip_wrapper, "zf = zipfile.ZipFile('{}', mode='w', allowZip64=True)".format(self.conf.argdict['outputArchFile'].value[0])
+ print >> zip_wrapper, "import zipfile, tarfile, os, shutil"
+ if os.path.exists(self.conf.argdict['outputArchFile'].value[0]):
+ #appending input file(s) to existing archive
+ print >> zip_wrapper, "zf = zipfile.ZipFile('{}', mode='a', allowZip64=True)".format(self.conf.argdict['outputArchFile'].value[0])
+ else:
+ #creating new archive
+ print >> zip_wrapper, "zf = zipfile.ZipFile('{}', mode='w', allowZip64=True)".format(self.conf.argdict['outputArchFile'].value[0])
print >> zip_wrapper, "for f in {}:".format(self.conf.argdict['inputDataFile'].value)
- print >> zip_wrapper, " print 'Zipping file {}'.format(f)"
- print >> zip_wrapper, " zf.write(f, compress_type=zipfile.ZIP_STORED)"
+ print >> zip_wrapper, " if not zipfile.is_zipfile(f) and not tarfile.is_tarfile(f):"
+ print >> zip_wrapper, " print 'Zipping {}'.format(os.path.basename(f))"
+ print >> zip_wrapper, " zf.write(f, arcname=os.path.basename(f), compress_type=zipfile.ZIP_STORED)"
+ print >> zip_wrapper, "if os.path.isdir('tmp'):"
+ print >> zip_wrapper, " for root, dirs, files in os.walk('tmp'):"
+ print >> zip_wrapper, " for name in files:"
+ print >> zip_wrapper, " print 'Zipping {}'.format(name)"
+ print >> zip_wrapper, " zf.write(os.path.join(root, name), name, compress_type=zipfile.ZIP_STORED)"
+ print >> zip_wrapper, " shutil.rmtree('tmp')"
print >> zip_wrapper, "zf.close()"
os.chmod('zip_wrapper.py', 0755)
except (IOError, OSError) as e:
@@ -1903,4 +1966,32 @@ class archiveExecutor(scriptExecutor):
errMsg
)
self._cmd.append('zip_wrapper.py')
+
+ elif self._exe == 'unarchive':
+ if not zipfile.is_zipfile(self.conf.argdict['inputArchFile'].value[0]) and not tarfile.is_tarfile(self.conf.argdict['inputArchFile'].value[0]):
+ raise trfExceptions.TransformExecutionException(trfExit.nameToCode('TRF_INPUT_FILE_ERROR'),
+ 'The input file is not a zip or tar archive - aborting unpacking')
+ self._cmd = ['python']
+ try:
+ with open('unarchive_wrapper.py', 'w') as unarchive_wrapper:
+ print >> unarchive_wrapper, "import zipfile, tarfile"
+ print >> unarchive_wrapper, "for f in {}:".format(self.conf.argdict['inputArchFile'].value)
+ print >> unarchive_wrapper, " if zipfile.is_zipfile(f):"
+ print >> unarchive_wrapper, " archive = zipfile.ZipFile(f, mode='r')"
+ print >> unarchive_wrapper, " elif tarfile.is_tarfile(f):"
+ print >> unarchive_wrapper, " archive = tarfile.open(f, 'r:*')"
+ print >> unarchive_wrapper, " path = '{}'".format(self.conf.argdict['path'])
+ print >> unarchive_wrapper, " print 'Extracting archive {0} to {1}'.format(f,path)"
+ print >> unarchive_wrapper, " archive.extractall(path)"
+ print >> unarchive_wrapper, " archive.close()"
+ os.chmod('unarchive_wrapper.py', 0755)
+ except (IOError, OSError) as e:
+ errMsg = 'error writing unarchive wrapper {fileName}: {error}'.format(fileName = 'unarchive_wrapper.py',
+ error = e
+ )
+ msg.error(errMsg)
+ raise trfExceptions.TransformExecutionException(trfExit.nameToCode('TRF_EXEC_SETUP_WRAPPER'),
+ errMsg
+ )
+ self._cmd.append('unarchive_wrapper.py')
super(archiveExecutor, self).preExecute(input=input, output=output)
diff --git a/Tools/PyJobTransforms/scripts/Archive_tf.py b/Tools/PyJobTransforms/scripts/Archive_tf.py
index d943496b1ce796faaed3df4ec6883c29e51e5376..2798ca5be0863301dc24b4fc24754a442662124e 100755
--- a/Tools/PyJobTransforms/scripts/Archive_tf.py
+++ b/Tools/PyJobTransforms/scripts/Archive_tf.py
@@ -36,27 +36,35 @@ def main():
sys.exit(trf.exitCode)
def getTransform():
- trf = transform(executor = archiveExecutor(name = 'Archiver', exe = 'zip'))
+ executorSet = set()
+ executorSet.add(archiveExecutor(name = 'Archiver', exe = 'zip', inData = ['Data'], outData = ['Arch']))
+ executorSet.add(archiveExecutor(name = 'Unarchiver', exe = 'unarchive', inData = ['Arch'], outData = ['outNULL']))
+ trf = transform(executor = executorSet)
addMyArgs(trf.parser)
return trf
def addMyArgs(parser):
# Use arggroup to get these arguments in their own sub-section (of --help)
- parser.defineArgGroup('Archive_tf', 'Archive transform options')
+ parser.defineArgGroup('Archiver', 'Options')
+ parser.defineArgGroup('Unarchiver', 'Options')
parser.defineArgGroup('Tar archiver', 'Options')
- parser.add_argument('--exe', group='Archive_tf',
- help='Archiving command. Default is zip', choices=['zip', 'tar'],
- default='zip')
- parser.add_argument('--inputDataFile', '--inputFile', nargs='+',
+ parser.add_argument('--exe', group='Archiver',
+ help='Archiving command, default is zip', choices=['zip', 'tar'])
+ parser.add_argument('--inputDataFile', '--inputFile', nargs='+',
type=trfArgClasses.argFactory(trfArgClasses.argFile, io='input', type='misc'),
- help='Input file(s)', group='Archive_tf')
+ help='Input file(s)', group='Archiver')
+ parser.add_argument('--inputArchFile',
+ type=trfArgClasses.argFactory(trfArgClasses.argFile, io='input', type='misc'),
+ help='Input archive file', group='Unarchiver')
+ parser.add_argument('--path', group='Unarchiver',
+ help='Specifies a different directory to extract to. The default is the current working directory', default='.')
parser.add_argument('--outputArchFile', '--outputFile',
type=trfArgClasses.argFactory(trfArgClasses.argFile, io='output', type='misc'),
- help='Output archive file', group='Archive_tf')
+ help='Output archive file', group='Archiver')
parser.add_argument('--compressionType', group='Tar archiver',
- help='Underlying compression type of tar. Default is none', choices=['gzip', 'bzip2', 'none'],
+ help='Underlying compression type of tar. The default is none', choices=['gzip', 'bzip2', 'none'],
default='none')
if __name__ == '__main__':