diff --git a/Tools/PyUtils/bin/meta-diff.py b/Tools/PyUtils/bin/meta-diff.py
index c51424c780775eba39f07c8c4c3712c259d64bc2..bee8a42dcfa6064f38a864b662298ac8b8d15489 100755
--- a/Tools/PyUtils/bin/meta-diff.py
+++ b/Tools/PyUtils/bin/meta-diff.py
@@ -5,264 +5,110 @@
from __future__ import print_function
-import sys
-import json
import argparse
-import time
-import logging
import os
+import sys
+
+from PyUtils.MetaDiff import meta_diff
-# escape sequence [?1034h which appear on several runs due to smm capability (Meta Mode On) for xterm.
+# escape sequence [?1034h which appear on several runs due to smm capability
+# (Meta Mode On) for xterm.
if 'TERM' in os.environ:
del os.environ['TERM']
-msg = logging.getLogger('MetaDiff')
-
-from PyUtils.MetaReader import read_metadata
-
-def print_diff(parent_key, obj1, obj2, diff_format):
-
- if diff_format == 'simple':
- if obj1 is None:
- print('{} has been inserted'.format(parent_key))
- elif obj2 is None:
- print('{} has been deleted'.format(parent_key))
- else:
- print('{} has changed from \'{}\' to \'{}\''.format(parent_key, obj1, obj2))
- else:
- print('')
- if parent_key is not None:
- print('{}:'.format(parent_key))
- print('> {}'.format(obj1))
- print('----------')
- print('< {}'.format(obj2))
-
-def print_diff_type(parent_key, obj1, obj2, diff_format):
-
- if diff_format == 'simple':
- if obj1 is None:
- print('{} has been inserted'.format(parent_key))
- elif obj2 is None:
- print('{} has been deleted'.format(parent_key))
- else:
- print('{} has changed changed type from {} (value: \'{}\') to {} (value: \'{}\')'.format(parent_key, type(obj1), obj1, type(obj2), obj2))
- else:
- print('')
- if parent_key is not None:
- print('{}:'.format(parent_key))
- print('> {} (type: {})'.format(obj1, type(obj1)))
- print('----------')
- print('< {} (type: {})'.format(obj2, type(obj2)))
-
-def print_diff_dict_keys(parent_key, obj1, obj2, diff_format):
- value1 = None
- value2 = None
-
- if obj1 is not None:
- value1 = ', '.join(['{}: {}'.format(k, '{...}' if isinstance(v, dict) else v) for k,v in sorted(obj1.items())])
+def main():
+ """Handle command line arguments and call meta_diff"""
+
+ parser = argparse.ArgumentParser(
+ description='Compare the metadata content fo two files')
+
+ parser.add_argument(
+ 'files',
+ nargs=2,
+ metavar='FILE',
+ help='The names of two files to compare')
+
+ parser.add_argument(
+ '-v', '--verbose',
+ action='store_true',
+ help='print detailed output on screen')
+
+ parser.add_argument(
+ '-s', '--ordered',
+ action='store_true',
+ help='When comparing lists, check the element order too.')
+
+ parser.add_argument(
+ '-d', '--drop',
+ nargs='*',
+ default=None,
+ metavar='KEY',
+ help='Keys to drop from metadata retrieved from file')
+
+ parser.add_argument(
+ '-m', '--mode',
+ default='lite',
+ metavar='MODE',
+ type=str,
+ choices=['tiny', 'lite', 'full', 'peeker'],
+ help='''\
+ This flag provides the user capability to select the amount of
+ metadata retrieved. There three options:
+ tiny (only those values used in PyJobTransforms),
+ lite (same output as dump-athfile)
+ and full ( all available data found)
+ ''')
+
+ parser.add_argument(
+ '-t', '--type',
+ default=None,
+ metavar='TYPE',
+ type=str,
+ choices=['POOL', 'BS'],
+ help='''\
+ The file type of the input filename. By default, it tries to
+ determine itself the file type of the input.
+ ''')
+
+ parser.add_argument(
+ '-f', '--filter',
+ default=[],
+ metavar='FILTER',
+ nargs='+',
+ type=str,
+ help="Expression to select specific metadata fields to retrieve.")
- if obj2 is not None:
- value2 = ', '.join(['{}: {}'.format(k, '{...}' if isinstance(v, dict) else v) for k,v in sorted(obj2.items())])
-
- if diff_format == 'simple':
- if obj1 is None:
- print('{} has been inserted'.format(parent_key))
- elif obj2 is None:
- print('{} has been deleted'.format(parent_key))
- else:
- print('{} has changed from \'{}\' to \'{}\''.format(parent_key, value1, value2))
- else:
- print('')
- if parent_key is not None:
- print('{}:'.format(parent_key))
-
- print('> ' + value1)
- print('----------')
- print('< ' + value2)
-
-def print_same(parent_key, obj1, obj2, diff_format):
- if diff_format == 'simple':
- print('{} is the same'.format(parent_key))
- else:
- print('')
- print('{}: same'.format(parent_key))
-
-def compare(obj1, obj2, parent_key=None, ordered=False, show_same=False, diff_format='simple'):
-
-
- same = True;
-
- if isinstance(obj1, dict) and isinstance(obj2, dict):
- if sorted(obj1.keys()) != sorted(obj2.keys()):
- print_diff_dict_keys(parent_key, obj1, obj2, diff_format)
- same = False
- else:
- for key in sorted(set(obj1.keys() + obj2.keys())):
-
- child_key = ''
- if parent_key is not None:
- child_key += parent_key + '/'
- child_key += key
-
- same &= compare(obj1[key] if key in obj1 else None, obj2[key] if key in obj2 else None, child_key, ordered, show_same, diff_format)
-
- elif isinstance(obj1, list) and isinstance(obj2, list):
- if ordered:
- if sorted(obj1) != sorted(obj2):
- print_diff(parent_key, obj1, obj2, diff_format)
- same = False
- else:
- if show_same:
- print_same(parent_key, obj1, obj2, diff_format)
- else:
- if obj1 != obj2:
- print_diff(parent_key, obj1, obj2, diff_format)
- same = False
- else:
- if show_same:
- print_same(parent_key, obj1, obj2, diff_format)
- elif isinstance(obj1, set) and isinstance(obj2, set):
- if obj1 != obj2:
- print_diff(parent_key, obj1, obj2, diff_format)
- same = False
- else:
- if show_same:
- print_same(parent_key, obj1, obj2, diff_format)
- elif type(obj1) == type(obj2):
- if obj1 != obj2:
- print_diff(parent_key, obj1, obj2, diff_format)
- same = False
- else:
- if show_same:
- print_same(parent_key, obj1, obj2, diff_format)
- else:
- print_diff_type(parent_key, obj1, obj2, diff_format)
- same = False
-
- return same
-
-
-
-
-def _main():
- # Parsing the arguments provided by user
- parser = argparse.ArgumentParser(description='This script reads metadata from a given file')
-
- parser.add_argument('-v',
- '--verbose',
- action='store_true',
- help='print detailed output on screen')
- parser.add_argument('--ordered',
- action='store_true',
- help='When comparing lists, check the element order too.')
- parser.add_argument('-o',
- '--output',
- metavar='FILE',
- default=None,
- help="Saves the output in a file. By default, the output is written on the screen (stdout) in a prettier format for better readabiilty.")
- parser.add_argument('-m',
- '--mode',
- default= 'lite',
- metavar='MODE',
- type=str,
- choices=['tiny', 'lite', 'full', 'peeker'],
- help="This flag provides the user capability to select the amount of metadata retrieved. There three options: "
- "tiny (only those values used in PyJobTransforms), "
- "lite (same output as dump-athfile) "
- "and full ( all available data found) ")
-
- parser.add_argument('-c',
- '--check',
- action='store_true',
- help='If difference are found, produce a non-zero return code')
-
- parser.add_argument('-s',
- '--show-same',
- action='store_true',
- help='Show also values that are the same')
-
- parser.add_argument('-t',
- '--type',
- default= None,
- metavar='TYPE',
- type=str,
- choices=['POOL', 'BS'],
- help="The file type of the input filename. By default, it tries to determine itself the file type of the input.")
-
- parser.add_argument('-f',
- '--filter',
- default= [],
- metavar='FILTER',
- nargs = '+',
- type=str,
- help="The metadata keys to filter. ")
- parser.add_argument('-d',
- '--diff-format',
- default= 'simple',
- type=str,
- choices=['simple', 'diff'],
- help="Show 'simple' or 'diff' style differences ")
- parser.add_argument('--promote',
- default=None,
- type=bool,
- help="Force promotion or not of the metadata keys ")
+ parser.add_argument(
+ '-x', '--diff-format',
+ default= 'simple',
+ type=str,
+ choices=['simple', 'diff'],
+ help="Switch between 'simple' or 'diff' style differences ")
-
-
- parser.add_argument('filename1', help='First file to compare.')
- parser.add_argument('filename2', help='Second file to compare.')
-
- args = parser.parse_args()
+ parser.add_argument(
+ '--promote',
+ default=None,
+ type=bool,
+ help="Force promotion or not of the metadata keys ")
- check = args.check
- show_same = args.show_same
- verbose = args.verbose
- ordered = args.ordered
- filename1 = args.filename1
- filename2 = args.filename2
- output = args.output
- mode = args.mode
- diff_format = args.diff_format
- file_type = args.type
- meta_key_filter = args.filter
-
-
-
- msg.setLevel(logging.INFO if verbose else logging.WARNING)
- # create a stream handler
- handler = logging.StreamHandler()
- handler.setLevel(logging.INFO if verbose else logging.WARNING)
- # create a logging format
- formatter = logging.Formatter('%(name)s %(levelname)s %(message)s')
- handler.setFormatter(formatter)
- # add the handlers to the logger
- msg.addHandler(handler)
+ args = parser.parse_args()
- metadatas = read_metadata([filename1, filename2], file_type, mode=mode, meta_key_filter=meta_key_filter, promote=args.promote)
- metadata1 = metadatas[filename1]
- metadata2 = metadatas[filename2]
-
- del metadata1['file_guid']
- del metadata2['file_guid']
-
- del metadata1['file_size']
- del metadata2['file_size']
-
- same = compare(metadata1, metadata2, ordered=ordered, diff_format=diff_format, show_same=show_same)
-
-
- if check and not same:
+ try:
+ diff = meta_diff(
+ args.files, verbose=args.verbose, ordered=args.ordered,
+ drop=args.drop, mode=args.mode, meta_key_filter=args.filter,
+ file_type=args.type, promote=args.promote, diff_format=args.diff_format)
+ except (ValueError, IndexError):
+ print("you must supply two files to compare")
sys.exit(1)
+ if diff:
+ print('\n'.join(diff))
+ sys.exit(1)
-if __name__ == '__main__':
- res = _main()
-
-
-
-
-
-
+ sys.exit(0)
+if __name__ == '__main__':
+ main()
diff --git a/Tools/PyUtils/python/MetaDiff.py b/Tools/PyUtils/python/MetaDiff.py
index df6ba6b03b80e38df1f47b52d5c75c1eb5f93807..99156734068548fc5e9f27e9388a77adef4d75ad 100644
--- a/Tools/PyUtils/python/MetaDiff.py
+++ b/Tools/PyUtils/python/MetaDiff.py
@@ -9,55 +9,83 @@ import logging
from PyUtils.MetaReader import read_metadata
-def print_diff(parent_key, obj1, obj2):
+def print_diff(parent_key, obj1, obj2, diff_format):
"""build comparison string for two non-dictionary objects"""
result = '\n'
-
- if parent_key is not None:
- result += '{}:\n'.format(parent_key)
- result += '''\
- > {}
- ----------
- < {}
- '''.format(obj1, obj2)
+
+ if diff_format == 'simple':
+ if obj1 is None:
+ result += '{} has been inserted'.format(parent_key)
+ elif obj2 is None:
+ result += '{} has been deleted'.format(parent_key)
+ else:
+ result += '{} has changed from \'{}\' to \'{}\''.format(parent_key, obj1, obj2)
+ result += '\n'
+ else:
+ if parent_key is not None:
+ result += '{}:\n'.format(parent_key)
+ result += '''\
+ > {}
+ ----------
+ < {}
+ '''.format(obj1, obj2)
return result
-def print_diff_type(parent_key, obj1, obj2):
+def print_diff_type(parent_key, obj1, obj2, diff_format):
"""Build diff string for objet of different type"""
result = '\n'
- if parent_key is not None:
- result += '{}:\n'.format(parent_key)
- result += '''\
- > {} (type: {})
- ----------
- < {} (type: {})
- '''.format(obj1, type(obj1), obj2, type(obj2))
+ if diff_format == 'simple':
+ if obj1 is None:
+ result += '{} has been inserted'.format(parent_key)
+ elif obj2 is None:
+ result += '{} has been deleted'.format(parent_key)
+ else:
+ result += '{} has changed changed type from {} (value: \'{}\') to {} (value: \'{}\')'.format(parent_key, type(obj1), obj1, type(obj2), obj2)
+ result += '\n'
+ else:
+ if parent_key is not None:
+ result += '{}:\n'.format(parent_key)
+ result += '''\
+ > {} (type: {})
+ ----------
+ < {} (type: {})
+ '''.format(obj1, type(obj1), obj2, type(obj2))
return result
-def print_diff_dict_keys(parent_key, obj1, obj2):
+def print_diff_dict_keys(parent_key, obj1, obj2, diff_format):
"""build diff style string for dictionary objects"""
result = '\n'
- if parent_key is not None:
- result += '{}:\n'.format(parent_key)
- result += '> ' + ', '.join([
- '{}: {}'.format(k, '{...}' if isinstance(v, dict) else v)
- for k, v in sorted(obj1.items())])
- result += '\n----------\n'
- result += '< ' + ', '.join([
- '{}: {}'.format(k, '{...}' if isinstance(v, dict) else v)
- for k, v in sorted(obj2.items())])
+ value1 = ', '.join(['{}: {}'.format(k, '{...}' if isinstance(v, dict) else v)
+ for k, v in sorted(obj1.items())]);
+ value2 = ', '.join(['{}: {}'.format(k, '{...}' if isinstance(v, dict) else v)
+ for k, v in sorted(obj2.items())])
+
+ if diff_format == 'simple':
+ if obj1 is None:
+ result += '{} has been inserted'.format(parent_key)
+ elif obj2 is None:
+ result += '{} has been deleted'.format(parent_key)
+ else:
+ result += '{} has changed from \'{}\' to \'{}\''.format(parent_key, value1, value2)
+ else:
+ if parent_key is not None:
+ result += '{}:\n'.format(parent_key)
+ result += '> ' + value1
+ result += '\n----------\n'
+ result += '< ' + value2
+
result += '\n'
return result
-def compare(obj1, obj2, parent_key=None, ordered=False):
+def compare(obj1, obj2, parent_key=None, ordered=False, diff_format='simple'):
"""Caclulate difference between two objects
Keyword arguments:
@@ -82,26 +110,26 @@ def compare(obj1, obj2, parent_key=None, ordered=False):
if isinstance(obj1, dict):
if sorted(obj1.keys()) != sorted(obj2.keys()):
- result += [print_diff_dict_keys(parent_key, obj1, obj2)]
+ result += [print_diff_dict_keys(parent_key, obj1, obj2, diff_format)]
else:
for key in sorted(set(obj1.keys() + obj2.keys())):
if parent_key:
child_key = '{}/{}'.format(parent_key, key)
else:
child_key = key
- result += compare(obj1[key], obj2[key], child_key, ordered)
+ result += compare(obj1[key], obj2[key], child_key, ordered, diff_format)
else:
- result += [print_diff(parent_key, obj1, obj2)]
+ result += [print_diff(parent_key, obj1, obj2, diff_format)]
else:
- result += [print_diff_type(parent_key, obj1, obj2)]
+ result += [print_diff_type(parent_key, obj1, obj2, diff_format)]
return result
-def meta_diff(files, verbose=False, ordered=False, drop=None,
- mode='lite', meta_key_filter=None, file_type=None, promote=False):
+def meta_diff(files, verbose=False, ordered=False, drop=None, mode='lite',
+ meta_key_filter=None, file_type=None, promote=False, diff_format='simple'):
"""
Compare the in-file metadata in two given files. Uses PyUtils.MetaReader
to obtain file content. Generates list of string that show difference.
@@ -116,8 +144,9 @@ def meta_diff(files, verbose=False, ordered=False, drop=None,
Allowed values are: tiny, lite, peeker, and full
meta_key_filter -- MetaReader argument selecting keys to retrieve (default
get all)
- file_type -- Type of files, POOL or BS (default: auto-configure)
- promote -- MetaReader argument (default: False)
+ file_type -- Type of files, POOL or BS (default: auto-configure)
+ promote -- MetaReader argument (default: False)
+ diff_format -- Show 'simple' or 'diff' style differences (default: 'simple')
"""
if len(files) != 2:
raise ValueError("Wrong number of files passes, need two")
@@ -136,4 +165,4 @@ def meta_diff(files, verbose=False, ordered=False, drop=None,
except TypeError:
pass
- return compare(metadata[files[0]], metadata[files[1]], ordered=ordered)
+ return compare(metadata[files[0]], metadata[files[1]], ordered=ordered, diff_format=diff_format)