Commit afe22a70 authored by Walter Lampl's avatar Walter Lampl
Browse files

Merge branch 'conftool' into 'master'

Various confTool improvements

See merge request atlas/athena!30478
parents a97aad91 17c38359
#!/usr/bin/env python
#
# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
#
from __future__ import print_function
import argparse
import ast
import collections
import json
import pickle
import pprint
import json
import re
import sys
import argparse
import ast
def main():
......@@ -103,13 +105,18 @@ def __loadSingleFile(fname, args):
conf = []
if fname.endswith(".pkl"):
with open(fname, "rb") as input_file:
conf_dict = collections.defaultdict()
while True:
try:
conf.append(pickle.load(input_file))
cfg = pickle.load(input_file)
if type(cfg) == collections.defaultdict:
conf_dict.update(cfg)
else:
conf.append(cfg)
except EOFError:
break
conf.append(conf_dict)
print("... Read", len(conf), "items from python pickle file: ", fname)
elif fname.endswith(".json"):
def __keepPlainStrings(element):
......@@ -164,7 +171,9 @@ def __printComps(conf):
def __compareConfig(configRef, configChk, args):
# Find superset of all components:
allComps = set(list(configRef.keys()) + list(configChk.keys()))
allComps = list(set(configRef.keys()) | set(configChk.keys()))
allComps.sort()
print("Step 1: reference file #components:", len(configRef))
print("Step 2: file to check #components:", len(configChk))
......@@ -187,6 +196,7 @@ def __compareConfig(configRef, configChk, args):
" \033[92m exists only in Ref \033[0m \033[0m \n",
)
continue
refValue = configRef[component]
chkValue = configChk[component]
......@@ -196,7 +206,7 @@ def __compareConfig(configRef, configChk, args):
else:
print("\033[91m Component", component, "differ \033[0m")
if not args.allComponentPrint:
__compareComponent(refValue, chkValue, "\t", args)
__compareComponent(refValue, chkValue, "\t", args, component)
else:
print(
"\t\033[92mRef\033[0m\t",
......@@ -208,25 +218,26 @@ def __compareConfig(configRef, configChk, args):
)
def __compareComponent(compRef, compChk, prefix, args):
def __compareComponent(compRef, compChk, prefix, args, component):
if isinstance(compRef, dict):
allProps = set(list(compRef.keys()) + list(compChk.keys()))
allProps = list(set(compRef.keys()) | set(compChk.keys()))
allProps.sort()
ignoreList = ["StoreGateSvc", "OutputLevel", "MuonEDMHelperSvc"]
for prop in allProps:
if prop not in compRef.keys():
print(
"%s%s: \033[94m exists only in Chk \033[0m \033[91m<< !!!\033[0m"
% (prefix, prop)
"%s%s = %s: \033[94m exists only in Chk \033[0m \033[91m<< !!!\033[0m"
% (prefix, prop, compChk[prop])
)
continue
if prop not in compChk.keys():
print(
"%s%s: \033[92m exists only in Ref \033[0m \033[91m<< !!!\033[0m"
% (prefix, prop)
"%s%s = %s: \033[92m exists only in Ref \033[0m \033[91m<< !!!\033[0m"
% (prefix, prop, compRef[prop])
)
continue
......@@ -243,38 +254,83 @@ def __compareComponent(compRef, compChk, prefix, args):
else:
diffmarker = " \033[91m<< !!!\033[0m"
print(
"%s%s : \033[92m %s \033[0m vs \033[94m %s \033[0m %s"
% (prefix, prop, str(refVal), str(chkVal), diffmarker)
)
if not (component == 'IOVDbSvc' and prop == 'Folders'):
print(
"%s%s : \033[92m %s \033[0m vs \033[94m %s \033[0m %s"
% (prefix, prop, str(refVal), str(chkVal), diffmarker)
)
try:
refVal = ast.literal_eval(str(refVal))
chkVal = ast.literal_eval(str(chkVal))
refVal = ast.literal_eval(str(refVal)) if refVal else ''
chkVal = ast.literal_eval(str(chkVal)) if chkVal else ''
except SyntaxError:
pass
except ValueError:
pass # literal_eval exception when parsing particular strings
if refVal and (isinstance(refVal, list) or isinstance(refVal, dict)):
__compareComponent(refVal, chkVal, "\t" + prefix + ">> ", args)
if component == 'IOVDbSvc' and prop == 'Folders':
__compareIOVDbFolders(refVal, chkVal, "\t", args)
else:
__compareComponent(refVal, chkVal, "\t" + prefix + ">> ", args, component)
elif isinstance(compRef, list) and len(compRef) > 1:
if len(compRef) != len(compChk):
diff = list(set(compRef) - set(compChk))
inRef = all(elem in compRef for elem in compChk)
missingIn = "Ref" if inRef else "Chk"
color = "\033[92m" if inRef else "\033[94m"
diffRef = list(set(compRef) - set(compChk))
diffChk = list(set(compChk) - set(compRef))
if diffRef:
print(
"%s exists only in %s : %s %s \033[0m \033[91m<< !!!\033[0m"
% (prefix, missingIn, color, str(diff))
"%s exists only in Ref : \033[92m %s \033[0m \033[91m<< !!!\033[0m"
% (prefix, str(diffRef))
)
else:
for refVal, chkVal in zip(compRef, compChk):
if diffChk:
print(
"%s exists only in Chk : \033[94m %s \033[0m \033[91m<< !!!\033[0m"
% (prefix, str(diffChk))
)
if len(compRef) == len(compChk):
for i, (refVal, chkVal) in enumerate(zip(compRef, compChk)):
if refVal != chkVal:
print(
"%s : \033[92m %s \033[0m vs \033[94m %s \033[0m \033[91m<< !!!\033[0m"
% (prefix, str(refVal), str(chkVal))
"%s : \033[92m %s \033[0m vs \033[94m %s \033[0m \033[91m<< at index %s !!!\033[0m"
% (prefix, str(refVal), str(chkVal), str(i))
)
__compareComponent(refVal, chkVal, "\t" + prefix + ">> ", args)
__compareComponent(refVal, chkVal, "\t" + prefix + ">> ", args, '')
def __parseIOVDbFolder(definition):
result = {}
# db
db_match = re.search(r'<db>(.*)</db>', definition)
if db_match:
result['db'] = db_match.group(1)
definition = definition.replace(db_match.group(0), '')
# tag
tag_match = re.search(r'<tag>(.*)</tag>', definition)
if tag_match:
result['tag'] = tag_match.group(1)
definition = definition.replace(tag_match.group(0), '')
# cache -- ignore for now
cache_match = re.search(r'<cache>(.*)</cache>', definition)
if cache_match:
definition = definition.replace(cache_match.group(0), '')
# noover
noover_match = re.search(r'<noover/>', definition)
if noover_match:
result['noover'] = True
definition = definition.replace(noover_match.group(0), '')
# name
result['name'] = definition.strip()
return json.dumps(result)
def __compareIOVDbFolders(compRef, compChk, prefix, args):
refParsed = []
chkParsed = []
for item in compRef:
refParsed.append(__parseIOVDbFolder(item))
for item in compChk:
chkParsed.append(__parseIOVDbFolder(item))
__compareComponent(refParsed, chkParsed, prefix, args, '')
if __name__ == "__main__":
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment