From 989ba7d587dbef3da8efea83fa6b6c34514420d4 Mon Sep 17 00:00:00 2001 From: Chris Malena Delitzsch <chris.malena.delitzsch@cern.ch> Date: Tue, 2 Feb 2021 16:40:19 +0000 Subject: [PATCH] Fixing first set of flake8 warnings in JetValidation --- .../Jet/JetValidation/CMakeLists.txt | 2 +- .../python/CompareJetTestFilesExec.py | 132 ---- .../Jet/JetValidation/python/JetTestCheck.py | 715 ------------------ .../Jet/JetValidation/python/JetTesters.py | 383 ---------- .../python/PhysicsValidationHistos.py | 10 +- .../Jet/JetValidation/python/RTTConfig.py | 2 - 6 files changed, 5 insertions(+), 1239 deletions(-) delete mode 100755 Reconstruction/Jet/JetValidation/python/CompareJetTestFilesExec.py delete mode 100755 Reconstruction/Jet/JetValidation/python/JetTestCheck.py delete mode 100644 Reconstruction/Jet/JetValidation/python/JetTesters.py diff --git a/Reconstruction/Jet/JetValidation/CMakeLists.txt b/Reconstruction/Jet/JetValidation/CMakeLists.txt index 8a352220d2e3..7d9b458a8fba 100644 --- a/Reconstruction/Jet/JetValidation/CMakeLists.txt +++ b/Reconstruction/Jet/JetValidation/CMakeLists.txt @@ -4,6 +4,6 @@ atlas_subdir( JetValidation ) # Install files from the package: -atlas_install_python_modules( python/*.py ) +atlas_install_python_modules( python/*.py POST_BUILD_CMD ${ATLAS_FLAKE8} ) atlas_install_joboptions( share/*.py ) atlas_install_scripts( scripts/jetMakeRefSamples.py ) diff --git a/Reconstruction/Jet/JetValidation/python/CompareJetTestFilesExec.py b/Reconstruction/Jet/JetValidation/python/CompareJetTestFilesExec.py deleted file mode 100755 index ef16b8a358c9..000000000000 --- a/Reconstruction/Jet/JetValidation/python/CompareJetTestFilesExec.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python - -# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration - -from __future__ import print_function - -################################################## -### CompareJetTestFileExec -### -### *** BEING DEPRECATED *** -### the code in this file has been recopied in JetTestCheck.py in the __main__ part of it. -### -### *** BEING DEPRECATED *** -### An executable to compare the jet content of 2 xAOD files. -### -### Usage : -### python CompareJetTestFilesExec.py xaod1.root xaod2.root -### -### python CompareJetTestFilesExec.py -h to get the list of options -### -### -### Currently the files compare only the AntiKt4LCTopoJets and AntiKt10LCTopoJets -### containers. -### -################################################# - -import sys -import argparse - -testedContainers = ["AntiKt4LCTopoJets","AntiKt10LCTopoJets"] -################################################## -# Read and store argument values # -################################################## - -parser = argparse.ArgumentParser(description='Create distributions for many variables and save them in a ROT file') - -parser.add_argument('inFile1', type=str, help='1st input root file') -parser.add_argument('inFile2', type=str, default="", nargs='?', help='2nd input root file. If none, then container "XYZ" in inFile1 will be compared against "XYZSuffix" where Suffix is set by --suffix') -parser.add_argument('--treeName1', type=str, default="CollectionTree", help="Name of TTree (default: %(default)s)", metavar='name1') -parser.add_argument('--treeName2', type=str, default="CollectionTree", help="Name of TTree", metavar='name2') -parser.add_argument( '--containers', type=str, default= testedContainers, nargs='*',help="list of containers (ex: --containers cont1 cont2 cont3)", metavar='jetcontname') -parser.add_argument('--details', action='store_true', help="Display status for each variable, even if no error") -parser.add_argument('--ESD', action='store_true', help="Use this flag if both input files are ESD") -parser.add_argument('--AODvsESD', action='store_true', help="Use this flag if file1 is AOD and file2 is ESD") -parser.add_argument('--suffix', type=str, default='Test', help="In case using only 1 input files, uses this suffix to relate containers to be matched. Default to 'Test'") -parser.add_argument('--clusters', action='store_true', help="Use this flag if file1 is AOD and file2 is ESD") - - -#parser.add_argument('-exV', '--excludedVar', type=str, default=[], nargs='*') -#parser.add_argument('-t1', '--type1', type=str, default="xAOD", nargs='?') -#parser.add_argument('--', type=str, default="") -#parser.add_argument('-b' , '--saveBinning', action='store_true') -#parser.add_argument('-pdf' , '--savePDF', action='store_true') -#parser.add_argument('-cal' , '--calibScale', action='store_false') - - -args = parser.parse_args() - -# reset testedContainers : -testedContainers = args.containers -print ("Will test ", testedContainers) - -from JetValidation.JetTestCheck import * - -# if we have only 1 input file, we compare "XXXJets" against "XXXJetsTest" containers -if args.inFile2 == '': - # define a renamer function - branchRenamer = lambda n : n.replace('Jets','Jets'+args.suffix) - # testing only the 2 first entries - VarPairVector.maxVecSize = 2 - # do this as long as area calculation is not reproducible : - # in this case calib differs and misordering can happen for low pt jets -else: - # we compare containers with same name in 2 different files. - # No need to rename, renamer function does nothing : - branchRenamer = lambda n : n - - -# Filter -------------- -def branchFilter(s ): - """Return True if branch named s needs to be compared """ - return any( s.startswith( b ) for b in testedContainers ) #and "Area" not in s - -# Name splitting ---------- -# In order to nicely categorize results, the system needs to know to which container -# a variable "AntiKt4LCTopoJets.Var" belongs. The function below returns ("AntiKt4LCTopoJets","Var") -def splitVarName_xAOD( fullname ): - try: - base, varname = fullname.split('.') - return base, varname - except: - return "none", None - -# act if reading ESD ----------- -if args.ESD: - testedContainers = ["xAOD::JetAuxContainer_v1_"+b for b in testedContainers ] -elif args.AODvsESD: - branchRenamer = lambda n : "xAOD::JetAuxContainer_v1_"+n - - - -# Static aux container branches ----- -additionalBranches = [] -for c in testedContainers: - bname = c+'Aux.' - additionalBranches += [ bname+v for v in ("pt","eta","phi","m") ] - -# add basic clusters comp -if args.clusters: - bname = "CaloCalTopoClusterAux." - if args.ESD: - bname = "xAOD::CaloClusterAuxContainer_v1_"+bname - branchRenamer = lambda n : n.replace("_v1_CaloCalTopoCluster", "_v2_CaloCalTopoClusters") - additionalBranches += [bname+v for v in ("rawE","rawEta","rawPhi") ] - -# Build the main Comparator -vc = VarComparator( args.inFile1 , args.inFile2 if args.inFile2 else None , treename = args.treeName1, treename2=args.treeName2 )# , treename2="susy") -vc.splitVarName = splitVarName_xAOD - - -vc.init_trees( base_vars = [], - branch_filter = branchFilter, - branch_replace = branchRenamer, - checkMissing = False, - additionalBranches = additionalBranches, - ) - -#numErr = vc.full_compare(debug=True, maxEvent=1) -numErr = vc.full_compare(debug=False, maxEvent=100, detailedSummary=args.details) - -exit(numErr) # RTT: success if exit code is 0, failure otherwise - diff --git a/Reconstruction/Jet/JetValidation/python/JetTestCheck.py b/Reconstruction/Jet/JetValidation/python/JetTestCheck.py deleted file mode 100755 index 03f87c55b830..000000000000 --- a/Reconstruction/Jet/JetValidation/python/JetTestCheck.py +++ /dev/null @@ -1,715 +0,0 @@ -#!/usr/bin/env python - -# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration - -from __future__ import print_function - - -""" -This module allows a fast comparison of the ntuple -produced by JetTest. - -It reads in 2 TFile, containing similar TTree. -It get all branches from the trees containing the -chain 'Jet'. - -Then it compares tree1.branchA with tree2.branchA for each -branch and for each event. -The summary for a branch is 'ok' if the maximum relative -difference found is <MAX_REL_DIFF -""" - -__version__ = 'JetTest_module' -__author__ = 'Pierre-Antoine Delsart (delsart@in2p3.fr), \ - Pier-Olivier DeViveiros(viveiros@physics.utoronto.ca), \ - Sergei Chekanov (chakanau@hep.anl.gov)' -__doc__ = 'Module for jet comparison' - -import re,os,sys -import argparse - -import ROOT -re_bname = re.compile("([a-zA-Z0-9]+)_.*") - -import numpy -def rel_diff(v1,v2): - try: - return (v1 - v2)/float(min(abs(v1),abs(v2))) - except ZeroDivisionError: - return 0 -## def _gettypename(self): -## return "SimpleBr" -## ROOT.TBranch.GetTypeName = _gettypename -def fullDebug_on( self, r , val='' , val2=''): - print (self.name , self.b1, r, val , val2) -def fullDebug_off( self, r ,val='', v2='' ): - pass -fullDebug = fullDebug_off - - -## if hasattr( ROOT.vector("float"), "data") : # workaround differences in ROOT versions -## bufferFromVect = lambda v : v.data() -## else: -## bufferFromVect = lambda v : v.begin().base - -def bufferFromVect(v): - return v.data() - - -# ROOT <-> numpy -cppToNumpyType = { - 'vector<float>' : numpy.float32, - 'vector<vector<float> >' : numpy.float32, - 'vector<int>' : numpy.int32, - 'vector<vector<int> >' : numpy.int32, - 'vector<double>' : numpy.float64, # ??? - 'vector<double>' : numpy.bool , - } - - -def splitVarName_default( fullname ): - res = re_bname.search(fullname) - if not res : - base ="" - else: - base = res.group(1) - vname = bn[len(base):] - return base, vname - - - - -## ************************************* -class VarPairBase(object): - """Represents a variable to be compared from 2 sources. - Typicaly the same variable from 2 TTrees - (ex : tree1.AntiKt4LCTopoJetsAuxDyn.Width vs tree2.AntiKt4LCTopoJetsAuxDyn.Width) - but can also be 2 variables from the same tree - (ex : tree1.AntiKt4LCTopoJetsAuxDyn.Width vs tree1.AntiKt4LCTopoJets2AuxDyn.Width) - - the class holds TBranch, TTrees and provide methods to load and compare the variables. - loading and comparing are implemented in derived class depending if the variables are scalar, vector or vector<vector> - """ - tree1 = None - tree2 = None - - draw_selection = "" - draw_option = "" - draw_histospec = None - - - validVariable = True - - - - - @staticmethod - def buildPair(name, b1,b2, MAX_REL_DIFF, isMeV = False): - if isinstance(b1, ROOT.TBranchElement): - concreteType = VarPairVector - if 'vector<vector' in b1.GetTypeName(): - ## ROOT.vector("vector<float>") - ## ROOT.vector("vector<int>") - - concreteType = VarPairVectorVector - elif isMeV: - concreteType = VarPairVectorMeV - else: - concreteType = VarPairScalar - return concreteType(name, b1, b2, MAX_REL_DIFF) - - def __init__(self, name, b1,b2, MAX_REL_DIFF): - self.name = name - self.b1 = b1 - self.b2 = b2 - self.validVariable = (b1!=None) and (b2!=None) - if not self.validVariable: - return - self.bname1 = self.b1.GetName() - self.bname2 = self.b2.GetName() - - self.MAX_REL_DIFF = MAX_REL_DIFF - self.initEventSummaries(0) - self.initBranch() - self.initEventSummaries(self.b1.GetTree().GetEntries()) - self.diff_size_event = -1 - self.nvalues = 0 - - self.nbadDiff = 0 - - def resetBranchPointer(self, one, t, evt): - if one: - self.b1 = t.GetBranch(self.bname1) - self.b1.GetEntry( evt ) - self.v1 = getattr(t, self.bname1) - - else: - self.b2 = t.GetBranch(self.bname2) - self.b2.GetEntry( evt ) - self.v2 = getattr(t, self.bname2) - - def initEventSummaries(self, nevt): - self.max_rel_diff = numpy.zeros(nevt) - self.max_values = numpy.zeros(nevt) - self.num_errors = 0 - - def checkRes(self, evtNum , relDiff, a1 ): - m = relDiff.max() - if m < self.MAX_REL_DIFF : - return True - # save error info - maxInd = relDiff.argmax() - self.max_rel_diff[evtNum] = m - self.max_values[evtNum] = a1[maxInd] - self.num_errors += sum( relDiff > self.MAX_REL_DIFF) - - return False - - - - def summary(self, verbose=False): - name = self.name.rjust(21) - name = name+' '+ ('['+str(self.nvalues)+']').ljust(9) - maxDiff = min(1 ,self.max_rel_diff.max() ) - error = 0 - statusStr = '' - if maxDiff > self.MAX_REL_DIFF : - ind = self.max_rel_diff.argmax() - error += 1 - statusStr = statusStr+' max diff= %7.3f %% of % 10.3f | at evt %4d (occured %5d / %5d) '%( maxDiff*100,self.max_values[ind] ,ind,self.num_errors,self.nvalues) - if self.diff_size_event> -1: - error+= 1 - statusStr = statusStr+" vec size differed [evt "+str(self.diff_size_event)+"]" - if error == 0: - statusStr = 'ok' - if error >0 or verbose: - print ('%40s'%(self.name,), statusStr) - - - return (error, self.nvalues, self.num_errors) - - def draw(self,canvas=None): - if canvas : canvas.cd() - bname = self.b1.GetName() - if self.draw_histospec: - pass # not supported yet - else: - hname = '>>'+bname+"H" - self.tree1.Draw(bname+hname+'1',self.draw_selection, self.draw_option+'goff') - h1 = self.tree1.GetHistogram() - self.tree2.Draw(bname+hname+'2',self.draw_selection, self.draw_option+'goff') - h2 = self.tree1.GetHistogram() - - h2.SetLineColor(ROOT.kRed) - - self.histos = (h1,h2) - h1.Draw() - h2.Draw("same") - - -class VarPairVector(VarPairBase): - """Specialized representation of variables of type vector<T> """ - maxVecSize = -1 - - def checkxAODAuxStore(self, one ): - - name, b, tree = (self.bname1, self.b1, self.tree1) if one else (self.bname2, self.b2, self.tree2) - if 'Aux.' in name and not name.endswith('Aux.') : - #from xAODAuxStoreHelper import initStaticStoreBranch - # this properly extracts the vector<> object from the AuxStore - # and make it available in the tree - initStaticStoreBranch(tree,b) - - def initBranch(self): - b1, b2 = self.b1, self.b2 - #print ('initBranch ', self, self.name, self.b2) - - b1.GetEntry(0) - b2.GetEntry(0) - - # if branches comes from an xAOD static aux store apply special - # trick to make them available. - self.checkxAODAuxStore(one=True) - self.checkxAODAuxStore(one=False) - - self.v1 = getattr(self.tree1, self.bname1) - self.v2 = getattr(self.tree2, self.bname2) - if( type(self.v1) != type(self.v2) ): - self.validVariable = False - # raise Exception - return - - tname = b1.GetTypeName() - nptype = cppToNumpyType.get( tname , None) - if nptype is None : - print (" !!! Unknown type for variable ", b1.GetName(), tname) - self.validVariable = False - return - npa = numpy.ndarray - - # prepare functions which will convert to numpy array - if self.maxVecSize==-1: - def getarray(v): - return npa(v.size(),dtype=nptype,buffer=bufferFromVect(v)) # begin().base - else: - m = self.maxVecSize - def getarray(v): - return npa(min(m,v.size()),dtype=nptype,buffer=bufferFromVect(v)) - - self.getarray = getarray - - def getEntry(self,evt1 , evt2): - self.b1.GetEntry(evt1), self.b2.GetEntry(evt2) - return self.getarray(self.v1) , self.getarray(self.v2) - - def relDiff(self, a1, a2): - return 2*abs(a1-a2)/numpy.maximum( abs(a1+a2), 1e-10) # maximum : avoids division by 0 - - def compare(self, evt1, evt2): - #print ('compare ', self.bname1) - a1, a2 = self.getEntry(evt1, evt2) - n = min( len(a1) , len(a2) ) - #print ("comparing ",self.name, n, len(a1) , len(a2), '|| ', self.v1[0], self.v2[0], '|| ', a1[0], a2[0]) - #print ("comparing ",self.name, n, len(a1) , len(a2), '|| ', a1.shape , a2.shape) - sizematch = True - if len(a1) != len(a2): - sizematch = False - fullDebug( self, 'differing length : '+str(len(a1)), len(a2) ) - if self.diff_size_event == -1: - self.diff_size_event = evt1 - a1 = a1[:n] - a2 = a2[:n] - if n == 0: - return sizematch - self.nvalues += n - rd = self.relDiff(a1,a2) - #print (evt1, ' ', rd, abs(a1+a2)) - - result = self.checkRes(evt1, rd, a1 ) - if not result: - fullDebug( self, rd, a1, a2 ) - self.nbadDiff+=1 - return result and sizematch - -class VarPairVectorMeV(VarPairVector): - """Specialized representation of variables of type vector<T> representing energies. - The tolereance on the difference is evaluated differently. - """ - def relDiff(self, a1, a2): - d = abs(a1-a2) - return numpy.where( d > 0.1 , d , 0 )/abs(a1+0.001) - - -class VarPairVectorVector(VarPairVector): - """Specialized representation of variables of type vector<vector<T> > .""" - - def compare(self, evt1, evt2): - - self.b1.GetEntry(evt1), self.b2.GetEntry(evt2) - - - getarray = self.getarray - v1, v2 = self.v1, self.v2 - - nOuter = min( v1.size() ,v2.size() ) - sizematch = True - if(v1.size() != v2.size() ): - sizematch = False - fullDebug( self, 'differing length : '+str(v1.size()), str(v2.size()) ) - if nOuter== 0: - return sizematch - - result = True - for i in range(nOuter): - a1, a2 = getarray(v1[i]),getarray(v2[i]) - if a1.shape != a2.shape : - result = False - break - rd = self.relDiff(a1,a2) - result = result and self.checkRes(evt1, rd, a1 ) - - if not result: - self.nbadDiff+=1 - return result and sizematch - - - - - -class VarPairScalar(VarPairBase): - """Specialized representation of variables of simple type (int or float) """ - def initBranch(self): - b1, b2 = self.b1, self.b2 - t = b1.GetListOfLeaves()[0].GetTypeName() - #print (' VarPairScalar init ',b1.GetName(), t) - if t == 'Int_t': - t = numpy.int32 - elif t=='Float_t': - t = numpy.float32 - self.a1 = numpy.ndarray(1,dtype = t) - self.a2 = numpy.ndarray(1,dtype = t) - b1.GetTree().SetBranchAddress(b1.GetName(),self.a1) - b2.GetTree().SetBranchAddress(b2.GetName(),self.a2) - def getEntry(self,evt1, evt2): - self.b1.GetEntry(evt1), self.b2.GetEntry(evt2) - return self.a1, self.a2 - - def checkRes(self, evtNum , relDiff, a1 ): - if relDiff < self.MAX_REL_DIFF : - return True - self.max_rel_diff[evtNum] = relDiff - self.max_values[evtNum] = a1 - self.num_errors += 1 - return False - - - def compare(self, evt1, evt2): - a1, a2 = self.getEntry(evt1, evt2) - rd = 2*abs(a1-a2)/numpy.maximum( a1+a2, 1e-10) # maximum : avoids division by 0 - self.nvalues+=1 - fullDebug( self, a1[0], a2[0] ) - return self.checkRes(evt1,rd[0], a1[0]) - - def resetBranchPointer(self, one, t, evt): - if one: - self.b1 = t.GetBranch(self.bname1) - self.b1.GetTree().SetBranchAddress(b1.GetName(),self.a1) - self.b1.GetEntry( evt ) - self.b1 = b1 - #self.v1 = getattr(t, self.bname1) - - else: - b2 = t.GetBranch(self.bname2) - b2.GetTree().SetBranchAddress(b2.GetName(),self.a2) - b2.GetEntry( evt ) - self.b2 = b2 - #self.v2 = getattr(t, self.bname2) - - - -class VarComparator(object): - - """Manage lists of VarPair objects associated to 1 or 2 trees """ - - splitVarName = splitVarName_default - def __init__(self,input1, input2=None, MAX_REL_DIFF=0.01, treename="JetTestTree", treename2=""): - - self.files = [] - if treename2 == "": treename2 = treename - def setupTree( inputf , tn): - if isinstance(inputf, ROOT.TTree ) : - return inputf - if inputf is None : - return self.t1 - if not os.path.exists(inputf): - print (" Can't find ", inputf, '. Exit') - sys.exit(1) - f = ROOT.TFile( inputf ) - tree = f.Get(tn) - self.files.append(f) - return tree - self.t1 = setupTree(input1, treename) - self.t2 = setupTree(input2, treename2) - - self.d_branches = {} - self.MAX_REL_DIFF = MAX_REL_DIFF - - - def close(self): - for f in self.files: - f.Close() - self.files = [] - self.d_branches = {} - - def init_trees(self,base_vars=[], branch_filter = lambda s : 'Jet' in s, - branch_replace = lambda s:s , - completeBranchList=None, - additionalBranches=[], - checkMissing=True): - """Initializes the comparator to compare the 2 files given in constructor - It reads all vars in the tree which starts with the strings in base_vars - - If base_vars is empty, then it reads all branches whose names passes branch_filter(bname)==True - - branch_replace is a function used to adapt the branch name in t1 to the corresponding name in t2. - this allows to compare branches with subtlely different name - - completeBranchList : if non None, this is a the final list of branches to be read - - """ - - t1, t2 = self.t1, self.t2 - t1.LoadTree(0) - t2.LoadTree(0) - - VarPairBase.tree1 = t1 - VarPairBase.tree2 = t2 - - # create a branch list builder function according to arguments - if completeBranchList is not None: - branch_list_builder = lambda fullList : set(completeBranchList) - elif base_vars==[]: - branch_list_builder = lambda fullList : set( b for b in fullList if branch_filter(b) ) - else: - branch_list_builder = lambda fullList : set( b for b in fullList if any(b.startswith(v) for v in base_vars) ) - - - - if checkMissing: - selected_br1 = branch_list_builder( [b.GetName() for b in t1.GetListOfBranches()] ) - selected_br2 = branch_list_builder( [b.GetName() for b in t2.GetListOfBranches()] ) - - selected_br = selected_br1.intersection(selected_br2) - - for b in sorted(selected_br1 - selected_br2): - print (" Missing branches in tree 2 : ", b ) - for b in sorted(selected_br2 - selected_br1): - print (" Missing branches in tree 1 : ", b) - else: - print (base_vars , branch_filter) - selected_br = branch_list_builder( [b.GetName() for b in t1.GetListOfBranches()] ) - - selected_br.update( additionalBranches) - print (' Selected branches = ', selected_br) - - d_branches = self.d_branches - splitVarName = self.splitVarName - comparedBranches = [] - for bn in sorted(selected_br): - if bn.endswith('Aux.'): continue - base, vname = splitVarName( bn) - if vname is None : - continue - - l = d_branches.setdefault(base , [] ) - - def checkBranch(t,n): - b = t.GetBranch(n) - if b == None : - print ("!!!!!!!!! ",n,'branch does not exist !') - return b - # guess is the value is in MeV - isMeV = any ( bn.endswith( end ) for end in ('_e','_m','_pt', '.e','.m','.pt','Pt') ) - #print (' building ', bn, branch_replace(bn)) - v= VarPairBase.buildPair(vname, checkBranch(t1,bn) , checkBranch(t2,branch_replace(bn)), self.MAX_REL_DIFF , isMeV = isMeV) - if v.validVariable: - l.append( v ) - setattr(self, bn.replace('.','_') , v ) - comparedBranches +=[bn] - else: - print ("Don't know how to compare branch, ignoring : ", bn) - return d_branches - - - - def compare_event(self,evt1, evt2,stopOnError=False): - ok = True - for jet, l_var in self.d_branches.items(): - if stopOnError: - for v in l_var: - if not v.compare(evt1, evt2): - v.summary() - return False - else: - for v in l_var: - ok = v.compare(evt1, evt2) and ok #, doPrint = self.verbose) - return ok - - def resetBranches(self,one, evt): - tree = self.t1 if one else self.t2 - for jet, l_var in self.d_branches.items(): - for v in l_var: - v.resetBranchPointer(one, tree, evt) - - def full_compare(self,stopOnError=False,debug=True, maxEvent=10e10, detailedSummary=False): - global fullDebug, fullDebug_on, fullDebug_off - ok = True - fullDebug = fullDebug_on if debug else fullDebug_off - t1, t2 = self.t1, self.t2 - nEvt = min(maxEvent,self.t1.GetEntries(), self.t2.GetEntries() ) - for i in range(nEvt): - # support for TChain - evti1 = t1.LoadTree(i) - evti2 = t2.LoadTree(i) - # in case of TChain, one might need to reset branches. - if i!= evti1: self.resetBranches(True, evti1) - if i!= evti2: self.resetBranches(False, evti2) - - ok = ok & self.compare_event(evti1,evti2, stopOnError) - #print ('Event --->',i, ok, stopOnError) - if not ok and stopOnError: - break - - if not stopOnError or ok : - print() - err, values , wrongvalues = 0,0,0 - nvar = 0 - for jet, l_var in self.d_branches.items(): - nvar += len(l_var) - print ('Summary for ',jet, ' num var =', len(l_var),' **********************') - #print ('Variable with errors'.rjust(22)) - for v in l_var: - nerr , nvalues, nwrongvalues = v.summary(verbose=detailedSummary) - err += bool(nerr) - values += nvalues - wrongvalues +=nwrongvalues - print ('*******************************************') - print() - print ("Total Number of differing variable =",err, " / ", nvar) - print ("Total Number of differing values =",wrongvalues , " / " , values ) - return err - - if not ok: - return -999 - return 0 - #return ok - - -## ********************************************* -## Helpers to deal with xaod specifics -## (trick from Wim Lavrjisen) -cast_offset_str=""" -#include "TBranchElement.h" - #include "TObjArray.h" - #include "TStreamerElement.h" - #include "TStreamerInfo.h" - - Long_t cast_with_offset( TBranchElement* be ) { - Long_t offset = ((TStreamerElement*)be->GetInfo()->GetElements()->At(be->GetID()))->GetOffset(); - return (Long_t)(be->GetObject() + offset); - } - -""" -def initxAODHelper(): - from os.path import isfile - if not isfile("cast_with_offset.C"): - open("cast_with_offset.C","w").write(cast_offset_str) - ROOT.gROOT.LoadMacro("cast_with_offset.C+") - - # trick to run the actual code only once - initxAODHelper.func_code = (lambda:None).func_code - -def xAOD_initBranch( tree, name ): - - tree.SetBranchStatus( name, ROOT.kTRUE ) - be = tree.GetBranch( name ) - be.GetEntry(0) # tickle object creation - values = ROOT.BindObject( ROOT.cast_with_offset( be ), be.GetTypeName() ) - setattr( tree, name, values ) - return values - -def initStaticStoreBranch(tree, be): - initxAODHelper() - be.GetEntry(0) # tickle object creation - values = ROOT.BindObject( ROOT.cast_with_offset( be ), be.GetTypeName() ) - setattr( tree, be.GetName(), values ) - return values - -## ********************************************* - - - - - - - - - - -if __name__ == '__main__': - - - testedContainers = ["AntiKt4LCTopoJets","AntiKt10LCTopoJets"] - ################################################## - # Read and store argument values # - ################################################## - - parser = argparse.ArgumentParser(description='Compare variables from pairs of JetContainer and report if they differ') - - parser.add_argument('inFile1', type=str, help='1st input root file') - parser.add_argument('inFile2', type=str, default="", nargs='?', help='2nd input root file. If none, then container "XYZ" in inFile1 will be compared against "XYZSuffix" where Suffix is set by --suffix') - parser.add_argument('--treeName1', type=str, default="CollectionTree", help="Name of TTree (default: %(default)s)", metavar='name1') - parser.add_argument('--treeName2', type=str, default="CollectionTree", help="Name of TTree", metavar='name2') - parser.add_argument( '--containers', type=str, default= testedContainers, nargs='*',help="list of containers (ex: --containers cont1 cont2 cont3)", metavar='jetcontname') - parser.add_argument('--details', action='store_true', help="Display status for each variable, even if no error") - parser.add_argument('--ESD', action='store_true', help="Use this flag if both input files are ESD") - parser.add_argument('--AODvsESD', action='store_true', help="Use this flag if file1 is AOD and file2 is ESD") - parser.add_argument('--suffix', type=str, default='Test', help="In case using only 1 input files, uses this suffix to relate containers to be matched. Default to 'Test'") - parser.add_argument('--clusters', action='store_true', help="Add basic comparisons of clusters variables in CaloCalTopoClusterAux ") - - - args = parser.parse_args() - - # reset testedContainers : - testedContainers = args.containers - print ("Will test ", testedContainers) - - # if we have only 1 input file, we compare "XXXJets" against "XXXJetsTest" containers - if args.inFile2 == '': - # define a renamer function - branchRenamer = lambda n : n.replace('Jets','Jets'+args.suffix) - # testing only the 2 first entries - VarPairVector.maxVecSize = 2 - # do this as long as area calculation is not reproducible : - # in this case calib differs and misordering can happen for low pt jets - else: - # we compare containers with same name in 2 different files. - # No need to rename, renamer function does nothing : - branchRenamer = lambda n : n - - - # Filter -------------- - def branchFilter(s ): - """Return True if branch named s needs to be compared """ - return any( s.startswith( b ) for b in testedContainers ) #and "Area" not in s - - # Name splitting ---------- - # In order to nicely categorize results, the system needs to know to which container - # a variable "AntiKt4LCTopoJets.Var" belongs. The function below returns ("AntiKt4LCTopoJets","Var") - def splitVarName_xAOD( fullname ): - try: - base, varname = fullname.split('.') - return base, varname - except: - return "none", None - - # act if reading ESD ----------- - if args.ESD: - testedContainers = ["xAOD::JetAuxContainer_v1_"+b for b in testedContainers ] - elif args.AODvsESD: - branchRenamer = lambda n : "xAOD::JetAuxContainer_v1_"+n - - - - # Static aux container branches ----- - additionalBranches = [] - for c in testedContainers: - bname = c+'Aux.' - additionalBranches += [ bname+v for v in ("pt","eta","phi","m") ] - - # add basic clusters comp - if args.clusters: - bname = "CaloCalTopoClusterAux." - if args.ESD: - bname = "xAOD::CaloClusterAuxContainer_v1_"+bname - branchRenamer = lambda n : n.replace("_v1_CaloCalTopoCluster", "_v2_CaloCalTopoClusters") - additionalBranches += [bname+v for v in ("rawE","rawEta","rawPhi") ] - - # Build the main Comparator - vc = VarComparator( args.inFile1 , args.inFile2 if args.inFile2 else None , treename = args.treeName1, treename2=args.treeName2 )# , treename2="susy") - vc.splitVarName = splitVarName_xAOD - - - vc.init_trees( base_vars = [], - branch_filter = branchFilter, - branch_replace = branchRenamer, - checkMissing = False, - additionalBranches = additionalBranches, - ) - - #numErr = vc.full_compare(debug=True, maxEvent=1) - numErr = vc.full_compare(debug=False, maxEvent=100, detailedSummary=args.details) - - exit(numErr) # RTT: success if exit code is 0, failure otherwise - - diff --git a/Reconstruction/Jet/JetValidation/python/JetTesters.py b/Reconstruction/Jet/JetValidation/python/JetTesters.py deleted file mode 100644 index 705d01862b5c..000000000000 --- a/Reconstruction/Jet/JetValidation/python/JetTesters.py +++ /dev/null @@ -1,383 +0,0 @@ -# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration - -## @file: JetExamples/python/JetTesters.py -## @purpose: Algs to quickly dump jet collection content intot txt files and/or ntuples -## @author: P-A Delsart <delsart in2p3 fr> -## -## -## - -from __future__ import print_function - -__doc__ = '''Algs to quickly dump jet variables intot txt files and/or ntuples. - -Used in Run 1. Could be also useful in Run 2 ?? - - -This module implements a mini D3PD framework but - - allows to dump txt files - - written mostly in python so it is very quickly hackable -This allows immediate tests and comparisons and is mostly intended for testing jet devellopment -and debugging. - -There is a main python Algorithm JetTester : it is dedicated to one JetCollection and holds a list -of tools inheriting JetTestToolBase. -Each of these tools act on the JetCollection by inspecting and umping some variables in a common txt -file and/or root ntuple. - -There is also a tool written in plain c++ (ROOT.JetTesterCpp) used to test jet feature from the -c++ side. It is adapted below to be usable in the python side. - -Some configuration helper functions in python/JetTestConfig. - - -''' -__version__ = '$Revision: 1.5 $' -__author__ = 'P-A Delsart <delsart in2p3 fr>' - -import AthenaCommon.SystemOfUnits as Units -import AthenaPython.PyAthena as PyAthena -from AthenaPython.PyAthena import StatusCode - -import ROOT -import array - - -class JetTestToolBase(object): - def initialize(self, testerAlg): - self.tree = testerAlg.outTree - self.testerAlg = testerAlg - self.vectors = [] - self.prefix = self.testerAlg.CollectionName.replace('Jets','') - self.writetxt = self.testerAlg.outTxtFile.write - - def setupEvent(self, jetColl): - size = jetColl.size() - for v in self.vectors: - v.clear() - v.resize(size) - - def processJetCollection(self, jetColl): - pass - def processJet(self,i, jet): - pass - def finalizeEvent(self): - pass - - def buildPrefix(self): - return - - def makeBranch(self,name,btype, prefix = None): - if prefix is None: - prefix = self.prefix+'_' - bname = prefix+name - if isinstance(btype, str): - # assume it is a simple type. - obj = array.array(btype,[0]) - branch = self.tree.Branch(bname, obj, bname+'/'+btype.upper() ) - else: - # assume it is a vector - obj = btype() - branch = self.tree.Branch(bname, obj ) - self.vectors.append(obj) - - return obj - - - -class JetCollectionTestTool(JetTestToolBase): - def dump(self, txt): - self.writetxt( self.prefix+' %s\n'%(txt,) ) - -class JetTestTool(JetTestToolBase): - def blockDump(self,doBlock): - if doBlock: - self.dump = self.dump_dummy - else: - self.dump = self.dump_real - def dump(self, i, txt): - pass - def dump_dummy(self,i, txt): - pass - def dump_real(self, i, txt): - self.writetxt( self.prefix+' %2d %s\n'%(i,txt) ) - - -class JetCollN(JetCollectionTestTool): - def initialize(self, testerAlg): - super(JetCollN,self).initialize(testerAlg) - self.n = self.makeBranch('N','i') - def processJetCollection(self, jetColl): - n = jetColl.size() - self.n[0] = n - self.dump('Num Jet = %d'%(n,)) - - -class JetKinematics(JetTestTool): - def __init__(self, sigstate = 1, dopxpypz=False, dorap=False): - self.sigstate = sigstate - self.dopxpypz = dopxpypz - self.dorap = dorap - - def initialize(self, testerAlg): - super(JetKinematics,self).initialize(testerAlg) - vect_f = ROOT.vector(float) - self.e = self.makeBranch("e",vect_f) - self.pt = self.makeBranch("pt",vect_f) - self.eta = self.makeBranch("eta",vect_f) - self.phi = self.makeBranch("phi",vect_f) - if self.dopxpypz: - self.px = self.makeBranch("px",vect_f) - self.py = self.makeBranch("py",vect_f) - self.pz = self.makeBranch("pz",vect_f) - - def processJet(self, i, jet): - e, pt , eta, phi = jet.e(), jet.pt(), jet.eta(), jet.phi() - self.e[i] = e - self.pt[i] = pt - self.eta[i] = eta - self.phi[i] = phi - self.dump(i, 'final e, pt, (eta,phi) = %10.2f , %10.2f (% 1.3f,% 1.3f)'%(e,pt,eta,phi) ) - if self.dopxpypz: - px, py, pz = jet.px(), jet.py(), jet.pz() - self.dump(i, ' px, py, pz = %10.2f , %10.2f , %10.2f'%(px,py,pz) ) - self.px[i] = px - self.py[i] = py - self.pz[i] = pz - -class JetSigState(JetTestTool): - def initialize(self, testerAlg): - super(JetSigState,self).initialize(testerAlg) - vect_f = ROOT.vector(float) - self.em_pt = self.makeBranch("emscale_pt",vect_f) - self.em_eta = self.makeBranch("emscale_eta",vect_f) - self.c_pt = self.makeBranch("constscale_pt",vect_f) - self.c_eta = self.makeBranch("constscale_eta",vect_f) - - def processJet(self, i, jet): - for s,n in [ (0,'emscale '), (2,'constscale')]: - e, pt , eta, phi = jet.e(s), jet.pt(s), jet.eta(s), jet.phi(s) - ptV, etaV = self.vectors[s], self.vectors[s+1] - ptV[i] = pt - etaV[i] = eta - self.dump(i, n+' e, pt, (eta,phi) = %10.2f , %10.2f (% 1.3f,% 1.3f)'%(e,pt,eta,phi) ) - -class JetAuthor(JetTestTool): - def processJet(self, i, jet): - self.dump(i, 'author=%s . author+calibtag=%s'%(jet.jetAuthor(), jet.jetAuthorAndCalibTags()) ) - -class JetNConstituents(JetTestTool): - def initialize(self, testerAlg): - super(JetNConstituents,self).initialize(testerAlg) - vect_i = ROOT.vector(int) - self.size = self.makeBranch("size", vect_i) - self.constN = self.makeBranch("constN", vect_i) - - def processJet(self, i, jet): - s , n = jet.size(), jet.constituentsN() - self.size[i] = s - self.constN[i] = n - self.dump(i, "size =%d constitN=%d"%(s,n)) - - - -class JetMoments(JetTestTool): - def __init__(self, moments=['LArQuality', 'Timing', 'HECQuality', 'NegativeE', 'energy_EMB2', 'JVF', 'nTrk', 'sumPtTrk', 'NumTowers','WIDTH']): - self.moments = moments - def initialize(self, testerAlg): - super(JetMoments,self).initialize(testerAlg) - vect_f = ROOT.vector(float) - for m in self.moments: - self.makeBranch(m,vect_f) - - def processJet(self, i, jet): - getM = jet.getMoment - for m,v in zip(self.moments,self.vectors): - mV = getM(m) - v[i] = mV - self.dump(i, '%s %f'%(m,mV) ) - -class JetConstituents(JetTestTool): - def initialize(self, testerAlg): - super(JetConstituents,self).initialize(testerAlg) - vect = ROOT.vector(float) - self.c_emscale_e = self.makeBranch("const0_emscale_e", vect) - self.c_default_e = self.makeBranch("const0_default_e", vect) - - def processJet(self, i, jet): - print ('ERROR JetTesters.py, JetConstituents : does not weork with Run2 software !!') - # this is run 1 : this won't work - ## it0 = ROOT.JetConstituentIterator.first(jet,0) - ## itD = ROOT.JetConstituentIterator.first(jet) - ## itE = ROOT.JetConstituentIterator.last(jet) - ## count = 0 - ## list_e0 =[] - ## list_ed = [] - ## while (it0 != itE) and count<3: - ## e0, eD = it0.e(), itD.e() - ## if count == 0: - ## self.c_emscale_e[i] = e0 - ## self.c_default_e[i] = eD - ## list_e0.append(e0) - ## list_ed.append(eD) - ## it0.inc() - ## itD.inc() - ## count +=1 - ## self.dump(i, 'constit emscale e =' + ', '.join('%10.2f'%(e,) for e in list_e0) ) - ## self.dump(i, 'constit default e =' + ', '.join('%10.2f'%(e,) for e in list_ed) ) - - - - - - - -## class JetCppTester(ROOT.JetTesterCpp, JetTestTool, JetCollectionTestTool): - -## # recopy the dumping functions... -## dump_jcoll = JetCollectionTestTool.dump - -## def __init__(self, **args): -## ROOT.JetTesterCpp.__init__(self) - -## # set the c++ flags. False by default. -## for f in ["CellNavig","SigStateHelper"]: -## setattr(self, 'm_do'+f, args.get(f,False) ) - - -## def initialize(self, testerAlg): -## JetTestTool.initialize(self,testerAlg) -## self.m_prefix = testerAlg.CollectionName.replace('Jets','')+'_' -## ROOT.JetTesterCpp.initialize(self, testerAlg.outTree) - -## def processJetCollection(self, jcoll): -## self.m_outputStrings.clear() -## ROOT.JetTesterCpp.processJetCollection(self,jcoll) -## for s in self.m_outputStrings: -## self.dump_jcoll(s) - -## def processJet(self, i ,jet): -## self.m_outputStrings.clear() -## ROOT.JetTesterCpp.processJet(self,jet) -## for s in self.m_outputStrings: -## self.dump(i,s) - - -# ######################################################################### -# The Athena Algorith -# ######################################################################### -class JetTester (PyAthena.Alg): - outFileName = "jettest" - outRootFile = None - outTxtFile = None - outTree = None - enum = -1 - - fillingInstance = False - lastFiller = None - @classmethod - def setFillingInstance(cls, inst): - if cls.lastFiller : - cls.lastFiller.fillingInstance = False - cls.lastFiller = inst - inst.fillingInstance = True - - def __init__(self, name='JetTester', **kw): - ## init base class - kw['name'] = name - super(JetTester, self).__init__(**kw) - - ## properties and data members - self.CollectionType = kw.get('CollectionType', 'JetCollection') # default value - self.CollectionName = kw.get('CollectionName', '') # default value - self.JetTestTools = kw.get('JetTestTools', []) # default value - self.NJetTxt = kw.get('NJetTxt', 5) # default value - self.NoTxt = kw.get('NoTxt', False) - return - - def initialize(self): - self.msg.info('==> initialize...') - if not self.outRootFile: - JetTester.outRootFile = ROOT.TFile(self.outFileName+".root","recreate") - JetTester.outTxtFile = open(self.outFileName+".txt","w") - JetTester.outRootFile.cd() - JetTester.outTree = ROOT.TTree("JetTestTree","jet tests") - - - if not isinstance(self.JetTestTools[0], JetCollN): - self.JetTestTools = [JetCollN()] + self.JetTestTools - - for t in self.JetTestTools: - t.initialize(self) - - - singleT =[] - collecT = [] - for t in self.JetTestTools: - if isinstance(t, JetCppTester) : - singleT.append(t) - collecT.append(t) - continue - elif isinstance(t, JetTestTool) : singleT.append(t) - elif isinstance(t, JetCollectionTestTool) : collecT.append(t) - - self.singleJetTesters = singleT - self.collectionJetTesters = collecT - - - self.sg = PyAthena.py_svc ('StoreGateSvc') - return StatusCode.Success - - - - def execute(self): - e=self.sg.retrieve('EventInfo') - enum = e.event_ID().event_number() - if enum != JetTester.enum: - JetTester.enum = enum - self.outTxtFile.write('\n') - self.outTxtFile.write(' ##################################################################### \n') - self.outTxtFile.write(' Event %d\n'%enum) - self.outTxtFile.write(' ##################################################################### \n') - jetColl = self.sg.retrieve(self.CollectionType, self.CollectionName) - - self.outTxtFile.write(' ------------------------------------------ \n') - for t in self.JetTestTools: - t.setupEvent(jetColl) - - for t in self.collectionJetTesters: - t.processJetCollection(jetColl) - - # reset Txt dumping - for t in self.singleJetTesters: - t.blockDump(self.NoTxt) - - for i,jet in enumerate(jetColl): - if (i>= self.NJetTxt): - for t in self.singleJetTesters: - t.blockDump(True) - else: - self.outTxtFile.write('\n') - for t in self.singleJetTesters: - t.processJet(i,jet) - - for t in self.JetTestTools: - t.finalizeEvent() - - # only one alg should call TTree::Fill (which alg is set by config) - if self.fillingInstance: - - self.outTree.Fill() - return StatusCode.Success - - def finalize(self): - self.msg.info('==> finalize... ') - if self.outRootFile: - self.outRootFile.cd() - JetTester.outTree.Write() - JetTester.outRootFile.Close() - JetTester.outTxtFile.close() - JetTester.outRootFile = None - return StatusCode.Success - - # class JetTester diff --git a/Reconstruction/Jet/JetValidation/python/PhysicsValidationHistos.py b/Reconstruction/Jet/JetValidation/python/PhysicsValidationHistos.py index 96ebef72f773..9d5242f5f76c 100644 --- a/Reconstruction/Jet/JetValidation/python/PhysicsValidationHistos.py +++ b/Reconstruction/Jet/JetValidation/python/PhysicsValidationHistos.py @@ -3,8 +3,7 @@ from __future__ import print_function from JetMonitoring.JetHistoTools import jhm, selectionAndHistos -from JetMonitoring.JetMonitoringConf import JetAttributeHisto, HistoDefinitionTool, JetMonitoringTool, JetKinematicHistos, JetContainerHistoFiller -from AthenaCommon.AppMgr import ToolSvc +from JetMonitoring.JetMonitoringConf import JetMonitoringTool, JetKinematicHistos, JetContainerHistoFiller from AthenaCommon.AppMgr import ServiceMgr as svcMgr from PyUtils.MetaReader import read_metadata @@ -97,9 +96,8 @@ def commonPhysValTool(container, refcontainer="", onlyKinematics = False, global jhm.TileBar0, jhm.TileBar1, jhm.TileExt0, - jhm.TileExt1, - - ] + jhm.TileExt1 + ] if "PFlow" in container: filler.HistoTools += [ @@ -209,7 +207,7 @@ athenaMonTool = JetMonitoringTool(HistoTools = [ ], IntervalType=8) # 8 == HistoGroupBase::all -if (isMC==False): +if not isMC: athenaMonTool = JetMonitoringTool(HistoTools = [ commonPhysValTool( "AntiKt4LCTopoJets", akt4refContainer ,globalSelection = globalSelection), commonPhysValTool( "AntiKt4EMTopoJets", akt4refContainer ,globalSelection = globalSelection), diff --git a/Reconstruction/Jet/JetValidation/python/RTTConfig.py b/Reconstruction/Jet/JetValidation/python/RTTConfig.py index 085f309ef79c..cfa18e539041 100644 --- a/Reconstruction/Jet/JetValidation/python/RTTConfig.py +++ b/Reconstruction/Jet/JetValidation/python/RTTConfig.py @@ -14,8 +14,6 @@ def scheduleRTTJetTests(): from JetRec.JetRecConf import JetAlgorithm from JetRec.JetRecUtils import interpretJetName - from JetRec.JetRecStandardToolManager import calib_topo_ungroomed_modifiers, topo_ungroomed_modifiers - #calibarg = 'calib' if jetFlags.applyCalibrationName()!="none" else None #calibarg = 'calib' if jetFlags.applyCalibrationName!= "none" else "none" -- GitLab