From 7054dd1da9119adcaa795f739a35aa25450a2dad Mon Sep 17 00:00:00 2001 From: Kyle Cormier Date: Mon, 20 Jul 2020 13:59:30 +0200 Subject: [PATCH 1/5] Fixing GenLeptons + adding unfolding vars Making sure GenLeptons are being properly dressed and including some of the unfolding variables Identified by Sebastien that were not yet included in the unfolding plotter. --- python/unfoldingDefinitions.py | 95 ++++++++++++++++++++++++++++++++++ python/unfoldingPlotter.py | 25 ++++++--- 2 files changed, 112 insertions(+), 8 deletions(-) diff --git a/python/unfoldingDefinitions.py b/python/unfoldingDefinitions.py index 75194e1..6e36b43 100644 --- a/python/unfoldingDefinitions.py +++ b/python/unfoldingDefinitions.py @@ -22,6 +22,10 @@ jet_eta_bins = {} jet_eta_bins['reco'] = [ EqBin(25, -2.4, 2.4 ) for jet_i in range(10) ] jet_eta_bins['gen'] = [ EqBin( 8, -2.4, 2.4 ) for jet_i in range(10) ] +njet_bins = {} +njet_bins['reco'] = EqBin(6, 4, 10) +njet_bins['gen'] = EqBin(6, 4, 10) + def makeJetPlots(sel, jets, uname, maxJet=4, obs_delim='_obsis'): plots = [] dname = uname + obs_delim @@ -36,6 +40,11 @@ def makeJetPlots(sel, jets, uname, maxJet=4, obs_delim='_obsis'): plotopts=utils.getOpts(uname, **{"log-y": False}))) # plots.append(Plot.make1D(f"{dname}_jet{i+1}_phi", jets[i].phi, sel, # EqBin(50 // binScaling, -3.1416, 3.1416), title=f"{utils.getCounter(i+1)} jet phi", plotopts=utils.getOpts(uname, **{"log-y": False}))) + + + plots.append(Plot.make1D(f"{dname}_Njets", op.rng_len(jets) , sel, + njet_bins[ph_space], + title="Njets", plotopts=utils.getOpts(uname) )) return plots def makeJetMigrationPlots(sel, jets, genJets, uname, maxJet=4, binScaling=1, obs_delim='_obsis'): @@ -54,6 +63,11 @@ def makeJetMigrationPlots(sel, jets, genJets, uname, maxJet=4, binScaling=1, obs plotopts=utils.getOpts(uname, **{"log-y": False}))) # plots.append(Plot.make1D(f"{dname}_jet{i+1}_phi", jets[i].phi, sel, # EqBin(50 // binScaling, -3.1416, 3.1416), title=f"{utils.getCounter(i+1)} jet phi", plotopts=utils.getOpts(uname, **{"log-y": False}))) + + + plots.append(Plot.make2D(f"{migration_tag}_{dname}_Njets", (op.rng_len(jets), op.rng_len(genJets) ), sel, + (njet_bins['reco'], njet_bins['gen']), + title="Migration Matrix Njets", plotopts=utils.getOpts(uname) )) return plots @@ -135,3 +149,84 @@ def makeExtraJetMigrationPlots(sel, jets, genJets, uname, obs_delim='_obsis'): return plots +max_mbb_bins = {} +max_mbb_bins['reco'] = EqBin(25, 0., 1000.) +max_mbb_bins['gen'] = VarBin([0., 50., 100., 150., 250., 500., 1000.]) + + +def makeMaxMbbPairPlots( sel, maxMbbPair, uname, obs_delim='_obsis'): + plots = [] + dname = uname + obs_delim + ph_space = 'gen' if sel.name.startswith('genLevel_') else 'reco' + + plots.append(Plot.make1D(f"{dname}_largest_Mbb", op.invariant_mass(maxMbbPair[0].p4, maxMbbPair[1].p4), + sel, max_mbb_bins[ph_space], title="Largest Mbb", plotopts=utils.getOpts(uname))) + + return plots + +def makeMaxMbbPairMigrationPlots( sel, maxMbbPair, genMaxMbbPair, uname, obs_delim='_obsis'): + plots = [] + dname = uname + obs_delim + migration_tag = 'migrationMatrix' + + plots.append(Plot.make2D(f"{migration_tag}_{dname}_largest_Mbb", + (op.invariant_mass(maxMbbPair[0].p4, maxMbbPair[1].p4), op.invariant_mass(genMaxMbbPair[0].p4, genMaxMbbPair[1].p4) ), + sel, (max_mbb_bins['reco'], max_mbb_bins['gen']), title="Migration Matrix Largest Mbb", plotopts=utils.getOpts(uname))) + + return plots + +bjet_pt_bins = {} +bjet_pt_bins['reco'] = EqBin( 25, 30., 180. ) +bjet_pt_bins['gen'] = VarBin( [30., 40., 50., 70., 110., 180.] ) + +def makeBJetPlots(sel, jets, uname, maxJet=4, obs_delim='_obsis'): + plots = [] + dname = uname + obs_delim + ph_space = 'gen' if sel.name.startswith('genLevel_') else 'reco' + + if maxJet >= 4: + plots.append(Plot.make1D(f"{dname}_bjet4_pt", jets[3].pt, sel, + bjet_pt_bins[ph_space], title=f"b jet4 p_{{T}} (GeV)", + plotopts=utils.getOpts(uname))) + + return plots + +def makeBJetMigrationPlots(sel, jets, genJets, uname, maxJet=4, obs_delim='_obsis'): + plots = [] + dname = uname + obs_delim + migration_tag = 'migrationMatrix' + + if maxJet >= 4: + plots.append(Plot.make2D(f"{migration_tag}_{dname}_bjet4_pt", (jets[3].pt, genJets[3].pt), sel, + (bjet_pt_bins['reco'], bjet_pt_bins['gen']), title=f"Migration Matrix b jet4 p_{{T}} (GeV)", + plotopts=utils.getOpts(uname))) + + return plots + + +##ljet_pt_bins = {} +##ljet_pt_bins['reco'] = EqBin( 25, 30., 230. ) +##ljet_pt_bins['gen'] = VarBin( [30., 50., 80., 120., 230.] ) +## +##def makeLightJetPlots(sel, jets, uname, maxJet=4, obs_delim='_obsis'): +## plots = [] +## dname = uname + obs_delim +## ph_space = 'gen' if sel.name.startswith('genLevel_') else 'reco' +## +## if len(jets): +## plots.append(Plot.make1D(f"{dname}_ljet3_pt", jets[2].pt, sel, +## ljet_pt_bins[ph_space], title=f"light jet3 p_{{T}} (GeV)", +## plotopts=utils.getOpts(uname))) +## +## return plots +## +##def makeLightJetMigrationPlots(sel, jets, genJets, uname, maxJet=4, obs_delim='_obsis'): +## plots = [] +## dname = uname + obs_delim +## migration_tag = 'migrationMatrix' +## +## if maxJet >= 3: +## plots.append(Plot.make2D(f"{migration_tag}_{dname}_light_jet3_pt", (jets[2].pt, genJets[2].pt), sel, +## (ljet_pt_bins['reco'], ljet_pt_bins['gen']), title=f"Migration Matrix light jet3 p_{{T}} (GeV)", +## plotopts=utils.getOpts(uname))) + diff --git a/python/unfoldingPlotter.py b/python/unfoldingPlotter.py index d5a334b..9f87f62 100644 --- a/python/unfoldingPlotter.py +++ b/python/unfoldingPlotter.py @@ -109,6 +109,7 @@ class unfoldingPlotter(NanoAODHistoModule): # 2 DeepFlavour Medium tagged jets bJets = defs.bTagDef(cleanedJets, era, "M", "btagDeepFlavB") + lightJets = defs.lightTagDef(cleanedJets, era, "M", "btagDeepFlavB") bTagDef4 = [ op.rng_len(bJets) >= 4 ] # B-tag reweighting: compute product of the shape SFs for all selected jets @@ -133,6 +134,7 @@ class unfoldingPlotter(NanoAODHistoModule): # Find couple of extra bjets bJetCouples = op.combine(bJets, N=2) minDRbbPair = op.rng_min_element_by(bJetCouples, lambda bcoup: op.deltaR(bcoup[0].p4, bcoup[1].p4)) + maxMbbPair = op.rng_max_element_by(bJetCouples, lambda pair: op.invariant_mass(pair[0].p4, pair[1].p4)) # For ttbar processes we need plots with only particle-level selection, for acceptance, # and plots with reco- and particle-level selection for migration matrices. @@ -149,8 +151,8 @@ class unfoldingPlotter(NanoAODHistoModule): genElectrons = op.select(t.GenDressedLepton, genDefs.eleDef) genElectron = genElectrons[0] - genVetoMuons = op.select(t.Muon, genDefs.vetoMuonDef) - genVetoElectrons = op.select(t.Electron, genDefs.vetoEleDef) + genVetoMuons = op.select(t.GenDressedLepton, genDefs.vetoMuonDef) + genVetoElectrons = op.select(t.GenDressedLepton, genDefs.vetoEleDef) ######### Jet definition genJets = op.select(t.GenJet, genDefs.jetDef) @@ -171,19 +173,23 @@ class unfoldingPlotter(NanoAODHistoModule): genBJetCouples = op.combine(genBJets, N=2) genMinDRbbPair = op.rng_min_element_by(genBJetCouples, lambda bcoup: op.deltaR(bcoup[0].p4, bcoup[1].p4)) + genMaxMbbPair = op.rng_max_element_by(genBJetCouples, lambda pair: op.invariant_mass(pair[0].p4, pair[1].p4)) genOnlySel = genOnlyOneLep6Jet4BSel - plots += unfDefs.makeJetPlots(genOnlySel, genCleanedJets, genOnlySel.name, maxJet=6) - plots += unfDefs.makeExtraJetPlots(genOnlySel, genMinDRbbPair, genOnlySel.name) + plots += unfDefs.makeJetPlots( genOnlySel, genCleanedJets, genOnlySel.name, maxJet=6) + plots += unfDefs.makeExtraJetPlots( genOnlySel, genMinDRbbPair, genOnlySel.name ) + plots += unfDefs.makeMaxMbbPairPlots( genOnlySel, genMaxMbbPair, genOnlySel.name ) + plots += unfDefs.makeBJetPlots( genOnlySel, genBJets, genOnlySel.name ) + #plots += unfDefs.makeLightJetPlots( genOnlySel, genLightJets, genOnlySel.name ) for sel,lep,name in [(oneMu6Jet4BSel, muon, "1mu_6j_4b"), (oneEle6Jet4BSel, electron, "1ele_6j_4b")]: + plots += unfDefs.makeJetPlots(sel, cleanedJets, name, maxJet=6 ) plots += unfDefs.makeExtraJetPlots(sel, minDRbbPair, name) - - #maxMbbPair = op.rng_max_element_by(bJetCouples, lambda pair: op.invariant_mass(pair[0].p4, pair[1].p4)) - #plots.append(Plot.make1D(f"{name}_largest_Mbb", op.invariant_mass(maxMbbPair[0].p4, maxMbbPair[1].p4), - # sel, EqBin(25, 0., 1000.), title="Largest Mbb", plotopts=utils.getOpts(name))) + plots += unfDefs.makeMaxMbbPairPlots( sel, maxMbbPair, name ) + plots += unfDefs.makeBJetPlots( sel, bJets, name ) + #plots += unfDefs.makeLightJetPlots( sel, lightJets, name ) if do_migrations: #Gen definitions previously initialized when making acceptance plots @@ -196,6 +202,9 @@ class unfoldingPlotter(NanoAODHistoModule): plots += unfDefs.makeJetMigrationPlots( genOneLep6Jet4BSel, cleanedJets, genCleanedJets, name, maxJet=6 ) plots += unfDefs.makeExtraJetMigrationPlots( genOneLep6Jet4BSel, minDRbbPair, genMinDRbbPair, name) + plots += unfDefs.makeMaxMbbPairMigrationPlots(genOneLep6Jet4BSel, maxMbbPair, genMaxMbbPair, name ) + plots += unfDefs.makeBJetMigrationPlots( genOneLep6Jet4BSel, bJets, genBJets, name ) + #plots += unfDefs.makeLightJetMigrationPlots( genOneLep6Jet4BSel, lightJets, genLightJets, name ) return plots -- GitLab From 3dfcc5ee93056a385301c5353665e07391224885 Mon Sep 17 00:00:00 2001 From: Kyle Cormier Date: Mon, 20 Jul 2020 14:01:13 +0200 Subject: [PATCH 2/5] GenLepton fix in GenPlotter Including dressing in definition. --- python/genTtbbPlotter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/genTtbbPlotter.py b/python/genTtbbPlotter.py index 6d5c809..abeffdb 100644 --- a/python/genTtbbPlotter.py +++ b/python/genTtbbPlotter.py @@ -76,8 +76,8 @@ class genTtbbPlotter(NanoAODHistoModule): genElectrons = op.select(t.GenDressedLepton, genDefs.eleDef) genElectron = genElectrons[0] - genVetoMuons = op.select(t.Muon, genDefs.vetoMuonDef) - genVetoElectrons = op.select(t.Electron, genDefs.vetoEleDef) + genVetoMuons = op.select(t.GenDressedLepton, genDefs.vetoMuonDef) + genVetoElectrons = op.select(t.GenDressedLepton, genDefs.vetoEleDef) ##### Muon and electron selection (exclusive!), merged genOneLepSel = genDefs.buildLeptonSelection(noSel, genMuons, genVetoMuons, genElectrons, genVetoElectrons) -- GitLab From 3ee7e4040bfd14593656f841a76f417c764fad28 Mon Sep 17 00:00:00 2001 From: Kyle Cormier Date: Tue, 25 Aug 2020 15:58:57 +0200 Subject: [PATCH 3/5] Minor update for unfoldingPlotter --- python/unfoldingPlotter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/unfoldingPlotter.py b/python/unfoldingPlotter.py index 9f87f62..45e97b1 100644 --- a/python/unfoldingPlotter.py +++ b/python/unfoldingPlotter.py @@ -139,7 +139,7 @@ class unfoldingPlotter(NanoAODHistoModule): # For ttbar processes we need plots with only particle-level selection, for acceptance, # and plots with reco- and particle-level selection for migration matrices. do_migrations = False - signal_groups = ['ttbb','ttcc','ttbj','ttjj','TT_other'] + signal_groups = ['ttbb','ttcc','ttbj','ttjj','TT_other','ttB'] if sampleCfg["group"] in signal_groups: @@ -217,7 +217,6 @@ class unfoldingPlotter(NanoAODHistoModule): for (inputs, output), kwargs in taskList: if self.doSysts and self.isMC(output): - utils.produceMEScaleEnvelopes(self.plotList, self.qcdScaleVariations, os.path.join(resultsdir, output)) # finally, run plotIt as defined in HistogramsModule -- GitLab From ddb204613f31431b5cce08aaab2548cff6bf3f68 Mon Sep 17 00:00:00 2001 From: Kyle Cormier Date: Fri, 28 Aug 2020 18:10:47 +0200 Subject: [PATCH 4/5] Updating postprocessing in unfoldingPlotter to deal with the 4fs5fs overlap --- python/unfoldingPlotter.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/python/unfoldingPlotter.py b/python/unfoldingPlotter.py index 45e97b1..983141f 100644 --- a/python/unfoldingPlotter.py +++ b/python/unfoldingPlotter.py @@ -209,18 +209,15 @@ class unfoldingPlotter(NanoAODHistoModule): return plots def postProcess(self, taskList, config=None, workdir=None, resultsdir=None): - # Get list of plots (taken from bamboo.HistogramsModule) - if not self.plotList: - tup, smpName, smpCfg = self.getATree() - tree, noSel, backend, runAndLS = self.prepareTree(tup, sample=smpName, sampleCfg=smpCfg) - self.plotList = self.definePlots(tree, noSel, sample=smpName, sampleCfg=smpCfg) - - for (inputs, output), kwargs in taskList: - if self.doSysts and self.isMC(output): - utils.produceMEScaleEnvelopes(self.plotList, self.qcdScaleVariations, os.path.join(resultsdir, output)) + self.plotList = self.getPlotList(resultsdir=resultsdir) + for task in taskList: + if self.doSysts and self.isMC(task.outputFile): + utils.produceMEScaleEnvelopes(self.plotList, self.qcdScaleVariations, os.path.join(resultsdir, task.outputFile)) # finally, run plotIt as defined in HistogramsModule # also remove the file lists from the yml config: - for smpName, smpCfg in config["samples"].items(): + for smpName,smpCfg in list(config["samples"].items()): smpCfg.pop("files") + # remove overlap between 4FS and 5FS ttbb/ttB contributions for plotIt + utils.remove4F5FOverlapPlotIt(config["samples"]) super(unfoldingPlotter, self).postProcess(taskList, config, workdir, resultsdir) -- GitLab From f4bb184123088c6e336883571d79a35661707037 Mon Sep 17 00:00:00 2001 From: Kyle Cormier Date: Wed, 30 Sep 2020 11:18:15 +0200 Subject: [PATCH 5/5] Update unfolding Plotter for theory syst handling Following the came logic as controlPlotter to do the appropriate post-processing of the theory systematics. --- python/unfoldingPlotter.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/python/unfoldingPlotter.py b/python/unfoldingPlotter.py index 983141f..cf987b5 100644 --- a/python/unfoldingPlotter.py +++ b/python/unfoldingPlotter.py @@ -13,6 +13,7 @@ from bamboo.root import addIncludePath, loadHeader from bamboo.root import gbl as ROOT import numpy as np +import shutil import definitions as defs import utils @@ -64,7 +65,8 @@ class unfoldingPlotter(NanoAODHistoModule): if self.doSysts: logger.info("Adding QCD scale variations") - noSel = utils.addTheorySystematics(self, sample, tree, noSel) + #noSel = utils.addTheorySystematics(self, sample, tree, noSel) + noSel = utils.addTheorySystematics(self, sample, sampleCfg, tree, noSel) if "subprocess" in sampleCfg: noSel = utils.splitTTjetFlavours(sampleCfg, tree, noSel) @@ -209,15 +211,31 @@ class unfoldingPlotter(NanoAODHistoModule): return plots def postProcess(self, taskList, config=None, workdir=None, resultsdir=None): + # create backup directory with all merged results and make changes in that one + resultsdir_bu = os.path.join(workdir, "results_workdir") + if not os.path.isdir(resultsdir_bu): + shutil.copytree(resultsdir, resultsdir_bu) + resultsdir = resultsdir_bu + # remove batch output files (not needed anymore since they've been copied/merged into the results directory) + batchOut = os.path.join(workdir, "batch", "output") + if os.path.isdir(batchOut): + shutil.rmtree(batchOut) + self.plotList = self.getPlotList(resultsdir=resultsdir) for task in taskList: if self.doSysts and self.isMC(task.outputFile): utils.produceMEScaleEnvelopes(self.plotList, self.qcdScaleVariations, os.path.join(resultsdir, task.outputFile)) + # renormalize, copy and rename histograms from systematic variation files into nominal files + # also remove the systematic samples from the plotIt list + if self.doSysts: + utils.postProcSystSamples(taskList, config["samples"], resultsdir) + # finally, run plotIt as defined in HistogramsModule # also remove the file lists from the yml config: for smpName,smpCfg in list(config["samples"].items()): smpCfg.pop("files") + # remove overlap between 4FS and 5FS ttbb/ttB contributions for plotIt utils.remove4F5FOverlapPlotIt(config["samples"]) super(unfoldingPlotter, self).postProcess(taskList, config, workdir, resultsdir) -- GitLab