From 798fc8ab9852e28a3a479784ce7dcdf3bc047b28 Mon Sep 17 00:00:00 2001 From: alfroch Date: Mon, 22 Feb 2021 12:57:40 +0000 Subject: [PATCH 1/5] Merge Dips_Develop --- examples/plotting_eval_config_dips.yaml | 57 +++++-- umami/evaluation_tools/PlottingFunctions.py | 155 ++++++++++++++++++- umami/evaluation_tools/__init__.py | 2 +- umami/plotting_epoch_performance.py | 18 ++- umami/plotting_umami.py | 158 ++++++++------------ umami/train_Dips.py | 40 +++-- umami/train_tools/NN_tools.py | 117 ++++++++++++++- umami/train_tools/Plotting.py | 132 +++++++++++----- umami/train_tools/__init__.py | 3 + 9 files changed, 510 insertions(+), 172 deletions(-) diff --git a/examples/plotting_eval_config_dips.yaml b/examples/plotting_eval_config_dips.yaml index ccea4bd4..939d19a3 100644 --- a/examples/plotting_eval_config_dips.yaml +++ b/examples/plotting_eval_config_dips.yaml @@ -10,6 +10,8 @@ scores_Dips_ttbar: data_set_name: "ttbar" # data set to use. This chooses either the test_file ('ttbar') or the add_test_file ('zpext') prediction_labels: ["dips_pb", "dips_pc", "dips_pu"] # For umami use umami_pX or dips_pX text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$t\\bar{t}$ Test Sample" + AtlasTag: "Simulation, Work In Progress" + WorkingPoints: [0.77, 0.85] # Set Working Point Lines in plot confusion_matrix_Dips_ttbar: type: "confusion_matrix" @@ -25,16 +27,18 @@ Dips_light_flavour_ttbar: df_key: "rnnip_urej" # This is the rejection rate of the currently recommended DL1r tagger dips: # The name here has no impact on anything. data_set_name: "ttbar" - label: "Dips" + label: "DIPS" df_key: "dips_urej" # This is actually the rejection rate of the DL1r tragger trained with the umami framework plot_settings: # These settings are given to the umami.evaluation_tools.plotROCRatio() function by unpacking them. ylabel: "light" binomialErrors: true - xmin: 0.6 - ymax: 1e5 + xmin: 0.5 + ymax: 1000000 colors: ["orange", "blue"] figsize: [7, 6] # [width, hight] text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$t\\bar{t}$ Test Sample, fc=0.018" + AtlasTag: "Simulation, Work In Progress" + WorkingPoints: [0.77, 0.85] Dips_c_flavour_ttbar: type: "ROC" @@ -45,16 +49,18 @@ Dips_c_flavour_ttbar: df_key: "rnnip_crej" dips: data_set_name: "ttbar" - label: "Dips" + label: "DIPS" df_key: "dips_crej" plot_settings: ylabel: "c" binomialErrors: true - xmin: 0.6 + xmin: 0.5 ymax: 1000 colors: ["orange", "blue"] figsize: [7, 6] # [width, hight] text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$t\\bar{t}$ Test Sample, fc=0.018" + AtlasTag: "Simulation, Work In Progress" + WorkingPoints: [0.77, 0.85] # Dips, zpext @@ -63,6 +69,7 @@ scores_Dips_zpext: data_set_name: "zpext" # data set to use. This chooses either the test_file ('ttbar') or the add_test_file ('zpext') prediction_labels: ["dips_pb", "dips_pc", "dips_pu"] # For umami use umami_pX or dips_pX text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$Z'$ Test Sample" + AtlasTag: "Simulation, Work In Progress" confusion_matrix_Dips_zpext: type: "confusion_matrix" @@ -78,16 +85,18 @@ Dips_light_flavour_zpext: df_key: "rnnip_urej" # This is the rejection rate of the currently recommended DL1r tagger dips: # The name here has no impact on anything. data_set_name: "zpext" - label: "Dips" + label: "DIPS" df_key: "dips_urej" # This is actually the rejection rate of the DL1r tragger trained with the umami framework plot_settings: # These settings are given to the umami.evaluation_tools.plotROCRatio() function by unpacking them. ylabel: "light" binomialErrors: true - xmin: 0.6 + xmin: 0.5 ymax: 1000 colors: ["orange", "blue"] figsize: [7, 6] # [width, hight] text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$Z'$ Test Sample, fc=0.018" + AtlasTag: "Simulation, Work In Progress" + WorkingPoints: [0.77, 0.85] Dips_c_flavour_zpext: type: "ROC" @@ -98,23 +107,39 @@ Dips_c_flavour_zpext: df_key: "rnnip_crej" dips: data_set_name: "zpext" - label: "Dips" + label: "DIPS" df_key: "dips_crej" plot_settings: ylabel: "c" binomialErrors: true - xmin: 0.6 + xmin: 0.5 ymax: 100 colors: ["orange", "blue"] figsize: [7, 6] # [width, hight] text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$Z'$ Test Sample, fc=0.018" + AtlasTag: "Simulation, Work In Progress" + WorkingPoints: [0.77, 0.85] Dips_saliency_b_WP77_passed_ttbar: type: "saliency" - data_set_name: "ttbar" - title: "Saliency map for $b$ jets from \n $t\\bar{t}$ who passed WP = 77% \n with exactly 8 tracks" - evaluation_file: # Can be set manual. Overwrites the global eval parameters for this plot - WP: 0.77 - # u=0, c=1, b=2 - flavor: 2 - passed: True + plot_settings: + data_set_name: "ttbar" + title: "Saliency map for $b$ jets from \n $t\\bar{t}$ who passed WP = 77% \n with exactly 8 tracks" + target_beff: 0.77 + # u=0, c=1, b=2 + jet_flavour: 2 + PassBool: True + AtlasTag: "Simulation, Work In Progress" + FlipAxis: True + +Dips_saliency_b_WP77_failed_ttbar: + type: "saliency" + plot_settings: + data_set_name: "ttbar" + title: "Saliency map for $b$ jets from \n $t\\bar{t}$ who failed WP = 77% \n with exactly 8 tracks" + target_beff: 0.77 + # u=0, c=1, b=2 + jet_flavour: 2 + PassBool: False + AtlasTag: "Simulation, Work In Progress" + FlipAxis: True diff --git a/umami/evaluation_tools/PlottingFunctions.py b/umami/evaluation_tools/PlottingFunctions.py index a172fdc9..f48ddefd 100644 --- a/umami/evaluation_tools/PlottingFunctions.py +++ b/umami/evaluation_tools/PlottingFunctions.py @@ -1,9 +1,13 @@ +import pickle + import keras.backend as K import matplotlib.pyplot as plt import numpy as np from matplotlib import gridspec +from mpl_toolkits.axes_grid1 import make_axes_locatable from scipy.interpolate import pchip +import umami.tools.PyATLASstyle.PyATLASstyle as pas from umami.tools.PyATLASstyle.PyATLASstyle import makeATLAStag @@ -61,6 +65,7 @@ def plotROCRatio( text="", ylabel="Background rejection", tag="", + AtlasTag="Internal Simulation", figDir="../figures", subDir="mc16d", styles=None, @@ -78,6 +83,7 @@ def plotROCRatio( legcols=2, labelpad=None, which_axis="left", + WorkingPoints=None, x_label="$b$-jet efficiency", ylabel_right=None, ratio_id=0, @@ -153,6 +159,27 @@ def plotROCRatio( axis_dict["right"] = {} axis_dict["right"]["top"] = axis_dict["left"]["top"].twinx() + if WorkingPoints is not None: + for WP in WorkingPoints: + axis_dict["left"]["top"].axvline( + x=WP, ymax=0.7, color="red", linestyle="dashed", linewidth=1.0 + ) + + axis_dict["left"]["ratio"].axvline( + x=WP, color="red", linestyle="dashed", linewidth=1.0 + ) + + # Set the number above the line + axis_dict["left"]["top"].annotate( + text="{}%".format(int(WP * 100)), + xy=(WP, 0.82), + textcoords="offset points", + xycoords=("data", "figure fraction"), + ha="center", + va="bottom", + size=10, + ) + lines = [] f0_ratio = {} for i, (teff, beff, label, style, color, nte, which_a, r_id) in enumerate( @@ -275,9 +302,7 @@ def plotROCRatio( ncol=legcols, ) # , title="DL1r") - makeATLAStag( - axis_dict["left"]["top"], fig, "Internal Simulation", text, ymax=0.8 - ) + makeATLAStag(axis_dict["left"]["top"], fig, AtlasTag, text, ymax=0.9) plt.tight_layout() if len(tag) != 0: plt.savefig( @@ -289,3 +314,127 @@ def plotROCRatio( plt.savefig(plot_name, transparent=True) plt.close() # plt.show() + + +def plotSaliency( + plot_name, + FileDir, + epoch, + data_set_name, + title, + target_beff=0.77, + jet_flavour=2, + PassBool=True, + nFixedTrks=8, + fs=14, + xlabel="Tracks sorted by $s_{d0}$", + AtlasTag="Internal Simulation", + SecondTag=r"$\sqrt{s}$ = 13 TeV, $t\bar{t}$ PFlow Jets", + FlipAxis=False, +): + # Transform to percent + target_beff = 100 * target_beff + + # Little Workaround + AtlasTag = " " + AtlasTag + + with open(FileDir + f"/saliency_{epoch}_{data_set_name}.pkl", "rb") as f: + maps_dict = pickle.load(f) + + gradient_map = maps_dict[ + "{}_{}_{}".format(int(target_beff), jet_flavour, PassBool) + ] + + colorScale = np.max(np.abs(gradient_map)) + cmaps = ["RdBu", "PuOr", "PiYG"] + + nFeatures = gradient_map.shape[0] + + if FlipAxis is True: + fig = plt.figure(figsize=(0.7 * nFeatures, 0.7 * nFixedTrks)) + gradient_map = np.swapaxes(gradient_map, 0, 1) + + plt.yticks( + np.arange(nFixedTrks), np.arange(1, nFixedTrks + 1), fontsize=fs + ) + + plt.ylabel(xlabel, fontsize=fs) + plt.ylim(-0.5, nFixedTrks - 0.5) + + # ylabels. Order must be the same as in the Vardict + xticklabels = [ + "$s_{d0}$", + "$s_{z0}$", + "PIX1 hits", + "IBL hits", + "shared IBL hits", + "split IBL hits", + "shared pixel hits", + "split pixel hits", + "shared SCT hits", + r"$\log \ p_T^{frac}$", + r"$\log \ \Delta R$", + "nPixHits", + "nSCTHits", + "$d_0$", + r"$z_0 \sin \theta$", + ] + + plt.xticks(np.arange(nFeatures), xticklabels[:nFeatures], rotation=45) + + else: + fig = plt.figure(figsize=(0.7 * nFixedTrks, 0.7 * nFeatures)) + + plt.xticks( + np.arange(nFixedTrks), np.arange(1, nFixedTrks + 1), fontsize=fs + ) + + plt.xlabel(xlabel, fontsize=fs) + plt.xlim(-0.5, nFixedTrks - 0.5) + + # ylabels. Order must be the same as in the Vardict + yticklabels = [ + "$s_{d0}$", + "$s_{z0}$", + "PIX1 hits", + "IBL hits", + "shared IBL hits", + "split IBL hits", + "shared pixel hits", + "split pixel hits", + "shared SCT hits", + r"$\log \ p_T^{frac}$", + r"$\log \ \Delta R$", + "nPixHits", + "nSCTHits", + "$d_0$", + r"$z_0 \sin \theta$", + ] + + plt.yticks(np.arange(nFeatures), yticklabels[:nFeatures]) + + im = plt.imshow( + gradient_map, + cmap=cmaps[jet_flavour], + origin="lower", + vmin=-colorScale, + vmax=colorScale, + ) + + plt.title(title, fontsize=fs) + + ax = plt.gca() + pas.makeATLAStag( + ax, fig, first_tag=AtlasTag, second_tag=SecondTag, ymax=0.925 + ) + + # Plot colorbar and set size to graph size + divider = make_axes_locatable(ax) + cax = divider.append_axes("right", size="5%", pad=0.05) + colorbar = plt.colorbar(im, cax=cax) + colorbar.ax.set_title( + r"$\frac{\partial D_{b}}{\partial x_{ik}}$", size=1.5 * fs + ) + + # Save the figure + plt.savefig(plot_name, transparent=True, bbox_inches="tight") diff --git a/umami/evaluation_tools/__init__.py b/umami/evaluation_tools/__init__.py index ea38166a..c1a8bd86 100644 --- a/umami/evaluation_tools/__init__.py +++ b/umami/evaluation_tools/__init__.py @@ -1,3 +1,3 @@ # flake8: noqa from umami.evaluation_tools.Configuration import Configuration -from umami.evaluation_tools.PlottingFunctions import plotROCRatio +from umami.evaluation_tools.PlottingFunctions import plotROCRatio, plotSaliency diff --git a/umami/plotting_epoch_performance.py b/umami/plotting_epoch_performance.py index 59226396..6a5aab98 100644 --- a/umami/plotting_epoch_performance.py +++ b/umami/plotting_epoch_performance.py @@ -56,6 +56,12 @@ def GetParser(): --cfrac, --beff and --nJets have no impact on anything!""", ) + parser.add_argument( + "--dips", + action="store_true", + help="Setting the model epoch performance to dips.", + ) + args = parser.parse_args() return args @@ -76,9 +82,15 @@ def main(args, train_config, preprocess_config): beff = args.beff cfrac = args.cfrac - utt.plot_validation( - train_config, beff, cfrac, dict_file_name=output_file_name - ) + if args.dips: + utt.plot_validation_dips( + train_config, beff, cfrac, dict_file_name=output_file_name + ) + + else: + utt.plot_validation( + train_config, beff, cfrac, dict_file_name=output_file_name + ) if __name__ == "__main__": diff --git a/umami/plotting_umami.py b/umami/plotting_umami.py index bb198b79..a96af1a3 100644 --- a/umami/plotting_umami.py +++ b/umami/plotting_umami.py @@ -11,18 +11,15 @@ and has to be specified in the config file as 'evaluation_file'. import argparse import os -import pickle import h5py import matplotlib.pyplot as plt import numpy as np import pandas as pd import yaml -from mpl_toolkits.axes_grid1 import make_axes_locatable from yaml.loader import FullLoader import umami.evaluation_tools as uet -import umami.tools.PyATLASstyle.PyATLASstyle as pas from umami.evaluation_tools.PlottingFunctions import GetScore from umami.tools.PyATLASstyle.PyATLASstyle import makeATLAStag @@ -169,6 +166,12 @@ def plot_score(plot_name, plot_config, eval_params, eval_file_dir): # Get the epoch which is to be evaluated eval_epoch = int(eval_params["epoch"]) + if "AtlasTag" in list(plot_config.keys()): + AtlasTag = plot_config["AtlasTag"] + + else: + AtlasTag = "Internal Simulation" + if ("evaluation_file" not in plot_config) or ( plot_config["evaluation_file"] is None ): @@ -185,7 +188,10 @@ def plot_score(plot_name, plot_config, eval_params, eval_file_dir): df_results["discs"] = GetScore( *[df_results[pX] for pX in plot_config["prediction_labels"]] ) + plt.clf() + fig = plt.figure() + ax = fig.gca() plt.hist( [ df_results.query("labels==2")["discs"], @@ -199,104 +205,63 @@ def plot_score(plot_name, plot_config, eval_params, eval_file_dir): density=1, label=["b-jets", "c-jets", "l-jets"], ) + + # Increase ymax so atlas tag don't cut plot + ymin, ymax = plt.ylim() + plt.ylim(ymin=ymin, ymax=1.2 * ymax) + + # Set WP vertical lines if given in config + if "WorkingPoints" in list(plot_config.keys()): + + # Get WP from config + WP_List = plot_config["WorkingPoints"] + + # Iterate over WPs + for WP in WP_List: + + # Calculate x value of WP line + x_value = np.percentile( + df_results.query("labels==2")["discs"], (1 - WP) * 100 + ) + + # Draw WP line + plt.vlines( + x=x_value, + ymin=ymin, + ymax=0.8 * ymax, + colors="gray", + linestyles="dashed", + linewidth=1.0, + ) + + # Set the number above the line + ax.annotate( + "{}%".format(int(WP * 100)), + xy=(x_value, 0.8 * ymax), + textcoords="offset points", + ha="center", + va="bottom", + size=10, + ) + plt.legend() plt.xlabel("$D_{b}$") + text = "" if "text" in plot_config: text = plot_config["text"] - makeATLAStag(plt.gca(), plt.gcf(), "Internal Simulation", text) - plt.tight_layout() - plt.savefig(plot_name, transparent=True) - plt.close() - -def plot_saliency_maps( - plot_name, - plot_config, - eval_params, - eval_file_dir, - fs=14, - xlabel="Tracks sorted by $s_{d0}$", - firstTag="Internal Simulation", - secondTag=r"$\sqrt{s}$ = 13 TeV, $t\bar{t}$ PFlow Jets", -): - # Get parameters to determine gradient map - eval_epoch = int(eval_params["epoch"]) - target_beff = int(100 * plot_config["WP"]) - jet_flavour = int(plot_config["flavor"]) - PassBool = bool(plot_config["passed"]) - title = plot_config["title"] - df_name = plot_config["data_set_name"] - nFixedTrks = 8 - - if plot_config["evaluation_file"] is None: - with open( - eval_file_dir + f"/saliency_{eval_epoch}_{df_name}.pkl", "rb" - ) as f: - maps_dict = pickle.load(f) - - gradient_map = maps_dict[f"{target_beff}_{jet_flavour}_{PassBool}"] - - colorScale = np.max(np.abs(gradient_map)) - cmaps = ["RdBu", "PuOr", "PiYG"] - - nFeatures = gradient_map.shape[0] - fig = plt.figure(figsize=(0.7 * nFixedTrks, 0.7 * nFeatures)) - - im = plt.imshow( - gradient_map, - cmap=cmaps[jet_flavour], - origin="lower", - vmin=-colorScale, - vmax=colorScale, - ) - - plt.xticks( - np.arange(nFixedTrks), np.arange(1, nFixedTrks + 1), fontsize=fs + makeATLAStag( + ax=plt.gca(), + fig=plt.gcf(), + first_tag=AtlasTag, + second_tag=text, + ymax=0.9, ) - plt.xlabel(xlabel, fontsize=fs) - plt.xlim(-0.5, nFixedTrks - 0.5) - - # ylabels. Order must be the same as in the Vardict - yticklabels = [ - "$s_{d0}$", - "$s_{z0}$", - "PIX1 hits", - "IBL hits", - "shared IBL hits", - "split IBL hits", - "shared pixel hits", - "split pixel hits", - "shared SCT hits", - r"$\log \ p_T^{frac}$", - r"$\log \ \Delta R$", - "nPixHits", - "nSCTHits", - "$d_0$", - r"$z_0 \sin \theta$", - ] - - plt.yticks(np.arange(nFeatures), yticklabels[:nFeatures]) - - plt.title(title, fontsize=fs) - - ax = plt.gca() - pas.makeATLAStag( - ax, fig, first_tag=firstTag, second_tag=secondTag, ymax=0.925 - ) - - # Plot colorbar and set size to graph size - divider = make_axes_locatable(ax) - cax = divider.append_axes("right", size="5%", pad=0.05) - colorbar = plt.colorbar(im, cax=cax) - colorbar.ax.set_title( - r'$\frac{\partial D_{b}}{\partial x_{ik}}$', - size=1.5 * fs - ) - - # Save the figure - plt.savefig(plot_name, transparent=True, bbox_inches="tight") + plt.tight_layout() + plt.savefig(plot_name, transparent=True) + plt.close() def SetUpPlots(plotting_config, plot_directory, eval_file_dir, format): @@ -330,8 +295,11 @@ def SetUpPlots(plotting_config, plot_directory, eval_file_dir, format): plot_score(save_plot_to, plot_config, eval_params, eval_file_dir) elif plot_config["type"] == "saliency": - plot_saliency_maps( - save_plot_to, plot_config, eval_params, eval_file_dir + uet.plotSaliency( + plot_name=save_plot_to, + FileDir=eval_file_dir, + epoch=int(eval_params["epoch"]), + **plot_config["plot_settings"], ) print("saved plot as:", save_plot_to) diff --git a/umami/train_Dips.py b/umami/train_Dips.py index a12a6542..0f2b69c0 100755 --- a/umami/train_Dips.py +++ b/umami/train_Dips.py @@ -191,7 +191,7 @@ def Dips(args, train_config, preprocess_config): input_file=train_config.validation_file, var_dict=train_config.var_dict, preprocess_config=preprocess_config, - nJets=int(Val_params["nJets_val"]) + nJets=int(Val_params["n_jets"]), ) # Load the extra validation tracks if defined. @@ -201,7 +201,7 @@ def Dips(args, train_config, preprocess_config): input_file=train_config.add_validation_file, var_dict=train_config.var_dict, preprocess_config=preprocess_config, - nJets=int(Val_params["nJets_val"]) + nJets=int(Val_params["n_jets"]), ) else: @@ -286,14 +286,25 @@ def Dips(args, train_config, preprocess_config): min_lr=0.000001, ) + # Forming a dict for Callback + val_data_dict = { + "X_valid": X_valid, + "Y_valid": Y_valid, + "X_valid_add": X_valid_add, + "Y_valid_add": Y_valid_add, + } + # Set my_callback as callback. Writes history information # to json file. - my_callback = utt.MyCallback( + my_callback = utt.MyCallbackUmami( model_name=train_config.model_name, - X_valid=X_valid, - Y_valid=Y_valid, - X_valid_add=X_valid_add, - Y_valid_add=Y_valid_add, + val_data_dict=val_data_dict, + target_beff=train_config.Eval_parameters_validation["WP_b"], + charm_fraction=train_config.Eval_parameters_validation["fc_value"], + dict_file_name=utt.get_validation_dict_name( + **train_config.Eval_parameters_validation, + dir_name=train_config.model_name, + ), ) print("Start training") @@ -312,14 +323,11 @@ def Dips(args, train_config, preprocess_config): if __name__ == "__main__": args = GetParser() + + gpus = tf.config.experimental.list_physical_devices("GPU") + for gpu in gpus: + tf.config.experimental.set_memory_growth(gpu, True) + train_config = utt.Configuration(args.config_file) preprocess_config = Configuration(train_config.preprocess_config) - if args.performance_check: - utt.RunPerformanceCheck( - train_config, - compare_tagger=True, - tagger_comp_var=["rnnip_pu", "rnnip_pc", "rnnip_pb"], - comp_tagger_name="RNNIP", - ) - else: - Dips(args, train_config, preprocess_config) + Dips(args, train_config, preprocess_config) diff --git a/umami/train_tools/NN_tools.py b/umami/train_tools/NN_tools.py index 8eba3311..0a7cf428 100644 --- a/umami/train_tools/NN_tools.py +++ b/umami/train_tools/NN_tools.py @@ -18,7 +18,7 @@ from umami.tools import yaml_loader def get_validation_dict_name(WP_b, fc_value, n_jets, dir_name, plot_label=""): return os.path.join( dir_name, - f"validation_WP{str(WP_b).replace('.','p')}_fc{str(fc_value).replace('.','p')}_{n_jets}jets_Dict.json", + f"validation_WP{str(WP_b).replace('.','p')}_fc{str(fc_value).replace('.','p')}_{int(n_jets)}jets_Dict.json", ) @@ -165,6 +165,55 @@ def setup_output_directory(dir_name): outdir.mkdir() +class MyCallbackDips(Callback): + def __init__( + self, + val_data_dict=None, + log_file=None, + verbose=False, + model_name="test", + target_beff=0.77, + charm_fraction=0.018, + dict_file_name="DictFile.json", + ): + self.val_data_dict = val_data_dict + self.target_beff = target_beff + self.charm_fraction = charm_fraction + self.result = [] + self.log = open(log_file, "w") if log_file else None + self.verbose = verbose + self.model_name = model_name + setup_output_directory(self.model_name) + self.dict_list = [] + self.dict_file_name = dict_file_name + + def on_epoch_end(self, epoch, logs=None): + self.model.save(f"{self.model_name}/model_epoch{epoch}.h5") + dict_epoch = { + "epoch": epoch, + "learning_rate": logs["lr"].item(), + "loss": logs["loss"], + "acc": logs["accuracy"], + } + if self.val_data_dict: + result_dict = evaluate_model_dips( + self.model, + self.val_data_dict, + self.target_beff, + self.charm_fraction, + ) + # Once we use python >=3.9 (see https://www.python.org/dev/peps/pep-0584/#specification) switch to the following: dict_epoch |= result_dict + dict_epoch = {**dict_epoch, **result_dict} + + self.dict_list.append(dict_epoch) + with open(self.dict_file_name, "w") as outfile: + json.dump(self.dict_list, outfile, indent=4) + + def on_train_end(self, logs=None): + if self.log: + self.log.close() + + class MyCallbackUmami(Callback): def __init__( self, @@ -474,6 +523,72 @@ def evaluate_model(model, data_dict, target_beff=0.77, cfrac=0.018): return result_dict +def evaluate_model_dips(model, data_dict, target_beff=0.77, cfrac=0.018): + loss, accuracy = model.evaluate( + data_dict["X_valid_trk"], + data_dict["Y_valid"], + batch_size=5000, + use_multiprocessing=True, + workers=8, + verbose=0, + ) + + y_pred_dips = model.predict( + data_dict["X_valid_trk"], + batch_size=5000, + use_multiprocessing=True, + workers=8, + verbose=0, + ) + + c_rej, u_rej = GetRejection( + y_pred_dips, data_dict["Y_valid"], target_beff, cfrac + ) + + print("Dips:", "c-rej:", c_rej, "u-rej:", u_rej) + + ( + loss_add, + accuracy_add, + c_rej_add, + u_rej_add, + ) = (None, None, None, None) + + if data_dict["X_valid_add"] is not None: + loss_add, accuracy_add = model.evaluate( + data_dict["X_valid_trk_add"], + data_dict["Y_valid_add"], + batch_size=5000, + use_multiprocessing=True, + workers=8, + verbose=0, + ) + + y_pred_add = model.predict( + data_dict["X_valid_trk_add"], + batch_size=5000, + use_multiprocessing=True, + workers=8, + verbose=0, + ) + + c_rej_add, u_rej_add = GetRejection( + y_pred_add, data_dict["Y_valid_add"], target_beff, cfrac + ) + + result_dict = { + "val_loss": loss, + "val_acc": accuracy, + "val_loss_add": loss_add, + "val_acc_add": accuracy_add, + "c_rej": c_rej, + "u_rej": u_rej, + "c_rej_add": c_rej_add, + "u_rej_add": u_rej_add, + } + return result_dict + + def calc_validation_metrics( train_config, preprocess_config, diff --git a/umami/train_tools/Plotting.py b/umami/train_tools/Plotting.py index e2c38d86..2d255a74 100644 --- a/umami/train_tools/Plotting.py +++ b/umami/train_tools/Plotting.py @@ -84,16 +84,17 @@ def PlotRejPerEpoch( ax2.set_ylim(top=ax2.get_ylim()[1] * 1.2) makeATLAStag( - plt.gca(), - plt.gcf(), - "Internal Simulation", - ( + ax=plt.gca(), + fig=plt.gcf(), + first_tag="Internal Simulation", + second_tag=( plot_label + "\nfc={}".format(fc_value) + ", WP={:02d}%".format(int(target_beff * 100)) ), + ymax=0.9, ) - fig.legend(ncol=1, loc=(0.6, 0.1)) + fig.legend(ncol=1, loc=(0.6, 0.75)) plt.savefig(plot_name, transparent=True) plt.cla() plt.clf() @@ -118,7 +119,13 @@ def PlotLosses( df_results["val_loss_add"], label=r"validation loss - ext. $Z'$ sample", ) - makeATLAStag(plt.gca(), plt.gcf(), "Internal Simulation", plot_label) + makeATLAStag( + ax=plt.gca(), + fig=plt.gcf(), + first_tag="Internal Simulation", + second_tag=plot_label, + ymax=0.80, + ) plt.legend() ymin, ymax = plt.ylim() plt.ylim(ymin=ymin, ymax=1.1 * ymax) @@ -148,7 +155,13 @@ def PlotAccuracies( df_results["val_accuracy_add"], label=r"validation accuracy - ext. $Z'$ sample", ) - makeATLAStag(plt.gca(), plt.gcf(), "Internal Simulation", plot_label) + makeATLAStag( + ax=plt.gca(), + fig=plt.gcf(), + first_tag="Internal Simulation", + second_tag=plot_label, + ymax=0.80, + ) plt.legend() ymin, ymax = plt.ylim() plt.ylim(ymin=ymin, ymax=1.1 * ymax) @@ -250,12 +263,13 @@ def RunPerformanceCheck( compare_tagger=True, tagger_comp_var=["DL1r_pu", "DL1r_pc", "DL1r_pb"], comp_tagger_name="DL1r", + WP_b=0.77, + fc=0.018, + dict_file_name=None, ): print("Running performance check.") - WP_b = train_config.Eval_parameters_validation["WP_b"] - fc_value = train_config.Eval_parameters_validation["fc_value"] plot_label = train_config.Eval_parameters_validation["plot_label"] - + recommended_fc_values = {"DL1r": 0.018, "RNNIP": 0.08} c_rej, u_rej = None, None if compare_tagger: variables = ["HadronConeExclTruthLabelID"] @@ -267,45 +281,77 @@ def RunPerformanceCheck( df.replace({"HadronConeExclTruthLabelID": {4: 1, 5: 2}}, inplace=True) y_true = GetBinaryLabels(df["HadronConeExclTruthLabelID"].values) c_rej, u_rej = GetRejection( - df[tagger_comp_var[:]].values, y_true, WP_b, fc_value + df[tagger_comp_var[:]].values, + y_true, + WP_b, + recommended_fc_values[comp_tagger_name], ) - dictfile = f"{train_config.model_name}/DictFile.json" - df_results = pd.read_json(dictfile) + df_results = pd.read_json(dict_file_name) plot_dir = f"{train_config.model_name}/plots" print("saving plots to", plot_dir) os.makedirs(plot_dir, exist_ok=True) - plot_name = f"{plot_dir}/rej-plot_val.pdf" - PlotRejPerEpoch( - df_results, - plot_name, - c_rej, - u_rej, - labels={ - "c_rej": r"$c$-rej. - $t\bar{t}$", - "u_rej": r"light-rej. - $t\bar{t}$", - }, - comp_tagger_name=comp_tagger_name, - target_beff=WP_b, - fc_value=fc_value, - plot_label=plot_label, - ) - - if train_config.add_validation_file is not None: - plot_name = f"{plot_dir}/rej-plot_val_add.pdf" + if comp_tagger_name == "RNNIP": + plot_name = f"{plot_dir}/rej-plot_val.pdf" PlotRejPerEpoch( - df_results, - plot_name, # c_rej, u_rej, - rej_keys={"c_rej": "c_rej_add", "u_rej": "u_rej_add"}, + df_results=df_results, + plot_name=plot_name, + c_rej=c_rej, + u_rej=u_rej, labels={ - "c_rej": r"$c$-rej. - ext. $Z'$", - "u_rej": r"light-rej. - ext. $Z'$", + "c_rej": r"$c$-rej. - $t\bar{t}$", + "u_rej": r"light-rej. - $t\bar{t}$", }, + rej_keys={"c_rej": "c_rej", "u_rej": "u_rej"}, + comp_tagger_name=comp_tagger_name, target_beff=WP_b, - fc_value=fc_value, + fc_value=fc, plot_label=plot_label, ) + if train_config.add_validation_file is not None: + c_rej, u_rej = None, None + if compare_tagger: + variables = ["HadronConeExclTruthLabelID"] + variables += tagger_comp_var[:] + df = pd.DataFrame( + h5py.File(train_config.add_validation_file, "r")["/jets"][:][ + variables + ] + ) + df.query("HadronConeExclTruthLabelID <= 5", inplace=True) + df.replace( + {"HadronConeExclTruthLabelID": {4: 1, 5: 2}}, inplace=True + ) + y_true = GetBinaryLabels(df["HadronConeExclTruthLabelID"].values) + c_rej, u_rej = GetRejection( + df[tagger_comp_var[:]].values, + y_true, + WP_b, + recommended_fc_values[comp_tagger_name], + ) + + if comp_tagger_name == "RNNIP": + plot_name = f"{plot_dir}/rej-plot_val_add.pdf" + PlotRejPerEpoch( + df_results, + plot_name, + c_rej, + u_rej, + labels={ + "c_rej": r"$c$-rej. - ext. $Z'$", + "u_rej": r"light-rej. - ext. $Z'$", + }, + rej_keys={ + "c_rej": "c_rej_add", + "u_rej": "u_rej_add", + }, + comp_tagger_name=comp_tagger_name, + target_beff=WP_b, + fc_value=fc, + plot_label=plot_label, + ) + plot_name = f"{plot_dir}/loss-plot.pdf" PlotLosses(df_results, plot_name, plot_label=plot_label) @@ -470,3 +516,15 @@ def plot_validation(train_config, beff, cfrac, dict_file_name): fc=cfrac, dict_file_name=dict_file_name, ) + + +def plot_validation_dips(train_config, beff, cfrac, dict_file_name): + RunPerformanceCheck( + train_config, + compare_tagger=True, + tagger_comp_var=["rnnip_pu", "rnnip_pc", "rnnip_pb"], + comp_tagger_name="RNNIP", + WP_b=beff, + fc=cfrac, + dict_file_name=dict_file_name, + ) diff --git a/umami/train_tools/__init__.py b/umami/train_tools/__init__.py index be12f446..b678e644 100644 --- a/umami/train_tools/__init__.py +++ b/umami/train_tools/__init__.py @@ -6,10 +6,12 @@ from umami.train_tools.NN_tools import ( GetTestSample, GetTestSampleTrks, MyCallback, + MyCallbackDips, MyCallbackUmami, Sum, calc_validation_metrics, evaluate_model, + evaluate_model_dips, get_jet_feature_indicies, get_parameters_from_validation_dict_name, get_validation_dict_name, @@ -22,4 +24,5 @@ from umami.train_tools.Plotting import ( RunPerformanceCheck, RunPerformanceCheckUmami, plot_validation, + plot_validation_dips, ) -- GitLab From 42ef3374b03924d522d7b4dba1ab505af9eabc98 Mon Sep 17 00:00:00 2001 From: alfroch Date: Tue, 23 Feb 2021 09:48:29 +0000 Subject: [PATCH 2/5] Adding nicer pT and eta Plots to Preprocessing for Presentation --- umami/preprocessing.py | 13 +++++ umami/preprocessing_tools/__init__.py | 1 + umami/preprocessing_tools/utils.py | 79 +++++++++++++++++++++++++++ 3 files changed, 93 insertions(+) diff --git a/umami/preprocessing.py b/umami/preprocessing.py index 243031f4..a4bc1bd2 100644 --- a/umami/preprocessing.py +++ b/umami/preprocessing.py @@ -549,6 +549,19 @@ def RunUndersampling(args, config): plot_name=plot_name, binning={"pt_btagJes": 200, "absEta_btagJes": 20}, ) + plot_name_clean = config.GetFileName( + x + 1, + extension="", + option="downsampled-pt_eta-wider_bins", + custom_path="plots/", + ) + upt.MakePresentationPlots( + bjets=bjets, + ujets=ujets, + cjets=cjets, + plots_path=plot_name_clean, + binning={"pt_btagJes": 200, "absEta_btagJes": 20}, + ) def GetScaleDict(args, config): diff --git a/umami/preprocessing_tools/__init__.py b/umami/preprocessing_tools/__init__.py index eeb8b3ac..af2a5b03 100644 --- a/umami/preprocessing_tools/__init__.py +++ b/umami/preprocessing_tools/__init__.py @@ -21,6 +21,7 @@ from umami.preprocessing_tools.Resampling import ( from umami.preprocessing_tools.utils import ( GetBinaryLabels, MakePlots, + MakePresentationPlots, ScaleTracks, ShuffleDataFrame, ) diff --git a/umami/preprocessing_tools/utils.py b/umami/preprocessing_tools/utils.py index 679e8884..7fe173e7 100644 --- a/umami/preprocessing_tools/utils.py +++ b/umami/preprocessing_tools/utils.py @@ -6,6 +6,8 @@ import pandas as pd from dask.array.slicing import shuffle_slice from sklearn.preprocessing import LabelBinarizer +from umami.tools.PyATLASstyle.PyATLASstyle import makeATLAStag + def ShuffleDataFrame(df, seed=42, df_len=None, return_array=True): """Shuffles dask DataFrame. @@ -55,6 +57,83 @@ def GetBinaryLabels(df, column="label"): return lb.fit_transform(labels) +def MakePresentationPlots( + bjets, + ujets, + cjets, + plots_path="plots/", + binning={ + "pt_btagJes": np.linspace(10000, 2000000, 200), + "eta_btagJes": np.linspace(0, 2.5, 26), + }, + Log=True, +): + """Plots pt and eta distribution as nice plots for + presentation. + + Parameters + ---------- + bjets: array of b-jets + ujets: array of light jets + cjets: array of c-jets + + Returns + ------- + Save nice plots of pt and eta to plots_path + """ + + var_list = ["pt_btagJes", "absEta_btagJes"] + + for var in var_list: + plt.figure() + + plt.hist + + plt.hist( + [ujets[var] / 1000, cjets[var] / 1000, bjets[var] / 1000], + binning[var], + color=["#2ca02c", "#ff7f0e", "#1f77b4"], + label=["Light Jets", r"$c$-Jets", r"$b$-Jets"], + histtype="step", + stacked=False, + fill=False, + ) + + if Log is True: + plt.yscale("log") + ymin, ymax = plt.ylim() + plt.ylim(ymin=ymin, ymax=100 * ymax) + + elif Log is False: + ymin, ymax = plt.ylim() + plt.ylim(ymin=ymin, ymax=1.2 * ymax) + + if var == "pt_btagJes": + plt.xlabel(r"$p_T$ in GeV") + + elif var == "absEta_btagJes": + plt.xlabel(r"$\eta$") + + else: + plt.xlabel(var) + + plt.ylabel(r"Number of Jets") + plt.legend(loc="upper right") + + makeATLAStag( + ax=plt.gca(), + fig=plt.gcf(), + first_tag="Internal Simulation", + second_tag=r"$\sqrt{s}$ = 13 TeV, Combined $t\bar{t} + $ ext. $Z'$ PFlow Jets", + ymax=0.9, + ) + + plt.tight_layout() + plt.savefig(plots_path + "{}.pdf".format(var)) + plt.close() + plt.clf() + + def MakePlots( bjets, ujets, -- GitLab From f51719c4785690e2383f163def034db6c01add04 Mon Sep 17 00:00:00 2001 From: alfroch Date: Wed, 24 Feb 2021 12:35:19 +0000 Subject: [PATCH 3/5] Plotting updates. Only style related things in plots --- umami/evaluation_tools/PlottingFunctions.py | 9 +++++++-- umami/preprocessing_tools/utils.py | 15 ++++++++++----- umami/train_tools/Plotting.py | 12 ++++++------ 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/umami/evaluation_tools/PlottingFunctions.py b/umami/evaluation_tools/PlottingFunctions.py index f48ddefd..dff1e1f8 100644 --- a/umami/evaluation_tools/PlottingFunctions.py +++ b/umami/evaluation_tools/PlottingFunctions.py @@ -162,7 +162,11 @@ def plotROCRatio( if WorkingPoints is not None: for WP in WorkingPoints: axis_dict["left"]["top"].axvline( - x=WP, ymax=0.7, color="red", linestyle="dashed", linewidth=1.0 + x=WP, + ymax=0.65, + color="red", + linestyle="dashed", + linewidth=1.0, ) axis_dict["left"]["ratio"].axvline( @@ -172,7 +176,8 @@ def plotROCRatio( # Set the number above the line axis_dict["left"]["top"].annotate( text="{}%".format(int(WP * 100)), - xy=(WP, 0.82), + xy=(WP, 0.79), + xytext=(WP, 0.79), textcoords="offset points", xycoords=("data", "figure fraction"), ha="center", diff --git a/umami/preprocessing_tools/utils.py b/umami/preprocessing_tools/utils.py index 7fe173e7..8c70d760 100644 --- a/umami/preprocessing_tools/utils.py +++ b/umami/preprocessing_tools/utils.py @@ -1,12 +1,13 @@ import os +import matplotlib as mtp import matplotlib.pyplot as plt import numpy as np import pandas as pd from dask.array.slicing import shuffle_slice from sklearn.preprocessing import LabelBinarizer -from umami.tools.PyATLASstyle.PyATLASstyle import makeATLAStag +from umami.tools import applyATLASstyle, makeATLAStag def ShuffleDataFrame(df, seed=42, df_len=None, return_array=True): @@ -85,10 +86,9 @@ def MakePresentationPlots( var_list = ["pt_btagJes", "absEta_btagJes"] for var in var_list: + applyATLASstyle(mtp) plt.figure() - plt.hist - plt.hist( [ujets[var] / 1000, cjets[var] / 1000, bjets[var] / 1000], binning[var], @@ -102,7 +102,12 @@ def MakePresentationPlots( if Log is True: plt.yscale("log") ymin, ymax = plt.ylim() - plt.ylim(ymin=ymin, ymax=100 * ymax) + + if var == "pt_btagJes": + plt.ylim(ymin=ymin, ymax=100 * ymax) + + else: + plt.ylim(ymin=ymin, ymax=10 * ymax) elif Log is False: ymin, ymax = plt.ylim() @@ -123,7 +128,7 @@ def MakePresentationPlots( makeATLAStag( ax=plt.gca(), fig=plt.gcf(), - first_tag="Internal Simulation", + first_tag="Simulation, Work in Progress", second_tag=r"$\sqrt{s}$ = 13 TeV, Combined $t\bar{t} + $ ext. $Z'$ PFlow Jets", ymax=0.9, ) diff --git a/umami/train_tools/Plotting.py b/umami/train_tools/Plotting.py index 2d255a74..5e95073f 100644 --- a/umami/train_tools/Plotting.py +++ b/umami/train_tools/Plotting.py @@ -124,11 +124,11 @@ def PlotLosses( fig=plt.gcf(), first_tag="Internal Simulation", second_tag=plot_label, - ymax=0.80, + ymax=0.75, ) - plt.legend() + plt.legend(loc="upper right") ymin, ymax = plt.ylim() - plt.ylim(ymin=ymin, ymax=1.1 * ymax) + plt.ylim(ymin=ymin, ymax=1.2 * ymax) plt.xlabel("Epoch", fontsize=14, horizontalalignment="right", x=1.0) plt.ylabel("Loss") plt.savefig(plot_name, transparent=True) @@ -160,11 +160,11 @@ def PlotAccuracies( fig=plt.gcf(), first_tag="Internal Simulation", second_tag=plot_label, - ymax=0.80, + ymax=0.75, ) - plt.legend() + plt.legend(loc="upper right") ymin, ymax = plt.ylim() - plt.ylim(ymin=ymin, ymax=1.1 * ymax) + plt.ylim(ymin=ymin, ymax=1.2 * ymax) plt.xlabel("Epoch", fontsize=14, horizontalalignment="right", x=1.0) plt.ylabel("Accuracy") plt.savefig(plot_name, transparent=True) -- GitLab From f50eb17929d572ead8380641da46439766da6d51 Mon Sep 17 00:00:00 2001 From: alfroch Date: Wed, 24 Feb 2021 12:51:01 +0000 Subject: [PATCH 4/5] Adding the Possibility to define Working Points which are plotted in some plots. Also Adding steerable AtlasTag for Score plots --- examples/plotting_eval_config_dips.yaml | 29 +++++++----- umami/plotting_umami.py | 63 +++++++++++++++---------- 2 files changed, 55 insertions(+), 37 deletions(-) diff --git a/examples/plotting_eval_config_dips.yaml b/examples/plotting_eval_config_dips.yaml index 939d19a3..2f404272 100644 --- a/examples/plotting_eval_config_dips.yaml +++ b/examples/plotting_eval_config_dips.yaml @@ -9,9 +9,10 @@ scores_Dips_ttbar: type: "scores" data_set_name: "ttbar" # data set to use. This chooses either the test_file ('ttbar') or the add_test_file ('zpext') prediction_labels: ["dips_pb", "dips_pc", "dips_pu"] # For umami use umami_pX or dips_pX - text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$t\\bar{t}$ Test Sample" - AtlasTag: "Simulation, Work In Progress" - WorkingPoints: [0.77, 0.85] # Set Working Point Lines in plot + plot_settings: + text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$t\\bar{t}$ Validation Sample" + AtlasTag: "Simulation, Work In Progress" + WorkingPoints: [0.60, 0.70, 0.77, 0.85] # Set Working Point Lines in plot confusion_matrix_Dips_ttbar: type: "confusion_matrix" @@ -36,9 +37,9 @@ Dips_light_flavour_ttbar: ymax: 1000000 colors: ["orange", "blue"] figsize: [7, 6] # [width, hight] - text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$t\\bar{t}$ Test Sample, fc=0.018" + text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$t\\bar{t}$ Validation Sample, fc=0.018" AtlasTag: "Simulation, Work In Progress" - WorkingPoints: [0.77, 0.85] + WorkingPoints: [0.60, 0.70, 0.77, 0.85] Dips_c_flavour_ttbar: type: "ROC" @@ -58,9 +59,9 @@ Dips_c_flavour_ttbar: ymax: 1000 colors: ["orange", "blue"] figsize: [7, 6] # [width, hight] - text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$t\\bar{t}$ Test Sample, fc=0.018" + text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$t\\bar{t}$ Validation Sample, fc=0.018" AtlasTag: "Simulation, Work In Progress" - WorkingPoints: [0.77, 0.85] + WorkingPoints: [0.60, 0.70, 0.77, 0.85] # Dips, zpext @@ -68,8 +69,10 @@ scores_Dips_zpext: type: "scores" data_set_name: "zpext" # data set to use. This chooses either the test_file ('ttbar') or the add_test_file ('zpext') prediction_labels: ["dips_pb", "dips_pc", "dips_pu"] # For umami use umami_pX or dips_pX - text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$Z'$ Test Sample" - AtlasTag: "Simulation, Work In Progress" + plot_settings: + text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$t\\bar{t}$ Validation Sample" + AtlasTag: "Simulation, Work In Progress" + WorkingPoints: [0.60, 0.70, 0.77, 0.85] # Set Working Point Lines in plot confusion_matrix_Dips_zpext: type: "confusion_matrix" @@ -94,9 +97,9 @@ Dips_light_flavour_zpext: ymax: 1000 colors: ["orange", "blue"] figsize: [7, 6] # [width, hight] - text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$Z'$ Test Sample, fc=0.018" + text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$Z'$ Validation Sample, fc=0.018" AtlasTag: "Simulation, Work In Progress" - WorkingPoints: [0.77, 0.85] + WorkingPoints: [0.60, 0.70, 0.77, 0.85] Dips_c_flavour_zpext: type: "ROC" @@ -116,9 +119,9 @@ Dips_c_flavour_zpext: ymax: 100 colors: ["orange", "blue"] figsize: [7, 6] # [width, hight] - text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$Z'$ Test Sample, fc=0.018" + text: "\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$Z'$ Validation Sample, fc=0.018" AtlasTag: "Simulation, Work In Progress" - WorkingPoints: [0.77, 0.85] + WorkingPoints: [0.60, 0.70, 0.77, 0.85] Dips_saliency_b_WP77_passed_ttbar: type: "saliency" diff --git a/umami/plotting_umami.py b/umami/plotting_umami.py index a96af1a3..e5f50676 100644 --- a/umami/plotting_umami.py +++ b/umami/plotting_umami.py @@ -162,16 +162,18 @@ def plot_confusion_matrix(plot_name, plot_config, eval_params, eval_file_dir): plt.close() -def plot_score(plot_name, plot_config, eval_params, eval_file_dir): +def plot_score( + plot_name, + plot_config, + eval_params, + eval_file_dir, + AtlasTag="Internal Simulation", + text="\n$\\sqrt{s}=13$ TeV, PFlow Jets,\n$t\\bar{t}$ Test Sample", + WorkingPoints=None, +): # Get the epoch which is to be evaluated eval_epoch = int(eval_params["epoch"]) - if "AtlasTag" in list(plot_config.keys()): - AtlasTag = plot_config["AtlasTag"] - - else: - AtlasTag = "Internal Simulation" - if ("evaluation_file" not in plot_config) or ( plot_config["evaluation_file"] is None ): @@ -203,21 +205,18 @@ def plot_score(plot_name, plot_config, eval_params, eval_file_dir): stacked=False, fill=False, density=1, - label=["b-jets", "c-jets", "l-jets"], + label=[r"$b$-Jets", r"$c$-Jets", "Light Jets"], ) # Increase ymax so atlas tag don't cut plot ymin, ymax = plt.ylim() - plt.ylim(ymin=ymin, ymax=1.2 * ymax) + plt.ylim(ymin=ymin, ymax=1.3 * ymax) # Set WP vertical lines if given in config - if "WorkingPoints" in list(plot_config.keys()): - - # Get WP from config - WP_List = plot_config["WorkingPoints"] + if WorkingPoints is not None: # Iterate over WPs - for WP in WP_List: + for WP in WorkingPoints: # Calculate x value of WP line x_value = np.percentile( @@ -228,7 +227,7 @@ def plot_score(plot_name, plot_config, eval_params, eval_file_dir): plt.vlines( x=x_value, ymin=ymin, - ymax=0.8 * ymax, + ymax=WP * ymax, colors="gray", linestyles="dashed", linewidth=1.0, @@ -237,7 +236,8 @@ def plot_score(plot_name, plot_config, eval_params, eval_file_dir): # Set the number above the line ax.annotate( "{}%".format(int(WP * 100)), - xy=(x_value, 0.8 * ymax), + xy=(x_value, WP * ymax), + xytext=(x_value, WP * ymax), textcoords="offset points", ha="center", va="bottom", @@ -246,10 +246,7 @@ def plot_score(plot_name, plot_config, eval_params, eval_file_dir): plt.legend() plt.xlabel("$D_{b}$") - - text = "" - if "text" in plot_config: - text = plot_config["text"] + plt.ylabel("Normalised Number of Jets") makeATLAStag( ax=plt.gca(), @@ -284,15 +281,29 @@ def SetUpPlots(plotting_config, plot_directory, eval_file_dir, format): # Check for plot type and use the needed function if plot_config["type"] == "ROC": - plot_ROC(save_plot_to, plot_config, eval_params, eval_file_dir) + plot_ROC( + plot_name=save_plot_to, + plot_config=plot_config, + eval_params=eval_params, + eval_file_dir=eval_file_dir, + ) elif plot_config["type"] == "confusion_matrix": plot_confusion_matrix( - save_plot_to, plot_config, eval_params, eval_file_dir + plot_name=save_plot_to, + plot_config=plot_config, + eval_params=eval_params, + eval_file_dir=eval_file_dir, ) elif plot_config["type"] == "scores": - plot_score(save_plot_to, plot_config, eval_params, eval_file_dir) + plot_score( + plot_name=save_plot_to, + plot_config=plot_config, + eval_params=eval_params, + eval_file_dir=eval_file_dir, + **plot_config["plot_settings"], + ) elif plot_config["type"] == "saliency": uet.plotSaliency( @@ -302,7 +313,11 @@ def SetUpPlots(plotting_config, plot_directory, eval_file_dir, format): **plot_config["plot_settings"], ) - print("saved plot as:", save_plot_to) + print( + "saved plot as:", + save_plot_to.replace(eval_params["Path_to_models_dir"], ""), + "\n", + ) def main(args): -- GitLab From ed52526c26a1dacb4e3c02aae5b537cf42d563c3 Mon Sep 17 00:00:00 2001 From: alfroch Date: Wed, 24 Feb 2021 12:56:51 +0000 Subject: [PATCH 5/5] String Fix --- umami/preprocessing_tools/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/umami/preprocessing_tools/utils.py b/umami/preprocessing_tools/utils.py index 8c70d760..7285df0b 100644 --- a/umami/preprocessing_tools/utils.py +++ b/umami/preprocessing_tools/utils.py @@ -128,7 +128,7 @@ def MakePresentationPlots( makeATLAStag( ax=plt.gca(), fig=plt.gcf(), - first_tag="Simulation, Work in Progress", + first_tag="Internal Simulation", second_tag=r"$\sqrt{s}$ = 13 TeV, Combined $t\bar{t} + $ ext. $Z'$ PFlow Jets", ymax=0.9, ) -- GitLab