MVA Models manager

Introducing MVA Models Manager, inspired by Monitoring System ( !1483 (merged)), to simplify MVA models loading:

• MVA models are initialized inside algorithms, simplifying the model loading codes
• Only required models in a sequence would read data from .json files
• Provide an extendable framework to integrate gitlib2 in the future.

Was presented at 06 Aug WP2 meeting. Also adding documentation and examples of how new model can be created to Allen docs.

Example of usage:

#include "MultiLayerFCNN.cuh"
using NeuralNetworkType = Allen::MVAModels::MultiLayerNN;
NeuralNetworkType two_track_mva_nn {"two_track_mva_nn", "/allen_two_track_mva_model_June22.json"};

All existing MVA models are migrated to the framework in MR: SingleLayerFCNN (Matching Ghost Killer, Downstream Selector etc), MultiLayerFCNN (Two Track MVA, ElectronID NN), CatboostDesTree (MuonCatboostEvaluator).

FYI: @adeoyang, @dovombru, @mveghel

Dumpers/BinaryDumpers/src/ProvideConstants.cpp

@@ -16,12 +16,14 @@
 #include <Dumpers/IUpdater.h>
 #include "InputTools.h"
 #include "InputReader.h"
+#include "MVAModelsManager.h"
 #include "RegisterConsumers.h"
 #include "Constants.cuh"
-#include "NeuralNetworkDefinition.cuh"
 #include "BankTypes.h"
 #include "Logger.h"
 
+#include "MVAModelsManager.h"
+
 #include "AllenUpdater.h"
 
 namespace {
@@ -93,191 +95,6 @@ StatusCode ProvideConstants::initialize()
 
   m_constants.reserve_and_initialize(muon_field_of_interest_params, geometry_path);
 
-  CatboostModelReader muon_catboost_model_reader {geometry_path + "/allen_muon_catboost_model.json"};
-  m_constants.initialize_muon_catboost_model_constants(
-    muon_catboost_model_reader.n_trees(),
-    muon_catboost_model_reader.tree_depths(),
-    muon_catboost_model_reader.tree_offsets(),
-    muon_catboost_model_reader.leaf_values(),
-    muon_catboost_model_reader.leaf_offsets(),
-    muon_catboost_model_reader.split_border(),
-    muon_catboost_model_reader.split_feature());
-
-  LipschitzNNModelReader two_track_mva_model_reader {geometry_path + "/allen_two_track_mva_model_June22.json"};
-
-  m_constants.initialize_two_track_mva_model_constants(
-    two_track_mva_model_reader.weights(),
-    two_track_mva_model_reader.biases(),
-    two_track_mva_model_reader.layer_sizes(),
-    two_track_mva_model_reader.n_layers(),
-    two_track_mva_model_reader.monotone_constraints(),
-    two_track_mva_model_reader.nominal_cut(),
-    two_track_mva_model_reader.lambda());
-  LipschitzNNModelReader electronid_mva_model_reader {geometry_path + "/CaloPID/electron_mva_AllenJune2024.json"};
-
-  m_constants.initialize_electronid_mva_model_constants(
-    electronid_mva_model_reader.weights(),
-    electronid_mva_model_reader.biases(),
-    electronid_mva_model_reader.layer_sizes(),
-    electronid_mva_model_reader.n_layers(),
-    electronid_mva_model_reader.monotone_constraints(),
-    electronid_mva_model_reader.min_rescales(),
-    electronid_mva_model_reader.max_rescales(),
-    electronid_mva_model_reader.nominal_cut(),
-    electronid_mva_model_reader.lambda());
-  LipschitzNNModelReader muonid_mva_model_reader {geometry_path + "/muonid_mva_AllenJune2024.json"};
-
-  m_constants.initialize_muonid_mva_model_constants(
-    muonid_mva_model_reader.weights(),
-    muonid_mva_model_reader.biases(),
-    muonid_mva_model_reader.layer_sizes(),
-    muonid_mva_model_reader.n_layers(),
-    muonid_mva_model_reader.monotone_constraints(),
-    muonid_mva_model_reader.min_rescales(),
-    muonid_mva_model_reader.max_rescales(),
-    muonid_mva_model_reader.nominal_cut(),
-    muonid_mva_model_reader.lambda());
-
-  // Ghost killers
-  SingleLayerFCNNReader forward_ghostkiller_reader {geometry_path +
-                                                    "/GhostProbability/Hlt1_LongGhostKiller_Forward.json"};
-  m_constants.initialize_forward_ghostkiller_constants(
-    forward_ghostkiller_reader.mean(),
-    forward_ghostkiller_reader.std(),
-    forward_ghostkiller_reader.weights1(),
-    forward_ghostkiller_reader.bias1(),
-    forward_ghostkiller_reader.weights2(),
-    forward_ghostkiller_reader.bias2());
-  SingleLayerFCNNReader forward_no_ut_ghostkiller_reader {geometry_path +
-                                                          "/GhostProbability/Hlt1_LongGhostKiller_noUT_Forward.json"};
-  m_constants.initialize_forward_no_ut_ghostkiller_constants(
-    forward_no_ut_ghostkiller_reader.mean(),
-    forward_no_ut_ghostkiller_reader.std(),
-    forward_no_ut_ghostkiller_reader.weights1(),
-    forward_no_ut_ghostkiller_reader.bias1(),
-    forward_no_ut_ghostkiller_reader.weights2(),
-    forward_no_ut_ghostkiller_reader.bias2());
-  SingleLayerFCNNReader matching_ghostkiller_reader {geometry_path +
-                                                     "/GhostProbability/Hlt1_LongGhostKiller_Matching.json"};
-  m_constants.initialize_matching_ghostkiller_constants(
-    matching_ghostkiller_reader.mean(),
-    matching_ghostkiller_reader.std(),
-    matching_ghostkiller_reader.weights1(),
-    matching_ghostkiller_reader.bias1(),
-    matching_ghostkiller_reader.weights2(),
-    matching_ghostkiller_reader.bias2());
-
-  SingleLayerFCNNReader downstream_ghostkiller_reader {geometry_path +
-                                                       "/GhostProbability/Hlt1_DownstreamGhostKiller.json"};
-  m_constants.initialize_downstream_ghostkiller_constants(
-    downstream_ghostkiller_reader.mean(),
-    downstream_ghostkiller_reader.std(),
-    downstream_ghostkiller_reader.weights1(),
-    downstream_ghostkiller_reader.bias1(),
-    downstream_ghostkiller_reader.weights2(),
-    downstream_ghostkiller_reader.bias2());
-
-  SingleLayerFCNNReader matching_with_ut_ghostkiller_reader {
-    geometry_path + "/GhostProbability/Hlt1_LongGhostKiller_MatchingWithUT.json"};
-  m_constants.initialize_matching_with_ut_ghostkiller_constants(
-    matching_with_ut_ghostkiller_reader.mean(),
-    matching_with_ut_ghostkiller_reader.std(),
-    matching_with_ut_ghostkiller_reader.weights1(),
-    matching_with_ut_ghostkiller_reader.bias1(),
-    matching_with_ut_ghostkiller_reader.weights2(),
-    matching_with_ut_ghostkiller_reader.bias2());
-  SingleLayerFCNNReader matching_no_ut_v2_ghostkiller_reader {
-    geometry_path + "/GhostProbability/Hlt1_LongGhostKiller_MatchingNoUT_V2.json"};
-
-  m_constants.initialize_matching_no_ut_v2_ghostkiller_constants(
-    matching_no_ut_v2_ghostkiller_reader.mean(),
-    matching_no_ut_v2_ghostkiller_reader.std(),
-    matching_no_ut_v2_ghostkiller_reader.weights1(),
-    matching_no_ut_v2_ghostkiller_reader.bias1(),
-    matching_no_ut_v2_ghostkiller_reader.weights2(),
-    matching_no_ut_v2_ghostkiller_reader.bias2());
-
-  SingleLayerFCNNReader matching_with_ut_v2_ghostkiller_reader {
-    geometry_path + "/GhostProbability/Hlt1_LongGhostKiller_MatchingWithUT_V2.json"};
-  m_constants.initialize_matching_with_ut_v2_ghostkiller_constants(
-    matching_with_ut_v2_ghostkiller_reader.mean(),
-    matching_with_ut_v2_ghostkiller_reader.std(),
-    matching_with_ut_v2_ghostkiller_reader.weights1(),
-    matching_with_ut_v2_ghostkiller_reader.bias1(),
-    matching_with_ut_v2_ghostkiller_reader.weights2(),
-    matching_with_ut_v2_ghostkiller_reader.bias2());
-  // std::unique_ptr<SingleLayerFCNNReader> forward_no_ut_ghostkiller_reader, matching_no_ut_ghostkiller_reader,
-  // forward_ghostkiller_reader, matching_ghostkiller_reader;
-
-  SingleLayerFCNNReader ttrack_selector_reader {geometry_path + "/HLT1Downstream/Hlt1_Downstream_TTrackSelector.json"};
-  m_constants.initialize_ttrack_selector_constants(
-    ttrack_selector_reader.mean(),
-    ttrack_selector_reader.std(),
-    ttrack_selector_reader.weights1(),
-    ttrack_selector_reader.bias1(),
-    ttrack_selector_reader.weights2(),
-    ttrack_selector_reader.bias2());
-
-  SingleLayerFCNNReader downstream_composite_quality_reader {geometry_path +
-                                                             "/HLT1Downstream/Hlt1_Downstream_Composite_Quality.json"};
-  m_constants.initialize_downstream_composite_quality_evaluator_constants(
-    downstream_composite_quality_reader.mean(),
-    downstream_composite_quality_reader.std(),
-    downstream_composite_quality_reader.weights1(),
-    downstream_composite_quality_reader.bias1(),
-    downstream_composite_quality_reader.weights2(),
-    downstream_composite_quality_reader.bias2());
-
-  SingleLayerFCNNReader downstream_lambda_selector_reader {geometry_path +
-                                                           "/HLT1Downstream/Hlt1_Downstream_LambdaSelector.json"};
-  m_constants.initialize_downstream_lambda_selector_constants(
-    downstream_lambda_selector_reader.mean(),
-    downstream_lambda_selector_reader.std(),
-    downstream_lambda_selector_reader.weights1(),
-    downstream_lambda_selector_reader.bias1(),
-    downstream_lambda_selector_reader.weights2(),
-    downstream_lambda_selector_reader.bias2());
-
-  SingleLayerFCNNReader downstream_kshort_selector_reader {geometry_path +
-                                                           "/HLT1Downstream/Hlt1_Downstream_KshortSelector.json"};
-  m_constants.initialize_downstream_kshort_selector_constants(
-    downstream_kshort_selector_reader.mean(),
-    downstream_kshort_selector_reader.std(),
-    downstream_kshort_selector_reader.weights1(),
-    downstream_kshort_selector_reader.bias1(),
-    downstream_kshort_selector_reader.weights2(),
-    downstream_kshort_selector_reader.bias2());
-
-  SingleLayerFCNNReader downstream_detached_lambda_selector_reader {
-    geometry_path + "/HLT1Downstream/Hlt1_Downstream_DetachedLambdaSelector.json"};
-  m_constants.initialize_downstream_detached_lambda_selector_constants(
-    downstream_detached_lambda_selector_reader.mean(),
-    downstream_detached_lambda_selector_reader.std(),
-    downstream_detached_lambda_selector_reader.weights1(),
-    downstream_detached_lambda_selector_reader.bias1(),
-    downstream_detached_lambda_selector_reader.weights2(),
-    downstream_detached_lambda_selector_reader.bias2());
-
-  SingleLayerFCNNReader downstream_detached_kshort_selector_reader {
-    geometry_path + "/HLT1Downstream/Hlt1_Downstream_DetachedKshortSelector.json"};
-  m_constants.initialize_downstream_detached_kshort_selector_constants(
-    downstream_detached_kshort_selector_reader.mean(),
-    downstream_detached_kshort_selector_reader.std(),
-    downstream_detached_kshort_selector_reader.weights1(),
-    downstream_detached_kshort_selector_reader.bias1(),
-    downstream_detached_kshort_selector_reader.weights2(),
-    downstream_detached_kshort_selector_reader.bias2());
-
-  SingleLayerFCNNReader downstream_busca_selector_reader {geometry_path +
-                                                          "/HLT1Downstream/Hlt1_Downstream_BuScaSelector.json"};
-  m_constants.initialize_downstream_busca_selector_constants(
-    downstream_busca_selector_reader.mean(),
-    downstream_busca_selector_reader.std(),
-    downstream_busca_selector_reader.weights1(),
-    downstream_busca_selector_reader.bias1(),
-    downstream_busca_selector_reader.weights2(),
-    downstream_busca_selector_reader.bias2());
-
   // Allen Consumers
   std::unordered_set<BankTypes> subdetectors;
   for (unsigned bt = 0; bt < NBankTypes; ++bt) {