Skip to content
Snippets Groups Projects
Commit dac09c26 authored by Rosen Matev's avatar Rosen Matev :sunny:
Browse files

Merge branch 'fix-missing-inline' into 'master' 

fix duplicate symbols in SigmaNet

See merge request !2808
parents e91d5196 66a790bc
No related branches found
No related tags found
1 merge request!2808fix duplicate symbols in SigmaNet
Pipeline #3771920 passed
Stage: test
    Hide whitespace changes
    Inline Side-by-side
    Phys/SelTools/include/SelTools/SigmaNet.h
    + 139
    − 207
    View file @ dac09c26
    ......@@ -94,10 +94,27 @@ namespace Sel {
    using input_type = float;
    /////////////////////////////////// SigmaNet //////////////////////////////////////////////////////////
    //
    // Implementation of a monotonic Lipschitz neural net
    //
    ///////////////////////////////////////////////////////////////////////////////////////////////////////
    SigmaNet( MVA_config_dict config ) : m_config( std::move( config ) ) {}
    auto operator()( LHCb::span<input_type const> values ) const {
    if ( values.size() != m_input_size ) { throw exception( "'InputSize' does not agree with provided Input!!!" ); }
    return evaluate<m_size>( values, m_weights, m_biases, m_layer_sizes, m_monotone_constraints, m_lambda );
    }
    /** Return the list of expected inputs. */
    std::vector<std::string> const& variables() const { return m_variables; }
    void bind( Gaudi::Algorithm* alg ) {
    readConfig( *alg ); // Retrieve the configuration
    readWeightsFile( *alg ); // Read the .pkl File with the weights
    if ( alg->msgLevel( MSG::VERBOSE ) ) {
    alg->verbose() << "Variables found in " << m_weightfile << ": ";
    for ( auto& var : m_variables ) { alg->verbose() << "'" << var << "' "; }
    alg->verbose() << endmsg;
    }
    }
    private:
    MVA_config_dict m_config;
    ......@@ -116,233 +133,148 @@ namespace Sel {
    static constexpr long unsigned int m_size =
    128; // maximum width of the network. 128 is enough as the network can alternatively be made deeper.
    public:
    SigmaNet( MVA_config_dict config ) : m_config( std::move( config ) ) {}
    auto operator()( LHCb::span<input_type const> values ) const;
    /** Return the list of expected inputs. */
    std::vector<std::string> const& variables() const { return m_variables; }
    private:
    // Helper functions
    void bind( Gaudi::Algorithm* alg );
    // Gaudi exception handler
    GaudiException exception( std::string s ) const {
    return GaudiException{std::move( s ), "Sel::SigmaNet", StatusCode::FAILURE};
    }
    private:
    // Helper functions
    void readConfig( Gaudi::Algorithm const& alg );
    void readWeightsFile( Gaudi::Algorithm const& alg );
    bool checkWeightsFile();
    void setupReader();
    };
    inline auto SigmaNet::operator()( LHCb::span<input_type const> values ) const {
    if ( values.size() != m_input_size ) { throw exception( "'InputSize' does not agree with provided Input!!!" ); }
    // Evaluate MVA
    return evaluate<m_size>( values, m_weights, m_biases, m_layer_sizes, m_monotone_constraints, m_lambda );
    }
    inline void SigmaNet::bind( Gaudi::Algorithm* alg ) {
    readConfig( *alg ); // Retrieve the configuration
    readWeightsFile( *alg ); // Read the .pkl File with the weights
    if ( alg->msgLevel( MSG::VERBOSE ) ) {
    alg->verbose() << "Variables found in " << m_weightfile << ": ";
    for ( auto& var : m_variables ) { alg->verbose() << "'" << var << "' "; }
    alg->verbose() << endmsg;
    }
    }
    // Retrieve the configuration
    inline void SigmaNet::readConfig( Gaudi::Algorithm const& alg ) {
    if ( alg.msgLevel( MSG::VERBOSE ) ) { alg.verbose() << "Start reading config..." << endmsg; }
    constexpr auto mandatory_keys =
    std::array{"Name", "File", "Monotone_Constraints", "InputSize", "NLayers", "Lambda"};
    for ( auto const& key : mandatory_keys ) {
    if ( m_config.find( key ) == m_config.end() ) throw exception( "'" + std::string{key} + " not provided" );
    }
    if ( alg.msgLevel( MSG::VERBOSE ) ) {
    alg.verbose() << "Finished checking config input. Everythings alright..." << endmsg;
    }
    //////////////////////////////////// Read Input size //////////////////////////////////////////////////////////////
    //
    //
    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    m_input_size = std::stoi( get_mva_config<std::string>( m_config, "InputSize" ) );
    if ( alg.msgLevel( MSG::VERBOSE ) ) { alg.verbose() << "input_size: " << m_input_size << endmsg; }
    //////////////////////////////////// Read Name i/////////////7////////////////////////////////////////////////////
    //
    //
    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    m_name = get_mva_config<std::string>( m_config, "Name" );
    if ( alg.msgLevel( MSG::VERBOSE ) ) { alg.verbose() << "m_name: " << m_name << endmsg; }
    //////////////////////////////////// Read monotone constraints ////////////////////////////////////////////////////
    //
    //
    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    std::string monotone_constraints_string = get_mva_config<std::string>( m_config, "Monotone_Constraints" );
    if ( alg.msgLevel( MSG::VERBOSE ) ) {
    alg.verbose() << "monotone constraints: " << monotone_constraints_string << endmsg;
    }
    // Retrieve the configuration
    void readConfig( Gaudi::Algorithm const& alg ) {
    std::string delimiter = ",";
    if ( alg.msgLevel( MSG::VERBOSE ) ) { alg.verbose() << "Start reading config..." << endmsg; }
    std::vector<std::string> temp_str_monotone_constraints;
    boost::remove_erase_if( monotone_constraints_string, boost::is_any_of( "[] " ) );
    boost::split( temp_str_monotone_constraints, monotone_constraints_string, boost::is_any_of( delimiter ) );
    for ( const auto& str : temp_str_monotone_constraints ) {
    int int_mon_constraint = std::stoi( str );
    if ( int_mon_constraint != 1 && int_mon_constraint != 0 && int_mon_constraint != -1 ) {
    throw exception( "Monotone constraints can only be 1,0,-1." );
    constexpr auto mandatory_keys =
    std::array{"Name", "File", "Monotone_Constraints", "InputSize", "NLayers", "Lambda"};
    for ( auto const& key : mandatory_keys ) {
    if ( m_config.find( key ) == m_config.end() ) throw exception( "'" + std::string{key} + " not provided" );
    }
    m_monotone_constraints.push_back( int_mon_constraint );
    }
    ///////////////////////////////////////////////// Read lambda
    //
    //
    /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    m_lambda = std::stof( get_mva_config<std::string>( m_config, "Lambda" ) );
    if ( alg.msgLevel( MSG::VERBOSE ) ) { alg.verbose() << "labmda: " << m_lambda << endmsg; }
    //////////////////////////////////////////////// Read number of layers
    //
    //
    //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    m_n_layers = std::stoi( get_mva_config<std::string>( m_config, "NLayers" ) );
    assert( m_n_layers > 0 );
    if ( alg.msgLevel( MSG::VERBOSE ) ) { alg.verbose() << "nlayers: " << m_n_layers << endmsg; }
    ///////////////////////////////////////////////// Read input vars
    //
    //
    /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    std::string vars_string = get_mva_config<std::string>( m_config, "Variables" );
    boost::remove_erase_if( vars_string, boost::is_any_of( "[] " ) );
    m_variables.clear();
    boost::split( m_variables, vars_string, boost::is_any_of( delimiter ) );
    ///////////////////////////////////////////////// Read weight file
    //
    //
    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    std::string unresolved_weightfile = get_mva_config<std::string>( m_config, "File" );
    if ( alg.msgLevel( MSG::VERBOSE ) ) { alg.verbose() << "weightfile: " << unresolved_weightfile << endmsg; }
    System::resolveEnv( unresolved_weightfile, m_weightfile ).ignore(); // LEON
    }
    inline void SigmaNet::readWeightsFile( Gaudi::Algorithm const& alg ) {
    // Check that the weightfile exists
    if ( !checkWeightsFile() ) { throw exception( "Couldn't open file: " + m_weightfile ); }
    std::ifstream fin( m_weightfile );
    unsigned int num_layer = 0;
    unsigned int total_number_of_layers = 1;
    unsigned int weights_counter = 0;
    unsigned int bias_counter = 0;
    auto weight_sizes = std::vector<int>( m_n_layers, 0 );
    auto bias_sizes = std::vector<int>( m_n_layers, 0 );
    m_layer_sizes = std::vector<long unsigned int>( m_n_layers + 1, 0 );
    if ( alg.msgLevel( MSG::VERBOSE ) ) {
    alg.verbose() << "Finished checking config input. Everythings alright..." << endmsg;
    }
    auto jf = nlohmann::json::parse( fin );
    //////////////////////////////////// Read Input size
    m_input_size = std::stoi( get_mva_config<std::string>( m_config, "InputSize" ) );
    if ( alg.msgLevel( MSG::VERBOSE ) ) { alg.verbose() << "input_size: " << m_input_size << endmsg; }
    for ( auto it = jf.begin(); it != jf.end(); ++it ) {
    const std::string& current_key = it.key();
    //////////////////////////////////// Read Name
    m_name = get_mva_config<std::string>( m_config, "Name" );
    if ( alg.msgLevel( MSG::VERBOSE ) ) { alg.verbose() << "m_name: " << m_name << endmsg; }
    if ( current_key.find( "sigmanet.sigma" ) != std::string::npos ) {
    if ( jf.at( current_key ).at( 0 ) != m_lambda ) {
    throw exception( "Specified lambda does not match your model" );
    //////////////////////////////////// Read monotone constraints
    std::string monotone_constraints_string = get_mva_config<std::string>( m_config, "Monotone_Constraints" );
    if ( alg.msgLevel( MSG::VERBOSE ) ) {
    alg.verbose() << "monotone constraints: " << monotone_constraints_string << endmsg;
    }
    std::string delimiter = ",";
    std::vector<std::string> temp_str_monotone_constraints;
    boost::remove_erase_if( monotone_constraints_string, boost::is_any_of( "[] " ) );
    boost::split( temp_str_monotone_constraints, monotone_constraints_string, boost::is_any_of( delimiter ) );
    for ( const auto& str : temp_str_monotone_constraints ) {
    int int_mon_constraint = std::stoi( str );
    if ( int_mon_constraint != 1 && int_mon_constraint != 0 && int_mon_constraint != -1 ) {
    throw exception( "Monotone constraints can only be 1,0,-1." );
    }
    } else if ( current_key.find( "weight" ) != std::string::npos ) {
    m_monotone_constraints.push_back( int_mon_constraint );
    }
    ///////////////////////////////////////////////// Read lambda
    m_lambda = std::stof( get_mva_config<std::string>( m_config, "Lambda" ) );
    if ( alg.msgLevel( MSG::VERBOSE ) ) { alg.verbose() << "labmda: " << m_lambda << endmsg; }
    //////////////////////////////////////////////// Read number of layers
    m_n_layers = std::stoi( get_mva_config<std::string>( m_config, "NLayers" ) );
    assert( m_n_layers > 0 );
    if ( alg.msgLevel( MSG::VERBOSE ) ) { alg.verbose() << "nlayers: " << m_n_layers << endmsg; }
    ///////////////////////////////////////////////// Read input vars
    std::string vars_string = get_mva_config<std::string>( m_config, "Variables" );
    boost::remove_erase_if( vars_string, boost::is_any_of( "[] " ) );
    m_variables.clear();
    boost::split( m_variables, vars_string, boost::is_any_of( delimiter ) );
    ///////////////////////////////////////////////// Read weight file
    std::string unresolved_weightfile = get_mva_config<std::string>( m_config, "File" );
    if ( alg.msgLevel( MSG::VERBOSE ) ) { alg.verbose() << "weightfile: " << unresolved_weightfile << endmsg; }
    System::resolveEnv( unresolved_weightfile, m_weightfile ).ignore(); // LEON
    }
    for ( const auto& wl : jf.at( current_key ) ) {
    for ( const auto& w : wl ) {
    m_weights.push_back( w );
    weights_counter += 1;
    weight_sizes[num_layer] += 1;
    void readWeightsFile( Gaudi::Algorithm const& alg ) {
    // Check that the weightfile exists
    std::ifstream fin( m_weightfile ); // TODO: use ParamFileSvc instead of direct use of fstream
    if ( !fin ) { throw exception( "Couldn't open file: " + m_weightfile ); }
    unsigned int num_layer = 0;
    unsigned int total_number_of_layers = 1;
    unsigned int weights_counter = 0;
    unsigned int bias_counter = 0;
    auto weight_sizes = std::vector<int>( m_n_layers, 0 );
    auto bias_sizes = std::vector<int>( m_n_layers, 0 );
    m_layer_sizes = std::vector<long unsigned int>( m_n_layers + 1, 0 );
    auto jf = nlohmann::json::parse( fin );
    for ( const auto& [key, value] : jf.items() ) {
    if ( key.find( "sigmanet.sigma" ) != std::string::npos ) {
    if ( value.at( 0 ) != m_lambda ) { throw exception( "Specified lambda does not match your model" ); }
    } else if ( key.find( "weight" ) != std::string::npos ) {
    for ( const auto& wl : value ) {
    std::copy( wl.begin(), wl.end(), std::back_inserter( m_weights ) );
    weights_counter += wl.size();
    weight_sizes[num_layer] += wl.size();
    }
    num_layer += 1;
    if ( num_layer + 1 > total_number_of_layers ) { total_number_of_layers += 1; }
    } else if ( key.find( "bias" ) != std::string::npos ) {
    std::copy( value.begin(), value.end(), std::back_inserter( m_biases ) );
    bias_counter += value.size();
    bias_sizes[num_layer] += value.size();
    } else if ( key.find( "gamma" ) != std::string::npos ) {
    continue;
    } else {
    throw exception( "Error when reading json file" );
    }
    }
    num_layer += 1;
    if ( num_layer + 1 > total_number_of_layers ) { total_number_of_layers += 1; }
    } else if ( current_key.find( "bias" ) != std::string::npos ) {
    for ( const auto& b : jf.at( current_key ) ) {
    m_biases.push_back( b );
    bias_counter += 1;
    bias_sizes[num_layer] += 1;
    }
    } else if ( current_key.find( "gamma" ) != std::string::npos ) {
    continue;
    } else {
    throw exception( "Error when reading json file" );
    long unsigned int max_layer_size = *std::max_element( m_layer_sizes.begin(), m_layer_sizes.end() );
    if ( max_layer_size > m_size ) {
    throw exception( "Maximum layer size is larger than the maximum allowed size" );
    }
    }
    long unsigned int max_layer_size = *std::max_element( m_layer_sizes.begin(), m_layer_sizes.end() );
    if ( max_layer_size > m_size ) { throw exception( "Maximum layer size is larger than the maximum allowed size" ); }
    std::transform( weight_sizes.begin(), weight_sizes.end(), bias_sizes.begin(), m_layer_sizes.begin(),
    []( auto ws, auto bs ) { return ws / bs; } );
    m_layer_sizes.at( weight_sizes.size() ) = 1;
    if ( alg.msgLevel( MSG::VERBOSE ) ) {
    alg.verbose() << "Neural network parameters found:" << endmsg;
    alg.verbose() << "Lambda = " << m_lambda << endmsg;
    alg.verbose() << "NLayers = " << m_n_layers << endmsg;
    alg.verbose() << "LayerSizes = [";
    for ( int i = 0; i < m_n_layers + 1; i++ ) { alg.verbose() << m_layer_sizes.at( i ) << ","; }
    alg.verbose() << "]" << endmsg;
    alg.verbose() << "Monotone_Constraints = [";
    for ( unsigned int i = 0; i < m_input_size; i++ ) { alg.verbose() << m_monotone_constraints.at( i ) << ","; }
    alg.verbose() << "]" << endmsg;
    }
    std::transform( weight_sizes.begin(), weight_sizes.end(), bias_sizes.begin(), m_layer_sizes.begin(),
    []( auto ws, auto bs ) { return ws / bs; } );
    m_layer_sizes.at( weight_sizes.size() ) = 1;
    if ( alg.msgLevel( MSG::VERBOSE ) ) {
    alg.verbose() << "Neural network parameters found:" << endmsg;
    alg.verbose() << "Lambda = " << m_lambda << endmsg;
    alg.verbose() << "NLayers = " << m_n_layers << endmsg;
    alg.verbose() << "LayerSizes = [";
    for ( int i = 0; i < m_n_layers + 1; i++ ) { alg.verbose() << m_layer_sizes.at( i ) << ","; }
    alg.verbose() << "]" << endmsg;
    alg.verbose() << "Monotone_Constraints = [";
    for ( unsigned int i = 0; i < m_input_size; i++ ) { alg.verbose() << m_monotone_constraints.at( i ) << ","; }
    alg.verbose() << "]" << endmsg;
    }
    unsigned int total_parameters = 0;
    for ( int layer_idx = 1; layer_idx < m_n_layers + 1; ++layer_idx ) {
    long unsigned int n_inputs = m_layer_sizes[layer_idx - 1];
    long unsigned int n_outputs = m_layer_sizes[layer_idx];
    total_parameters += n_outputs * n_inputs;
    total_parameters += n_outputs;
    }
    unsigned int total_parameters = 0;
    for ( int layer_idx = 1; layer_idx < m_n_layers + 1; ++layer_idx ) {
    long unsigned int n_inputs = m_layer_sizes[layer_idx - 1];
    long unsigned int n_outputs = m_layer_sizes[layer_idx];
    total_parameters += n_outputs * n_inputs;
    total_parameters += n_outputs;
    }
    if ( total_parameters != ( weights_counter + bias_counter ) ) {
    throw exception( std::string( "Missmatch between weight_file( " ) +
    std::to_string( weights_counter + bias_counter ) +
    std::string( " Parameters ) and specified architecture ( " ) +
    std::to_string( total_parameters ) + std::string( " Parameters ) !!!" ) );
    if ( total_parameters != ( weights_counter + bias_counter ) ) {
    throw exception( "Mismatch between weight_file( " + std::to_string( weights_counter + bias_counter ) +
    " Parameters ) and specified architecture ( " + std::to_string( total_parameters ) +
    " Parameters ) !!!" );
    }
    }
    }
    inline bool SigmaNet::checkWeightsFile() {
    // Check existence of WeightFile: locally
    return std::ifstream{m_weightfile}.good();
    }
    };
    } // namespace Sel
    0% or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment