Commit 3d6510a5 authored by Luciano Orsini's avatar Luciano Orsini Committed by Dainius Simelevicius
Browse files

references #73: support for configurable sgid_index in XML </Endpoint>

parent b15b669b
......@@ -36,7 +36,7 @@ namespace ibvla
std::string name_;
AcceptorListener * listener_;
Acceptor (const std::string & name, ibvla::AcceptorListener * listener, size_t mtu, ibvla::CompletionQueue cqr, toolbox::mem::Pool * rpool, size_t ibPort, size_t ibPath);
Acceptor (const std::string & name, ibvla::AcceptorListener * listener, size_t mtu, ibvla::CompletionQueue cqr, toolbox::mem::Pool * rpool, size_t ibPort, size_t ibPath, int sgid_index);
void listen (const std::string & hostname, const std::string & port) ;
void accept (ConnectionRequest &id, QueuePair &qp) ;
......@@ -55,6 +55,7 @@ namespace ibvla
toolbox::mem::Pool * rpool_;
size_t ibPort_;
size_t ibPath_;
int sgid_index_;
private:
};
}
......
......@@ -23,12 +23,13 @@ namespace ibvla
class Connector
{
public:
Connector (size_t mtu, bool sendWithTimeout);
Connector (size_t mtu, bool sendWithTimeout, int sgid_index);
void connect (ibvla::QueuePair &qp, const std::string & destname, const std::string & destport, size_t path) ;
void connect (ibvla::QueuePair &qp, const std::string & destname, const std::string & destport, size_t path);
ibv_mtu mtu_;
bool sendWithTimeout_;
int sgid_index_;
};
}
......
......@@ -34,8 +34,8 @@
#include "toolbox/task/WorkLoopFactory.h"
ibvla::Acceptor::Acceptor (const std::string & name, ibvla::AcceptorListener * listener, size_t mtu, ibvla::CompletionQueue cqr, toolbox::mem::Pool * rpool, size_t ibPort,size_t ibPath)
: name_(name), listener_(listener), cqr_(cqr), rpool_(rpool), ibPort_(ibPort), ibPath_(ibPath)
ibvla::Acceptor::Acceptor (const std::string & name, ibvla::AcceptorListener * listener, size_t mtu, ibvla::CompletionQueue cqr, toolbox::mem::Pool * rpool, size_t ibPort, size_t ibPath, int sgid_index)
: name_(name), listener_(listener), cqr_(cqr), rpool_(rpool), ibPort_(ibPort), ibPath_(ibPath), sgid_index_(sgid_index)
{
listenfd_ = -1;
......@@ -252,17 +252,13 @@ void ibvla::Acceptor::accept (ibvla::ConnectionRequest &id, ibvla::QueuePair &qp
attr.max_dest_rd_atomic = 1;
attr.min_rnr_timer = 0;
/*
* FIXME: RoCE: gid_index is either configurable or we have to find the best of all possible gid indexes (based on e.g. RoCE version supported)
* Number 3 would work only for d3vrubu(s)
*/
int gid_index = 3;
std::cout << "DBG: Acceptor using GID index: " << gid_index << '\n';
// RoCE uses sgid_index_
std::cout << "DBG: Acceptor using GID index: " << sgid_index_ << '\n';
// For RoCE:
attr.ah_attr.is_global = 1;
attr.ah_attr.grh.dgid = id.remote_gid_;
attr.ah_attr.grh.sgid_index = gid_index;
attr.ah_attr.grh.sgid_index = sgid_index_;
attr.ah_attr.grh.hop_limit = 2;
attr.ah_attr.grh.traffic_class = 0;
attr.ah_attr.sl = 0;
......@@ -316,7 +312,7 @@ void ibvla::Acceptor::accept (ibvla::ConnectionRequest &id, ibvla::QueuePair &qp
// Communicate information back to remote to complete connection
// get port attributes
struct ibv_port_attr p_att = qp.pd_.context_.queryPort(qp_attr.port_num);
union ibv_gid gid = qp.pd_.context_.queryGID(qp_attr.port_num, gid_index);
union ibv_gid gid = qp.pd_.context_.queryGID(qp_attr.port_num, sgid_index_);
char msg[sizeof "0000:000000:000000:0000000000000000:0000000000000000"];
sprintf(msg, "%04x:%06x:%06x:%016lx:%016lx", (p_att.lid + (unsigned int)ibPath_), qp.getNum(), attr.sq_psn, gid.global.subnet_prefix, gid.global.interface_id);
......
......@@ -33,7 +33,7 @@
#include <string.h>
#include <sstream>
ibvla::Connector::Connector (size_t mtu, bool sendWithTimeout)
ibvla::Connector::Connector (size_t mtu, bool sendWithTimeout, int sgid_index): sgid_index_(sgid_index)
{
srand48(time(NULL)); // seed the random generator for QP packet number initialization
......@@ -44,7 +44,7 @@ ibvla::Connector::Connector (size_t mtu, bool sendWithTimeout)
/*
* Blocking connect, once complete the given QP will be in a connected ReadyToSend state
*/
void ibvla::Connector::connect (ibvla::QueuePair &qp, const std::string & destname, const std::string & destport, size_t path)
void ibvla::Connector::connect (ibvla::QueuePair &qp, const std::string & destname, const std::string & destport, size_t path)
{
struct addrinfo *res, *t;
struct addrinfo hints;
......@@ -117,13 +117,9 @@ void ibvla::Connector::connect (ibvla::QueuePair &qp, const std::string & destna
struct ibv_port_attr p_att = qp.pd_.context_.queryPort(qp_attr.port_num);
/*
* FIXME: RoCE: gid_index is either configurable or we have to find the best of all possible gid indexes (based on e.g. RoCE version supported)
* Number 3 would work only for d3vrubu(s)
*/
int gid_index = 3;
union ibv_gid gid = qp.pd_.context_.queryGID(qp_attr.port_num, gid_index);
std::cout << "DBG: Connector using GID index: " << gid_index << '\n';
// RoCE uses sgid_index_
union ibv_gid gid = qp.pd_.context_.queryGID(qp_attr.port_num, sgid_index_);
std::cout << "DBG: Connector using GID index: " << sgid_index_ << '\n';
sprintf(msg, "%04x:%06x:%06x:%016lx:%016lx", (p_att.lid + (unsigned int)path), qp.getNum(), psn, gid.global.subnet_prefix, gid.global.interface_id);
//sprintf(msg, "%04x:%06x:%06x", (p_att.lid + (unsigned int)path), qp.getNum(), psn);
......@@ -187,7 +183,7 @@ void ibvla::Connector::connect (ibvla::QueuePair &qp, const std::string & destna
// For RoCE:
attr.ah_attr.is_global = 1;
attr.ah_attr.grh.dgid = r_gid;
attr.ah_attr.grh.sgid_index = gid_index;
attr.ah_attr.grh.sgid_index = sgid_index_;
attr.ah_attr.grh.hop_limit = 2;
attr.ah_attr.grh.traffic_class = 0;
attr.ah_attr.sl = 0;
......
......@@ -25,7 +25,7 @@ namespace pt
public:
//! Create address from url
Address (const std::string & url, const std::string& service, size_t ibPort, size_t ibPath);
Address (const std::string & url, const std::string& service, size_t ibPort, size_t ibPath, int sgid_index);
virtual ~Address ();
......@@ -62,6 +62,8 @@ namespace pt
//! Get the path
std::string getPath ();
int getSGIDIndex();
//! Compare with another address
bool equals (pt::Address::Reference address);
......@@ -72,6 +74,7 @@ namespace pt
size_t ibPort_;
size_t ibPath_;
size_t sgid_index_;
};
}
}
......
......@@ -106,7 +106,7 @@ namespace pt
//
void config (pt::Address::Reference address) ;
void connect (ibvla::QueuePair & qp, const std::string & host, size_t port, size_t path) ;
void connect (ibvla::QueuePair & qp, const std::string & host, size_t port, size_t path, int sgid_index) ;
std::list<pt::Messenger::Reference> getMessengers ();
......
......@@ -19,8 +19,8 @@
// only for debugging inet_ntoa
pt::ibv::Address::Address (const std::string & url, const std::string& service, size_t ibPort, size_t ibPath)
: url_(url), ibPort_(ibPort), ibPath_(ibPath)
pt::ibv::Address::Address (const std::string & url, const std::string& service, size_t ibPort, size_t ibPath, int sgid_index)
: url_(url), ibPort_(ibPort), ibPath_(ibPath), sgid_index_(sgid_index)
{
/*
if (service_ != "i2o")
......@@ -110,6 +110,11 @@ std::string pt::ibv::Address::getServiceParameters ()
return url_.getPath();
}
int pt::ibv::Address::getSGIDIndex()
{
return sgid_index_;
}
bool pt::ibv::Address::equals (pt::Address::Reference address)
{
return ((this->toString() == address->toString()) && (this->getService() == address->getService()));
......
......@@ -123,7 +123,7 @@ void pt::ibv::I2OMessenger::postConnect ()
try
{
isConnected_ = true;
pt_->connect(qp_, daddress.getHost(), daddress.getPortNum(), laddress.getIBPath());
pt_->connect(qp_, daddress.getHost(), daddress.getPortNum(), laddress.getIBPath(), laddress.getSGIDIndex());
}
catch (pt::exception::Exception & e)
{
......
......@@ -288,6 +288,12 @@ pt::Address::Reference pt::ibv::PeerTransport::createAddress (std::map<std::stri
path = "0";
}
std::string ssgidindex = address["sgidindex"];
if (ssgidindex == "")
{
ssgidindex = "3";
}
if (protocol == "ibv")
{
std::string url = protocol;
......@@ -316,7 +322,8 @@ pt::Address::Reference pt::ibv::PeerTransport::createAddress (std::map<std::stri
//return this->createAddress(url, service, ibport);
size_t ibPortNum = toolbox::toUnsignedLong(ibport);
size_t ibPath = toolbox::toUnsignedLong(path);
return pt::Address::Reference(new pt::ibv::Address(url, service, ibPortNum, ibPath));
int sgidindex = toolbox::toUnsignedInt(ssgidindex);
return pt::Address::Reference(new pt::ibv::Address(url, service, ibPortNum, ibPath, sgidindex));
}
else
{
......@@ -484,7 +491,7 @@ void pt::ibv::PeerTransport::config (pt::Address::Reference address)
ibvla::CompletionQueue cqr = context_.createCompletionQueue(app->completionQueueSize_, 0, 0);
ibvla::CompletionWorkLoop * cwlr = new ibvla::CompletionWorkLoop("pt::ibv::completionworkloopr-" + a.getPort(), cqr, this);
cwlr_ = cwlr;
ibvla::Acceptor * acceptor = new ibvla::Acceptor("pt::ibv::acceptor-" + a.getPort(), this, app->deviceMTU_, cqr, rpool, a.getIBPort(), a.getIBPath());
ibvla::Acceptor * acceptor = new ibvla::Acceptor("pt::ibv::acceptor-" + a.getPort(), this, app->deviceMTU_, cqr, rpool, a.getIBPort(), a.getIBPath(), a.getSGIDIndex());
//acceptor_->listen("10.177.33.93", "3000");
acceptor->listen(a.getHost(), a.getPort());
......@@ -552,7 +559,7 @@ void pt::ibv::PeerTransport::removeAllServiceListeners ()
listener_ = 0;
}
void pt::ibv::PeerTransport::connect (ibvla::QueuePair & qp, const std::string & host, size_t port, size_t path)
void pt::ibv::PeerTransport::connect (ibvla::QueuePair & qp, const std::string & host, size_t port, size_t path, int sgid_index)
{
LOG4CPLUS_DEBUG(this->getOwnerApplication()->getApplicationLogger(), "schedule connection");
......@@ -562,7 +569,7 @@ void pt::ibv::PeerTransport::connect (ibvla::QueuePair & qp, const std::string &
{
std::stringstream p;
p << port;
ibvla::Connector conn(app->deviceMTU_, app->sendWithTimeout_);
ibvla::Connector conn(app->deviceMTU_, app->sendWithTimeout_, sgid_index);
conn.connect(qp, host, p.str(), path);
}
catch (ibvla::exception::Exception & e)
......
......@@ -11,7 +11,7 @@
<!-- Declare a context that contain applcation -->
<xc:Context url="http://dvrubu-c2f34-21-02.cms:1972">
<xc:Endpoint protocol="ibv" service="i2o" hostname="dvrubu-c2f34-21-02.cms" port="30001" network="infini" />
<xc:Endpoint protocol="ibv" service="i2o" hostname="dvrubu-c2f34-21-02.cms" port="30001" network="infini" sgidindex="3" />
<!-- Declare an application -->
......@@ -53,7 +53,7 @@
<xc:Context url="http://dvrubu-c2f34-21-03.cms:1972">
<xc:Endpoint protocol="ibv" service="i2o" hostname="dvrubu-c2f34-21-03.cms" port="60001" network="infini" />
<xc:Endpoint protocol="ibv" service="i2o" hostname="dvrubu-c2f34-21-03.cms" port="60001" network="infini" sgidindex="3" />
<!-- Declare an application -->
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment