Commit caafa8a9 authored by Luciano Orsini's avatar Luciano Orsini Committed by Dainius Simelevicius
Browse files

references #73: support for optional sgid_index

parent a4b241f4
......@@ -36,7 +36,8 @@ namespace ibvla
std::string name_;
AcceptorListener * listener_;
Acceptor (const std::string & name, ibvla::AcceptorListener * listener, size_t mtu, ibvla::CompletionQueue cqr, toolbox::mem::Pool * rpool, size_t ibPort, size_t ibPath, int sgid_index);
Acceptor (const std::string & name, ibvla::AcceptorListener * listener, size_t mtu, ibvla::CompletionQueue cqr, toolbox::mem::Pool * rpool,
size_t ibPort, size_t ibPath, int sgid_index, int is_global);
void listen (const std::string & hostname, const std::string & port) ;
void accept (ConnectionRequest &id, QueuePair &qp) ;
......@@ -56,6 +57,7 @@ namespace ibvla
size_t ibPort_;
size_t ibPath_;
int sgid_index_;
int is_global_;
private:
};
}
......
......@@ -23,13 +23,14 @@ namespace ibvla
class Connector
{
public:
Connector (size_t mtu, bool sendWithTimeout, int sgid_index);
Connector (size_t mtu, bool sendWithTimeout, int sgid_index, int is_global);
void connect (ibvla::QueuePair &qp, const std::string & destname, const std::string & destport, size_t path);
ibv_mtu mtu_;
bool sendWithTimeout_;
int sgid_index_;
int is_global_;
};
}
......
......@@ -34,8 +34,8 @@
#include "toolbox/task/WorkLoopFactory.h"
ibvla::Acceptor::Acceptor (const std::string & name, ibvla::AcceptorListener * listener, size_t mtu, ibvla::CompletionQueue cqr, toolbox::mem::Pool * rpool, size_t ibPort, size_t ibPath, int sgid_index)
: name_(name), listener_(listener), cqr_(cqr), rpool_(rpool), ibPort_(ibPort), ibPath_(ibPath), sgid_index_(sgid_index)
ibvla::Acceptor::Acceptor (const std::string & name, ibvla::AcceptorListener * listener, size_t mtu, ibvla::CompletionQueue cqr, toolbox::mem::Pool * rpool, size_t ibPort, size_t ibPath, int sgid_index, int is_global)
: name_(name), listener_(listener), cqr_(cqr), rpool_(rpool), ibPort_(ibPort), ibPath_(ibPath), sgid_index_(sgid_index), is_global_(is_global)
{
listenfd_ = -1;
......@@ -252,19 +252,25 @@ void ibvla::Acceptor::accept (ibvla::ConnectionRequest &id, ibvla::QueuePair &qp
attr.max_dest_rd_atomic = 1;
attr.min_rnr_timer = 0;
// RoCE uses sgid_index_
std::cout << "DBG: Acceptor using GID index: " << sgid_index_ << '\n';
// For RoCE:
attr.ah_attr.is_global = 1;
attr.ah_attr.grh.dgid = id.remote_gid_;
attr.ah_attr.grh.sgid_index = sgid_index_;
attr.ah_attr.grh.hop_limit = 2;
attr.ah_attr.grh.traffic_class = 0;
attr.ah_attr.sl = 0;
//attr.ah_attr.is_global = 0;
//attr.ah_attr.sl = 0; //!!!!!!!!?????????
if ( is_global_)
{
// RoCE uses sgid_index_
std::cout << "DBG: Acceptor using GID index: " << sgid_index_ << '\n';
// For RoCE:
attr.ah_attr.is_global = 1;
attr.ah_attr.grh.dgid = id.remote_gid_;
attr.ah_attr.grh.sgid_index = sgid_index_;
attr.ah_attr.grh.hop_limit = 2;
attr.ah_attr.grh.traffic_class = 0;
}
else
{
attr.ah_attr.is_global = 0;
}
attr.ah_attr.src_path_bits = ibPath_;
//std::cout << "setting src_path to " << ibPath_ << std::endl;
......@@ -312,14 +318,23 @@ void ibvla::Acceptor::accept (ibvla::ConnectionRequest &id, ibvla::QueuePair &qp
// Communicate information back to remote to complete connection
// get port attributes
struct ibv_port_attr p_att = qp.pd_.context_.queryPort(qp_attr.port_num);
union ibv_gid gid = qp.pd_.context_.queryGID(qp_attr.port_num, sgid_index_);
char msg[sizeof "0000:000000:000000:0000000000000000:0000000000000000"];
sprintf(msg, "%04x:%06x:%06x:%016lx:%016lx", (p_att.lid + (unsigned int)ibPath_), qp.getNum(), attr.sq_psn, gid.global.subnet_prefix, gid.global.interface_id);
//sprintf(msg, "%04x:%06x:%06x", (p_att.lid + (unsigned int)ibPath_), qp.getNum(), attr.sq_psn);
std::cout << "DBG: Acceptor send : '" << msg << "\'\n";
if ( is_global_)
{
union ibv_gid gid = qp.pd_.context_.queryGID(qp_attr.port_num, sgid_index_);
sprintf(msg, "%04x:%06x:%06x:%016lx:%016lx", (p_att.lid + (unsigned int)ibPath_), qp.getNum(), attr.sq_psn, gid.global.subnet_prefix, gid.global.interface_id);
//sprintf(msg, "%04x:%06x:%06x", (p_att.lid + (unsigned int)ibPath_), qp.getNum(), attr.sq_psn);
std::cout << "DBG: Acceptor send : '" << msg << "\'\n";
}
else
{
sprintf(msg, "%04x:%06x:%06x:%016lx:%016lx", (p_att.lid + (unsigned int)ibPath_), qp.getNum(), attr.sq_psn, (long unsigned int)0x0, (long unsigned int)0x0);
}
//std::cout << "REPLY to remote address, lid='" << p_att.lid << "', qpn='" << qp.getNum() << "', psn='" << attr.sq_psn << "', msg='" << msg << std::endl;
if (write(id.sockfd_, msg, sizeof msg) != sizeof msg)
......
......@@ -33,7 +33,7 @@
#include <string.h>
#include <sstream>
ibvla::Connector::Connector (size_t mtu, bool sendWithTimeout, int sgid_index): sgid_index_(sgid_index)
ibvla::Connector::Connector (size_t mtu, bool sendWithTimeout, int sgid_index, int is_global): sgid_index_(sgid_index), is_global_(is_global)
{
srand48(time(NULL)); // seed the random generator for QP packet number initialization
......@@ -117,14 +117,24 @@ void ibvla::Connector::connect (ibvla::QueuePair &qp, const std::string & destna
struct ibv_port_attr p_att = qp.pd_.context_.queryPort(qp_attr.port_num);
// RoCE uses sgid_index_
union ibv_gid gid = qp.pd_.context_.queryGID(qp_attr.port_num, sgid_index_);
std::cout << "DBG: Connector using GID index: " << sgid_index_ << '\n';
if ( is_global_ )
{
// RoCE uses sgid_index_
union ibv_gid gid = qp.pd_.context_.queryGID(qp_attr.port_num, sgid_index_);
std::cout << "DBG: Connector using GID index: " << sgid_index_ << '\n';
sprintf(msg, "%04x:%06x:%06x:%016lx:%016lx", (p_att.lid + (unsigned int)path), qp.getNum(), psn, gid.global.subnet_prefix, gid.global.interface_id);
std::cout << "DBG: Connector send : '" << msg << "\'\n";
}
else
{
sprintf(msg, "%04x:%06x:%06x:%016lx:%016lx", (p_att.lid + (unsigned int)path), qp.getNum(), psn, (long unsigned int)0, (long unsigned int)0);
sprintf(msg, "%04x:%06x:%06x:%016lx:%016lx", (p_att.lid + (unsigned int)path), qp.getNum(), psn, gid.global.subnet_prefix, gid.global.interface_id);
}
//sprintf(msg, "%04x:%06x:%06x", (p_att.lid + (unsigned int)path), qp.getNum(), psn);
std::cout << "DBG: Connector send : '" << msg << "\'\n";
//std::cout << "Sending connection info to remote address, lid='" << p_att.lid << "', qpn='" << qp.getNum() << "', psn='" << psn << "', msg='" << msg << std::endl;
if (write(sockfd, msg, sizeof msg) != sizeof msg)
......@@ -180,16 +190,20 @@ void ibvla::Connector::connect (ibvla::QueuePair &qp, const std::string & destna
// min_rnr_timer == receiver side, time to wait for receive buffer before sending RNR NAK
attr.min_rnr_timer = 0; // 655.36 ms (0.6 seconds)
// For RoCE:
attr.ah_attr.is_global = 1;
attr.ah_attr.grh.dgid = r_gid;
attr.ah_attr.grh.sgid_index = sgid_index_;
attr.ah_attr.grh.hop_limit = 2;
attr.ah_attr.grh.traffic_class = 0;
attr.ah_attr.sl = 0;
//attr.ah_attr.is_global = 0;
//attr.ah_attr.sl = 0;
// For RoCE:
if ( is_global_ )
{
attr.ah_attr.is_global = 1;
attr.ah_attr.grh.dgid = r_gid;
attr.ah_attr.grh.sgid_index = sgid_index_;
attr.ah_attr.grh.hop_limit = 2;
attr.ah_attr.grh.traffic_class = 0;
}
else
{
attr.ah_attr.is_global = 0;
}
attr.ah_attr.src_path_bits = path;
......
......@@ -25,7 +25,7 @@ namespace pt
public:
//! Create address from url
Address (const std::string & url, const std::string& service, size_t ibPort, size_t ibPath, int sgid_index);
Address (const std::string & url, const std::string& service, size_t ibPort, size_t ibPath, int sgid_index, int is_global);
virtual ~Address ();
......@@ -74,7 +74,9 @@ namespace pt
size_t ibPort_;
size_t ibPath_;
size_t sgid_index_;
int sgid_index_;
int is_global_;
};
}
}
......
......@@ -19,8 +19,8 @@
// only for debugging inet_ntoa
pt::ibv::Address::Address (const std::string & url, const std::string& service, size_t ibPort, size_t ibPath, int sgid_index)
: url_(url), ibPort_(ibPort), ibPath_(ibPath), sgid_index_(sgid_index)
pt::ibv::Address::Address (const std::string & url, const std::string& service, size_t ibPort, size_t ibPath, int sgid_index, int is_global)
: url_(url), ibPort_(ibPort), ibPath_(ibPath), sgid_index_(sgid_index), is_global_(is_global)
{
/*
if (service_ != "i2o")
......@@ -115,6 +115,11 @@ int pt::ibv::Address::getSGIDIndex()
return sgid_index_;
}
int pt::ibv::Address::getIsGlobal()
{
return is_global_;
}
bool pt::ibv::Address::equals (pt::Address::Reference address)
{
return ((this->toString() == address->toString()) && (this->getService() == address->getService()));
......
......@@ -123,7 +123,7 @@ void pt::ibv::I2OMessenger::postConnect ()
try
{
isConnected_ = true;
pt_->connect(qp_, daddress.getHost(), daddress.getPortNum(), laddress.getIBPath(), laddress.getSGIDIndex());
pt_->connect(qp_, daddress.getHost(), daddress.getPortNum(), laddress.getIBPath(), laddress.getSGIDIndex(), laddress.getIsGlobal());
}
catch (pt::exception::Exception & e)
{
......
......@@ -269,7 +269,7 @@ pt::TransportType pt::ibv::PeerTransport::getType ()
//
pt::Address::Reference pt::ibv::PeerTransport::createAddress (const std::string& url, const std::string& service)
{
return pt::Address::Reference(new pt::ibv::Address(url, service, 1, 0, 3)); // 1 == default IB port number, 0 = default src_path_bits, 3 = default sgid_index
return pt::Address::Reference(new pt::ibv::Address(url, service, 1, 0, 3, 0)); // 1 == default IB port number, 0 = default src_path_bits, 3 = default sgid_index. is_global = 0
}
pt::Address::Reference pt::ibv::PeerTransport::createAddress (std::map<std::string, std::string, std::less<std::string> >& address)
......@@ -288,10 +288,13 @@ pt::Address::Reference pt::ibv::PeerTransport::createAddress (std::map<std::stri
path = "0";
}
std::string ssgidindex = address["sgidindex"];
if (ssgidindex == "")
int isglobal = 0;
std::string ssgidindex = "0";
if (address.find("sgidindex") != address.end())
{
ssgidindex = "3";
isglobal = 1;
ssgidindex = address["sgidindex"];
}
if (protocol == "ibv")
......@@ -323,7 +326,7 @@ pt::Address::Reference pt::ibv::PeerTransport::createAddress (std::map<std::stri
size_t ibPortNum = toolbox::toUnsignedLong(ibport);
size_t ibPath = toolbox::toUnsignedLong(path);
int sgidindex = toolbox::toLong(ssgidindex);
return pt::Address::Reference(new pt::ibv::Address(url, service, ibPortNum, ibPath, sgidindex));
return pt::Address::Reference(new pt::ibv::Address(url, service, ibPortNum, ibPath, sgidindex, isglobal));
}
else
{
......@@ -491,7 +494,7 @@ void pt::ibv::PeerTransport::config (pt::Address::Reference address)
ibvla::CompletionQueue cqr = context_.createCompletionQueue(app->completionQueueSize_, 0, 0);
ibvla::CompletionWorkLoop * cwlr = new ibvla::CompletionWorkLoop("pt::ibv::completionworkloopr-" + a.getPort(), cqr, this);
cwlr_ = cwlr;
ibvla::Acceptor * acceptor = new ibvla::Acceptor("pt::ibv::acceptor-" + a.getPort(), this, app->deviceMTU_, cqr, rpool, a.getIBPort(), a.getIBPath(), a.getSGIDIndex());
ibvla::Acceptor * acceptor = new ibvla::Acceptor("pt::ibv::acceptor-" + a.getPort(), this, app->deviceMTU_, cqr, rpool, a.getIBPort(), a.getIBPath(), a.getSGIDIndex(), a.getIsGlobal());
//acceptor_->listen("10.177.33.93", "3000");
acceptor->listen(a.getHost(), a.getPort());
......@@ -559,7 +562,7 @@ void pt::ibv::PeerTransport::removeAllServiceListeners ()
listener_ = 0;
}
void pt::ibv::PeerTransport::connect (ibvla::QueuePair & qp, const std::string & host, size_t port, size_t path, int sgid_index)
void pt::ibv::PeerTransport::connect (ibvla::QueuePair & qp, const std::string & host, size_t port, size_t path, int sgid_index, int is_global)
{
LOG4CPLUS_DEBUG(this->getOwnerApplication()->getApplicationLogger(), "schedule connection");
......@@ -569,7 +572,7 @@ void pt::ibv::PeerTransport::connect (ibvla::QueuePair & qp, const std::string &
{
std::stringstream p;
p << port;
ibvla::Connector conn(app->deviceMTU_, app->sendWithTimeout_, sgid_index);
ibvla::Connector conn(app->deviceMTU_, app->sendWithTimeout_, sgid_index, is_global);
conn.connect(qp, host, p.str(), path);
}
catch (ibvla::exception::Exception & e)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment