Commit b15b669b authored by Remi Mommsen's avatar Remi Mommsen Committed by Dainius Simelevicius
Browse files

references #73: first version which allows to connect over RoCE in daq3val

parent b7605e07
......@@ -25,11 +25,12 @@ namespace ibvla
{
public:
ConnectionRequest (int lid, int psn, int qpn, Acceptor * acceptor, int sockfd, const std::string & ip, const std::string & hostname);
ConnectionRequest (int lid, int psn, int qpn, union ibv_gid gid, Acceptor * acceptor, int sockfd, const std::string & ip, const std::string & hostname);
int remote_lid_;
int remote_psn_;
int remote_qpn_;
union ibv_gid remote_gid_;
Acceptor * acceptor_;
......
......@@ -17,8 +17,8 @@
// !!! Edit this line to reflect the latest package version !!!
#define WORKSUITE_IBVLA_VERSION_MAJOR 2
#define WORKSUITE_IBVLA_VERSION_MINOR 0
#define WORKSUITE_IBVLA_VERSION_PATCH 1
#define WORKSUITE_IBVLA_VERSION_MINOR 1
#define WORKSUITE_IBVLA_VERSION_PATCH 0
// If any previous versions available E.g. #define WORKSUITE_ESOURCE_PREVIOUS_VERSIONS "3.8.0,3.8.1"
#define WORKSUITE_IBVLA_PREVIOUS_VERSIONS "2.0.0"
......
......@@ -63,7 +63,7 @@ bool ibvla::Acceptor::process (toolbox::task::WorkLoop* wl)
return true;
}
char msg[sizeof "0000:000000:000000"];
char msg[sizeof "0000:000000:000000:0000000000000000:0000000000000000"];
#warning unsafe read
......@@ -72,7 +72,8 @@ bool ibvla::Acceptor::process (toolbox::task::WorkLoop* wl)
{
close(connfd);
std::stringstream ss;
ss << "WARNING : Failed to read";
//ss << "WARNING : Failed to read";
ss << "WARNING : Failed to read, received bytes " << n << ", expected bytes " << sizeof(msg) << ", msg: '" << msg << '\'';
std::cout << ss.str() << std::endl;
return true;
}
......@@ -80,7 +81,12 @@ bool ibvla::Acceptor::process (toolbox::task::WorkLoop* wl)
int lid;
int psn;
int qpn;
sscanf(msg, "%x:%x:%x", &lid, &qpn, &psn);
union ibv_gid gid;
sscanf(msg, "%x:%x:%x:%lx:%lx", &lid, &qpn, &psn, &gid.global.subnet_prefix, &gid.global.interface_id);
//sscanf(msg, "%x:%x:%x", &lid, &qpn, &psn);
std::cout << "DBG: Acceptor received : '" << msg << "\'\n";
std::string ip;
ip = inet_ntoa(readAddr.sin_addr);
......@@ -97,7 +103,7 @@ bool ibvla::Acceptor::process (toolbox::task::WorkLoop* wl)
//std::cout << "Received connection info from remote address " << ip << ", lid='" << lid << "', qpn='" << qpn << "', psn='" << psn << "', msg='" << msg << std::endl;
listener_->connectionRequest(ibvla::ConnectionRequest(lid, psn, qpn, this, connfd, ip, remoteHostName));
listener_->connectionRequest(ibvla::ConnectionRequest(lid, psn, qpn, gid, this, connfd, ip, remoteHostName));
return true;
}
......@@ -245,8 +251,24 @@ void ibvla::Acceptor::accept (ibvla::ConnectionRequest &id, ibvla::QueuePair &qp
attr.max_dest_rd_atomic = 1;
attr.min_rnr_timer = 0;
attr.ah_attr.is_global = 0;
attr.ah_attr.sl = 0; //!!!!!!!!?????????
/*
* FIXME: RoCE: gid_index is either configurable or we have to find the best of all possible gid indexes (based on e.g. RoCE version supported)
* Number 3 would work only for d3vrubu(s)
*/
int gid_index = 3;
std::cout << "DBG: Acceptor using GID index: " << gid_index << '\n';
// For RoCE:
attr.ah_attr.is_global = 1;
attr.ah_attr.grh.dgid = id.remote_gid_;
attr.ah_attr.grh.sgid_index = gid_index;
attr.ah_attr.grh.hop_limit = 2;
attr.ah_attr.grh.traffic_class = 0;
attr.ah_attr.sl = 0;
//attr.ah_attr.is_global = 0;
//attr.ah_attr.sl = 0; //!!!!!!!!?????????
attr.ah_attr.src_path_bits = ibPath_;
//std::cout << "setting src_path to " << ibPath_ << std::endl;
......@@ -294,9 +316,13 @@ void ibvla::Acceptor::accept (ibvla::ConnectionRequest &id, ibvla::QueuePair &qp
// Communicate information back to remote to complete connection
// get port attributes
struct ibv_port_attr p_att = qp.pd_.context_.queryPort(qp_attr.port_num);
union ibv_gid gid = qp.pd_.context_.queryGID(qp_attr.port_num, gid_index);
char msg[sizeof "0000:000000:000000:0000000000000000:0000000000000000"];
sprintf(msg, "%04x:%06x:%06x:%016lx:%016lx", (p_att.lid + (unsigned int)ibPath_), qp.getNum(), attr.sq_psn, gid.global.subnet_prefix, gid.global.interface_id);
//sprintf(msg, "%04x:%06x:%06x", (p_att.lid + (unsigned int)ibPath_), qp.getNum(), attr.sq_psn);
char msg[sizeof "0000:000000:000000"];
sprintf(msg, "%04x:%06x:%06x", (p_att.lid + (unsigned int)ibPath_), qp.getNum(), attr.sq_psn);
std::cout << "DBG: Acceptor send : '" << msg << "\'\n";
//std::cout << "REPLY to remote address, lid='" << p_att.lid << "', qpn='" << qp.getNum() << "', psn='" << attr.sq_psn << "', msg='" << msg << std::endl;
......
......@@ -13,7 +13,7 @@
#include "ibvla/Acceptor.h"
ibvla::ConnectionRequest::ConnectionRequest (int lid, int psn, int qpn, ibvla::Acceptor * acceptor, int sockfd, const std::string & ip, const std::string & hostname)
: remote_lid_(lid), remote_psn_(psn), remote_qpn_(qpn), acceptor_(acceptor), sockfd_(sockfd), ip_(ip), hostname_(hostname)
ibvla::ConnectionRequest::ConnectionRequest (int lid, int psn, int qpn, union ibv_gid gid, ibvla::Acceptor * acceptor, int sockfd, const std::string & ip, const std::string & hostname)
: remote_lid_(lid), remote_psn_(psn), remote_qpn_(qpn), remote_gid_(gid), acceptor_(acceptor), sockfd_(sockfd), ip_(ip), hostname_(hostname)
{
}
......@@ -53,7 +53,7 @@ void ibvla::Connector::connect (ibvla::QueuePair &qp, const std::string & destna
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
char msg[sizeof "0000:000000:000000"];
char msg[sizeof "0000:000000:000000:0000000000000000:0000000000000000"];
int n;
int sockfd = -1;
......@@ -117,7 +117,18 @@ void ibvla::Connector::connect (ibvla::QueuePair &qp, const std::string & destna
struct ibv_port_attr p_att = qp.pd_.context_.queryPort(qp_attr.port_num);
sprintf(msg, "%04x:%06x:%06x", (p_att.lid + (unsigned int)path), qp.getNum(), psn);
/*
* FIXME: RoCE: gid_index is either configurable or we have to find the best of all possible gid indexes (based on e.g. RoCE version supported)
* Number 3 would work only for d3vrubu(s)
*/
int gid_index = 3;
union ibv_gid gid = qp.pd_.context_.queryGID(qp_attr.port_num, gid_index);
std::cout << "DBG: Connector using GID index: " << gid_index << '\n';
sprintf(msg, "%04x:%06x:%06x:%016lx:%016lx", (p_att.lid + (unsigned int)path), qp.getNum(), psn, gid.global.subnet_prefix, gid.global.interface_id);
//sprintf(msg, "%04x:%06x:%06x", (p_att.lid + (unsigned int)path), qp.getNum(), psn);
std::cout << "DBG: Connector send : '" << msg << "\'\n";
//std::cout << "Sending connection info to remote address, lid='" << p_att.lid << "', qpn='" << qp.getNum() << "', psn='" << psn << "', msg='" << msg << std::endl;
if (write(sockfd, msg, sizeof msg) != sizeof msg)
......@@ -128,11 +139,13 @@ void ibvla::Connector::connect (ibvla::QueuePair &qp, const std::string & destna
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
if (read(sockfd, msg, sizeof msg) != sizeof msg)
int read_bytes;
if ( (read_bytes = read(sockfd, msg, sizeof msg)) != sizeof msg)
{
close(sockfd);
std::stringstream ss;
ss << "Failed to read from server:port '" << destname << ":" << destport;// << ", " << strerror(errno);
//ss << "Failed to read from server:port '" << destname << ":" << destport;// << ", " << strerror(errno);
ss << "Failed to read from server:port '" << destname << ":" << destport << ", received bytes " << read_bytes << ", expected bytes " << sizeof(msg) << ", msg: '" << msg << "', errno " << errno;
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
......@@ -146,11 +159,16 @@ void ibvla::Connector::connect (ibvla::QueuePair &qp, const std::string & destna
close(sockfd);
std::cout << "DBG: Connector received: '" << msg << "\'\n";
ibv_qp_attr attr;
memset(&attr, 0, sizeof(attr));
int r_lid, r_qpn, r_psn;
sscanf(msg, "%x:%x:%x", &r_lid, &r_qpn, &r_psn);
union ibv_gid r_gid;
sscanf(msg, "%x:%x:%x:%lx:%lx", &r_lid, &r_qpn, &r_psn, &r_gid.global.subnet_prefix, &r_gid.global.interface_id);
//sscanf(msg, "%x:%x:%x", &r_lid, &r_qpn, &r_psn);
//std::cout << "Received connection info from remote address, lid='" << r_lid << "', qpn='" << r_qpn << "', psn='" << r_psn << "', msg='" << msg << std::endl;
......@@ -165,8 +183,18 @@ void ibvla::Connector::connect (ibvla::QueuePair &qp, const std::string & destna
attr.max_dest_rd_atomic = 1;
// min_rnr_timer == receiver side, time to wait for receive buffer before sending RNR NAK
attr.min_rnr_timer = 0; // 655.36 ms (0.6 seconds)
attr.ah_attr.is_global = 0;
// For RoCE:
attr.ah_attr.is_global = 1;
attr.ah_attr.grh.dgid = r_gid;
attr.ah_attr.grh.sgid_index = gid_index;
attr.ah_attr.grh.hop_limit = 2;
attr.ah_attr.grh.traffic_class = 0;
attr.ah_attr.sl = 0;
//attr.ah_attr.is_global = 0;
//attr.ah_attr.sl = 0;
attr.ah_attr.src_path_bits = path;
//std::cout << "setting (send) src_path to " << path << std::endl;
......@@ -214,4 +242,3 @@ void ibvla::Connector::connect (ibvla::QueuePair &qp, const std::string & destna
XCEPT_RETHROW(ibvla::exception::Exception, ss.str(), e);
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment