Commit dcaa4761 authored by Luciano Orsini's avatar Luciano Orsini
Browse files

references #188: support for backward compatible iaName configuration

parent 655e15b3
......@@ -112,7 +112,6 @@ namespace pt
xdata::String iaName_;
xdata::String recvPoolName_;
xdata::String sendPoolName_;
//xdata::Boolean useNetworkInterface_;
xdata::String networkInterface_;
xdata::UnsignedInteger SGIDIndex_;
xdata::UnsignedInteger portNumber_;
......
......@@ -19,10 +19,10 @@
// !!! Edit this line to reflect the latest package version !!!
#define WORKSUITE_PTIBV_VERSION_MAJOR 3
#define WORKSUITE_PTIBV_VERSION_MINOR 0
#define WORKSUITE_PTIBV_VERSION_MINOR 1
#define WORKSUITE_PTIBV_VERSION_PATCH 0
// If any previous versions available E.g. #define WORKSUITE_PTIBV_PREVIOUS_VERSIONS "3.8.0,3.8.1"
#undef WORKSUITE_PTIBV_PREVIOUS_VERSIONS
#define WORKSUITE_PTIBV_PREVIOUS_VERSIONS "3.0.0"
//
// Template macros
......
......@@ -52,11 +52,10 @@ pt::ibv::Application::Application (xdaq::ApplicationStub* stub)
rcmsStateNotifier_(getApplicationLogger(), const_cast<xdaq::ApplicationDescriptor*>(getApplicationDescriptor()),getApplicationContext())
{
retrivedRcmsStateListener_ = false;
this->iaName_ = "unknown";
//this->useNetworkInterface_ = false;
this->iaName_ = ""; // for backward compatibility, if not set linux interface will be used, if set iaName will be used
this->networkInterface_ = "unknown";
this->portNumber_ = 1;
this->SGIDIndex_ = 0;
this->portNumber_ = 1; // for backward compatibility
this->SGIDIndex_ = 3; // for backward compatibility
this->senderPoolSize_ = 0x100000;
this->receiverPoolSize_ = 0x100000;
......@@ -124,7 +123,7 @@ pt::ibv::Application::Application (xdaq::ApplicationStub* stub)
xoap::bind(this, &pt::ibv::Application::fireEvent, "Fail", XDAQ_NS_URI );
xoap::bind(this, &pt::ibv::Application::fireEvent, "fail", XDAQ_NS_URI );
//this->getApplicationInfoSpace()->fireItemAvailable("iaName", &iaName_);
this->getApplicationInfoSpace()->fireItemAvailable("iaName", &iaName_); // export for bacward compatibility
//this->getApplicationInfoSpace()->fireItemAvailable("useNetworkInterface", &useNetworkInterface_);
this->getApplicationInfoSpace()->fireItemAvailable("networkInterface", &networkInterface_);
this->getApplicationInfoSpace()->fireItemAvailable("SGIDIndex", &SGIDIndex_);
......
......@@ -210,76 +210,106 @@ pt::ibv::PeerTransport::PeerTransport (xdaq::Application * parent)
LOG4CPLUS_INFO(app->getApplicationLogger(), "Found " << devices.size() << " devices");
int ltype = getlinktype(app->networkInterface_);
LOG4CPLUS_INFO(app->getApplicationLogger(), "Link type detected: " << linktype2string(ltype) );
union ibv_gid mgid; // GID derived from linux interface name
if ( ltype == ARPHRD_INFINIBAND )
{
isGlobal_ = false;
mgid = mac2gid(app->networkInterface_); // infiniban only
//std::cout << "Hardware address to GID is " << std::hex << bswap_64(mgid.global.subnet_prefix) << "-" << bswap_64(mgid.global.interface_id) << std::dec << std::endl;
}
else if ( ltype == ARPHRD_ETHER )
if ( app->iaName_ != "" ) // use of linux interface
{
isGlobal_ = true;
std::string ipv4 = getIPv4(app->networkInterface_.c_str());
uint64_t iaddr = inet_addr(ipv4.c_str()) & 0x00000000ffffffff;
mgid.global.interface_id = (iaddr << 32) | 0xffff0000;
mgid.global.subnet_prefix = 0x0;
//std::cout << "Original IPV5 adress binary " << std::hex << bswap_64(addr) << std::dec << std::endl;
//std::cout << "Expected GID binary " << std::hex << bswap_64(egid) << std::dec << std::endl;
//std::cout << "IP address to GID is " << std::hex << bswap_64(mgid.global.subnet_prefix) << "-" << bswap_64(mgid.global.interface_id) << std::dec << std::endl;
}
else
{
XCEPT_RAISE(pt::exception::Exception, "link type not supported for " + linktype2string(ltype));
}
ibvla::Device device;
int ltype = getlinktype(app->networkInterface_);
LOG4CPLUS_INFO(app->getApplicationLogger(), "Link type detected: " << linktype2string(ltype) );
std::list<ibvla::Device>::iterator i;
for (i = devices.begin(); i != devices.end(); i++)
{
//LOG4CPLUS_DEBUG(app->getApplicationLogger(), "Device name = '" << (*i).getName() << "', guid = ' 0x" << std::hex << bswap_64((*i).getGUID()) << std::dec << "'");
//printf("RDMA device[]: name=%s, GUID=0x%016Lx\n", (*i).getName().c_str(), (unsigned long long)ntohll((*i).getGUID()));
union ibv_gid mgid; // GID derived from linux interface name
if ( ltype == ARPHRD_INFINIBAND )
{
isGlobal_ = false;
mgid = mac2gid(app->networkInterface_); // infiniban only
//std::cout << "Hardware address to GID is " << std::hex << bswap_64(mgid.global.subnet_prefix) << "-" << bswap_64(mgid.global.interface_id) << std::dec << std::endl;
}
else if ( ltype == ARPHRD_ETHER )
{
isGlobal_ = true;
std::string ipv4 = getIPv4(app->networkInterface_.c_str());
uint64_t iaddr = inet_addr(ipv4.c_str()) & 0x00000000ffffffff;
mgid.global.interface_id = (iaddr << 32) | 0xffff0000;
mgid.global.subnet_prefix = 0x0;
//std::cout << "Original IPV5 adress binary " << std::hex << bswap_64(addr) << std::dec << std::endl;
//std::cout << "Expected GID binary " << std::hex << bswap_64(egid) << std::dec << std::endl;
//std::cout << "IP address to GID is " << std::hex << bswap_64(mgid.global.subnet_prefix) << "-" << bswap_64(mgid.global.interface_id) << std::dec << std::endl;
}
else
{
XCEPT_RAISE(pt::exception::Exception, "link type not supported for " + linktype2string(ltype));
}
ibvla::Device device;
std::list<ibvla::Device>::iterator i;
for (i = devices.begin(); i != devices.end(); i++)
{
//LOG4CPLUS_DEBUG(app->getApplicationLogger(), "Device name = '" << (*i).getName() << "', guid = ' 0x" << std::hex << bswap_64((*i).getGUID()) << std::dec << "'");
//printf("RDMA device[]: name=%s, GUID=0x%016Lx\n", (*i).getName().c_str(), (unsigned long long)ntohll((*i).getGUID()));
ibvla::Context context = ibvla::createContext((*i));
try
{
LOG4CPLUS_DEBUG(app->getApplicationLogger(), "going to query GID for " << (*i).getName() << "index " << app->SGIDIndex_ << " port " << app->portNumber_ );
union ibv_gid gid = context.queryGID(app->portNumber_, app->SGIDIndex_);
/*enum ibv_gid_type_sysfs sgid_type = 0;
int ret = ibv_query_gid_type(context_.context_, 1, sgid_index, &sgid_type);
*/
if ( mgid.global.interface_id == gid.global.interface_id )
{
//std::cout << "compare (not swapped)" << std::hex << mgid.global.interface_id << " with " << gid.global.interface_id << std::dec << std::endl;
//std::cout << "found matching IP for device " << (*i).getName() << std::endl;
//printf("index %d port %d %016lx:%016lx\n", (int)app->SGIDIndex_, (int)app->portNumber_, bswap_64(gid.global.subnet_prefix), bswap_64(gid.global.interface_id));
context_ = context;
app->iaName_ = (*i).getName();
LOG4CPLUS_INFO(app->getApplicationLogger(), "found matching interface " << app->networkInterface_.toString() << " for device " << (*i).getName() << " at index " << app->SGIDIndex_ << " port " << app->portNumber_ << " GID " << std::hex << bswap_64(gid.global.subnet_prefix) << ":" << bswap_64(gid.global.interface_id) );
break;
}
}
catch( ibvla::exception::Exception & e )
{
XCEPT_RETHROW(pt::ibv::exception::Exception, "failed to create IB context ", e);
}
ibvla::destroyContext(context);
}
ibvla::Context context = ibvla::createContext((*i));
try
if (i == devices.end())
{
LOG4CPLUS_DEBUG(app->getApplicationLogger(), "going to query GID for " << (*i).getName() << "index " << app->SGIDIndex_ << " port " << app->portNumber_ );
union ibv_gid gid = context.queryGID(app->portNumber_, app->SGIDIndex_);
ibvla::freeDeviceList (devices);
std::stringstream msg;
msg << "Could not find device for interface '" << app->networkInterface_.toString() << "' index " << app->SGIDIndex_ << " port " << app->portNumber_ << std::endl;
/*enum ibv_gid_type_sysfs sgid_type = 0;
int ret = ibv_query_gid_type(context_.context_, 1, sgid_index, &sgid_type);
*/
if ( mgid.global.interface_id == gid.global.interface_id )
XCEPT_RAISE(pt::exception::Exception, msg.str());
}
}
else // use iaName
{
ibvla::Device device;
std::list<ibvla::Device>::iterator i;
for (i = devices.begin(); i != devices.end(); i++)
{
if ((*i).getName() == app->iaName_.toString())
{
//std::cout << "compare (not swapped)" << std::hex << mgid.global.interface_id << " with " << gid.global.interface_id << std::dec << std::endl;
//std::cout << "found matching IP for device " << (*i).getName() << std::endl;
//printf("index %d port %d %016lx:%016lx\n", (int)app->SGIDIndex_, (int)app->portNumber_, bswap_64(gid.global.subnet_prefix), bswap_64(gid.global.interface_id));
context_ = context;
app->iaName_ = (*i).getName();
LOG4CPLUS_INFO(app->getApplicationLogger(), "found matching interface " << app->networkInterface_.toString() << " for device " << (*i).getName() << " at index " << app->SGIDIndex_ << " port " << app->portNumber_ << " GID " << std::hex << bswap_64(gid.global.subnet_prefix) << ":" << bswap_64(gid.global.interface_id) );
break;
LOG4CPLUS_INFO(app->getApplicationLogger(), "Found device name = '" << (*i).getName() << "', guid = ' 0x" << std::hex << bswap_64((*i).getGUID()) << std::dec << "'");
break;
}
}
catch( ibvla::exception::Exception & e )
if (i == devices.end())
{
XCEPT_RETHROW(pt::ibv::exception::Exception, "failed to create IB context ", e);
ibvla::freeDeviceList (devices);
std::stringstream msg;
msg << "Could not find device '" << app->iaName_.toString() << "', peer transport configuration failure" << std::endl;
XCEPT_RAISE(pt::exception::Exception, msg.str());
}
ibvla::destroyContext(context);
}
if (i == devices.end())
{
ibvla::freeDeviceList (devices);
std::stringstream msg;
msg << "Could not find device for interface '" << app->networkInterface_.toString() << "' index " << app->SGIDIndex_ << " port " << app->portNumber_ << std::endl;
context_ = ibvla::createContext((*i));
}
XCEPT_RAISE(pt::exception::Exception, msg.str());
}
pd_ = context_.allocateProtectionDomain();
......@@ -471,7 +501,32 @@ pt::Address::Reference pt::ibv::PeerTransport::createAddress (std::map<std::stri
path = "0";
}
bool isglobal = isGlobal_;
pt::ibv::Application * app = dynamic_cast<pt::ibv::Application *>(this->getOwnerApplication());
bool isglobal = this->isGlobal_;
int sgidindex = app->SGIDIndex_;
size_t ibPortNum = app->portNumber_;
// Backward compatible with iaName override isglobal and sgindex nad ibport
if (address.find("isglobal") != address.end())
{
sgidindex = 3;
isglobal = true;
}
if (address.find("sgidindex") != address.end())
{
isglobal = true;
sgidindex = toolbox::toLong(address["sgidindex"]);
}
if (address.find("ibport") != address.end())
{
ibPortNum = toolbox::toUnsignedLong(address["ibport"]);
}
//
if (protocol == "ibv")
{
......@@ -499,10 +554,7 @@ pt::Address::Reference pt::ibv::PeerTransport::createAddress (std::map<std::stri
}
//return this->createAddress(url, service, ibport);
pt::ibv::Application * app = dynamic_cast<pt::ibv::Application *>(this->getOwnerApplication());
size_t ibPortNum = app->portNumber_;
size_t ibPath = toolbox::toUnsignedLong(path);
int sgidindex = app->SGIDIndex_;
return pt::Address::Reference(new pt::ibv::Address(url, service, ibPortNum, ibPath, sgidindex, isglobal));
}
else
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment