Commit 1e8c45a4 authored by Dainius Simelevicius's avatar Dainius Simelevicius
Browse files

references #271: correctly interpreting return values and errno of ib verb functions

parent fd936733
// $Id$
/*************************************************************************
* XDAQ Components for Distributed Data Acquisition *
* Copyright (C) 2000-2014, CERN. *
* Copyright (C) 2000-2021, CERN. *
* All rights reserved. *
* Authors: L.Orsini, A.Petrucci, A.Forrest *
* Authors: L.Orsini, A.Petrucci, A.Forrest, D.Simelevicius *
* For the licensing terms see LICENSE. *
* For the list of contributors see CREDITS. *
*************************************************************************/
......@@ -18,9 +16,9 @@
#define WORKSUITE_IBVLA_VERSION_MAJOR 2
#define WORKSUITE_IBVLA_VERSION_MINOR 1
#define WORKSUITE_IBVLA_VERSION_PATCH 2
#define WORKSUITE_IBVLA_VERSION_PATCH 3
// If any previous versions available E.g. #define WORKSUITE_ESOURCE_PREVIOUS_VERSIONS "3.8.0,3.8.1"
#define WORKSUITE_IBVLA_PREVIOUS_VERSIONS "2.0.0,2.1.0,2.1.1"
#define WORKSUITE_IBVLA_PREVIOUS_VERSIONS "2.0.0,2.1.0,2.1.1,2.1.2"
//
// Template macros
......
// $Id$
/*************************************************************************
* XDAQ Components for Distributed Data Acquisition *
* Copyright (C) 2000-2014, CERN. *
* Copyright (C) 2000-2021, CERN. *
* All rights reserved. *
* Authors: L.Orsini, A.Petrucci, A.Forrest *
* Authors: L.Orsini, A.Petrucci, A.Forrest, D.Simelevicius *
* For the licensing terms see LICENSE. *
* For the list of contributors see CREDITS. *
*************************************************************************/
......@@ -27,19 +25,25 @@ ibvla::Context::~Context ()
{
}
ibv_device_attr ibvla::Context::queryDevice ()
ibv_device_attr ibvla::Context::queryDevice ()
{
errno = 0;
ibv_device_attr device_attr;
memset(&device_attr, 0, sizeof(device_attr));
int result = ibv_query_device(context_, &device_attr);
int ret = ibv_query_device(context_, &device_attr);
if (result != 0)
if (ret != 0)
{
std::stringstream ss;
ss << "ibv_query_device failed, errno = " << strerror(errno);
ss << "Failed to query device: ";
if (ret == ENOMEM)
{
ss << "Out of memory, " << strerror(ret) << "(" << ret << ")";
}
else
{
ss << "Not understood error, return value = " << ret;
}
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
......@@ -48,17 +52,27 @@ ibv_device_attr ibvla::Context::queryDevice ()
ibv_port_attr ibvla::Context::queryPort (uint8_t port_num)
{
errno = 0;
struct ibv_port_attr port_attr;
memset(&port_attr, 0, sizeof(port_attr));
int result = ibv_query_port(context_, port_num, &port_attr);
int ret = ibv_query_port(context_, port_num, &port_attr);
if (result != 0)
if (ret != 0)
{
std::stringstream ss;
ss << "ibv_query_port failed, errno = " << strerror(errno);
ss << "Failed to query port: ";
if (ret == EINVAL)
{
ss << "port_num is invalid, port_num = " << port_num << ", "<< strerror(ret) << "(" << ret << ")";
}
else if (ret == ENOMEM)
{
ss << "Out of memory, " << strerror(ret) << "(" << ret << ")";
}
else
{
ss << "Not understood error, return value = " << ret;
}
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
......@@ -68,16 +82,24 @@ ibv_port_attr ibvla::Context::queryPort (uint8_t port_num)
ibv_gid ibvla::Context::queryGID (uint8_t port_num, int index)
{
errno = 0;
union ibv_gid gid;
memset(&gid, 0, sizeof(gid));
int result = ibv_query_gid(context_, port_num, index, &gid);
int ret = ibv_query_gid(context_, port_num, index, &gid);
if (result != 0)
if (ret != 0)
{
std::stringstream ss;
ss << "ibv_query_gid failed, errno = " << strerror(errno);
ss << "Failed to query gid: ";
if (errno == EMFILE)
{
ss << "Too many files are opened by this process, " << strerror(errno) << "(" << errno << ")";
}
else
{
ss << "Not understood error, return value = " << ret << ", errno = " << errno;
}
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
......@@ -90,12 +112,20 @@ uint16_t ibvla::Context::queryPKey (uint8_t port_num, int index)
uint16_t pkey;
int result = ibv_query_pkey(context_, port_num, index, &pkey);
int ret = ibv_query_pkey(context_, port_num, index, &pkey);
if (result != 0)
if (ret != 0)
{
std::stringstream ss;
ss << "ibv_query_gid failed, errno = " << strerror(errno);
ss << "Failed to query P_Key: ";
if (errno == EMFILE)
{
ss << "Too many files are opened by this process, " << strerror(errno) << "(" << errno << ")";
}
else
{
ss << "Not understood error, return value = " << ret << ", errno = " << errno;
}
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
......@@ -104,8 +134,6 @@ uint16_t ibvla::Context::queryPKey (uint8_t port_num, int index)
ibvla::ProtectionDomain ibvla::Context::allocateProtectionDomain ()
{
errno = 0;
ibv_pd* pd = ibv_alloc_pd(context_);
if (pd == 0)
......@@ -120,16 +148,26 @@ ibvla::ProtectionDomain ibvla::Context::allocateProtectionDomain ()
return ibvla::ProtectionDomain(*this, pd);
}
void ibvla::Context::deallocateProtectionDomain (ProtectionDomain & pd)
void ibvla::Context::deallocateProtectionDomain (ProtectionDomain & pd)
{
errno = 0;
int ret = ibv_dealloc_pd(pd.pd_);
int i = ibv_dealloc_pd(pd.pd_);
if (i != 0)
if (ret != 0)
{
std::stringstream ss;
ss << "Failed to deallocate protection domain for context device '" << getDeviceName() << "', errno = " << strerror(errno);
ss << "Failed to deallocate protection domain for context device '" << getDeviceName() << "': ";
if (ret == EINVAL)
{
ss << "PD context is invalid, " << strerror(ret) << "(" << ret << ")";
}
else if (ret == EBUSY)
{
ss << "RDMA resources are still associated with this PD, " << strerror(ret) << "(" << ret << ")";
}
else
{
ss << "Not understood error, return value = " << ret;
}
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
......@@ -150,7 +188,19 @@ ibvla::CompletionQueue ibvla::Context::createCompletionQueue (int size, void* us
if (cq == 0)
{
std::stringstream ss;
ss << "Failed to create completion queue for context device '" << getDeviceName() << "', errno = " << strerror(errno);
ss << "Failed to create completion queue for context device '" << getDeviceName() << ": ";
if (errno == EINVAL)
{
ss << "Invalid cqe, channel or comp_vector, " << strerror(errno) << "(" << errno << ")";
}
else if (errno == ENOMEM)
{
ss << "Not enough resources to complete this operation, " << strerror(errno) << "(" << errno << ")";
}
else
{
ss << "Not understood error, errno = " << errno;
}
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
......@@ -166,14 +216,20 @@ ibvla::CompletionQueue ibvla::Context::createCompletionQueue (int size, void* us
void ibvla::Context::destroyCompletionQueue (ibvla::CompletionQueue& cq)
{
errno = 0;
int i = ibv_destroy_cq(cq.cq_);
int ret = ibv_destroy_cq(cq.cq_);
if (i != 0)
if (ret != 0)
{
std::stringstream ss;
ss << "Failed to destroy completion queue for device '" << getDeviceName() << "', errno = " << strerror(errno);
ss << "Failed to destroy completion queue for device '" << getDeviceName() << ": ";
if (ret == EBUSY)
{
ss << "One or more Work Queues is still associated with the CQ, " << strerror(ret) << "(" << ret << ")";
}
else
{
ss << "Not understood error, return value = " << ret;
}
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
}
......@@ -186,12 +242,12 @@ std::string ibvla::Context::getDeviceName ()
/*
* Performs a blocking call on the asynchronous event queue, and ACK's the event
*/
ibv_async_event ibvla::Context::waitAsyncEvent ()
ibv_async_event ibvla::Context::waitAsyncEvent ()
{
ibv_async_event e;
int i = ibv_get_async_event(context_, &e);
int ret = ibv_get_async_event(context_, &e);
if (i != 0)
if (ret != 0)
{
std::stringstream ss;
ss << "Failed to get async event for device '" << getDeviceName() << "'";
......
// $Id$
/*************************************************************************
* XDAQ Components for Distributed Data Acquisition *
* Copyright (C) 2000-2014, CERN. *
* Copyright (C) 2000-2021, CERN. *
* All rights reserved. *
* Authors: L.Orsini, A.Petrucci, A.Forrest *
* Authors: L.Orsini, A.Petrucci, A.Forrest, D.Simelevicius *
* For the licensing terms see LICENSE. *
* For the list of contributors see CREDITS. *
*************************************************************************/
......@@ -38,22 +36,45 @@ ibvla::MemoryRegion ibvla::ProtectionDomain::registerMemoryRegion (void *addr, s
if (mr == 0)
{
std::stringstream ss;
ss << "Failed to register memory region for protection domain '" << getContext().getDeviceName() << "', " << strerror(errno);
ss << "Failed to register memory region for protection domain '" << getContext().getDeviceName() << ": ";
if (errno == EINVAL)
{
ss << "Invalid access value, " << strerror(errno) << "(" << errno << ")";
}
else if (errno == ENOMEM)
{
ss << "Not enough resources (either in operating system or in RDMA device) to complete this operation, " << strerror(errno) << "(" << errno << ")";
}
else
{
ss << "Not understood error, errno = " << errno;
}
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
return ibvla::MemoryRegion(*this, mr);
}
void ibvla::ProtectionDomain::deregisterMemoryRegion (MemoryRegion & mr)
void ibvla::ProtectionDomain::deregisterMemoryRegion (MemoryRegion & mr)
{
errno = 0;
int ret = ibv_dereg_mr(mr.mr_);
if (ret != 0)
{
std::stringstream ss;
ss << "Failed to de-register memory region, " << strerror(errno);
ss << "Failed to de-register memory region: ";
if (ret == EINVAL)
{
ss << "MR context is invalid, " << strerror(ret) << "(" << ret << ")";
}
else if (ret == EBUSY)
{
ss << "MWs are still associated with this MR, " << strerror(ret) << "(" << ret << ")";
}
else
{
ss << "Not understood error, return value = " << ret;
}
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
}
......@@ -87,7 +108,27 @@ ibvla::QueuePair ibvla::ProtectionDomain::createQueuePair (ibvla::CompletionQueu
if (qp == 0)
{
std::stringstream ss;
ss << "Failed to create queue pair for device '" << getContext().getDeviceName() << "', errno = " << strerror(errno);
ss << "Failed to create queue pair for device '" << getContext().getDeviceName() << "': ";
if (errno == EINVAL)
{
ss << "Invalid pd, send_cq, recv_cq, srq or invalid value provided in max_send_wr, max_recv_wr, max_send_sge, max_recv_sge or in max_inline_data, " << strerror(errno) << "(" << errno << ")";
}
else if (errno == ENOMEM)
{
ss << "Not enough resources to complete this operation, " << strerror(errno) << "(" << errno << ")";
}
else if (errno == ENOSYS)
{
ss << "QP with this Transport Service Type isn't supported by this RDMA device, " << strerror(errno) << "(" << errno << ")";
}
else if (errno == EPERM)
{
ss << "Not enough permissions to create a QP with this Transport Service Type, " << strerror(errno) << "(" << errno << ")";
}
else
{
ss << "Not understood error, errno = " << errno;
}
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
......@@ -96,15 +137,20 @@ ibvla::QueuePair ibvla::ProtectionDomain::createQueuePair (ibvla::CompletionQueu
void ibvla::ProtectionDomain::destroyQueuePair (ibvla::QueuePair& qp)
{
int ret = 0;
errno = 0;
ret = ibv_destroy_qp(qp.qp_);
int ret = ibv_destroy_qp(qp.qp_);
if (ret != 0)
{
std::stringstream ss;
ss << "Failed to destroy queue pair for device '" << getContext().getDeviceName() << "', errno = " << strerror(errno);
ss << "Failed to destroy queue pair for device '" << getContext().getDeviceName() << "'";
if (ret == EBUSY)
{
ss << "QP is still attached to one or more multicast groups, " << strerror(ret) << "(" << ret << ")";
}
else
{
ss << "Not understood error, return value = " << ret;
}
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
}
......
// $Id$
/*************************************************************************
* XDAQ Components for Distributed Data Acquisition *
* Copyright (C) 2000-2014, CERN. *
* Copyright (C) 2000-2021, CERN. *
* All rights reserved. *
* Authors: L.Orsini, A.Petrucci, A.Forrest *
* Authors: L.Orsini, A.Petrucci, A.Forrest, D.Simelevicius *
* For the licensing terms see LICENSE. *
* For the list of contributors see CREDITS. *
*************************************************************************/
......@@ -28,12 +26,23 @@ ibvla::QueuePair::~QueuePair ()
void ibvla::QueuePair::modify (ibv_qp_attr &attr, int attr_mask)
{
errno = 0;
int ret = ibv_modify_qp(qp_, &attr, attr_mask);
if (ret != 0)
{
std::stringstream ss;
ss << "Failed to modify queue pair, errno = " << strerror(errno);
ss << "Failed to modify queue pair: ";
if (ret == EINVAL)
{
ss << "Invalid value provided in attr or in attr_mask, " << strerror(ret) << "(" << ret << ")";
}
else if (ret == ENOMEM)
{
ss << "Not enough resources to complete this operation, " << strerror(ret) << "(" << ret << ")";
}
else
{
ss << "Not understood error, return value = " << ret;
}
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
}
......@@ -45,7 +54,6 @@ void ibvla::QueuePair::modify (ibv_qp_attr &attr, int attr_mask)
*/
void ibvla::QueuePair::query (int attr_mask, ibv_qp_attr &attr, ibv_qp_init_attr &init_attr)
{
errno = 0;
memset(&attr, 0, sizeof(ibv_qp_attr));
memset(&init_attr, 0, sizeof(ibv_qp_init_attr));
......@@ -54,38 +62,44 @@ void ibvla::QueuePair::query (int attr_mask, ibv_qp_attr &attr, ibv_qp_init_attr
if (ret != 0)
{
std::stringstream ss;
ss << "Failed to query queue pair, errno = " << strerror(errno);
ss << "Failed to query queue pair: ";
if (ret == ENOMEM)
{
ss << "Not enough resources to complete this operation, " << strerror(ret) << "(" << ret << ")";
}
else
{
ss << "Not understood error, return value = " << ret;
}
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
//throw error("ibv_query_qp", result);
}
}
void ibvla::QueuePair::postSend (ibv_send_wr &wr)
{
errno = 0;
ibv_send_wr *bad_wr;
int ret = ibv_post_send(qp_, &wr, &bad_wr);
if (ret != 0)
{
std::stringstream ss;
// warning, errno is not being set correctly by this call
if (errno == EINVAL)
ss << "Failed to post send queue pair: ";
if (ret == EINVAL)
{
ss << "Failed to post send queue pair with invalid value in WR, " << strerror(errno) << "(" << errno << ")";
ss << "Invalid value provided in wr, " << strerror(ret) << "(" << ret << ")";
}
else if (errno == ENOMEM)
else if (ret == ENOMEM)
{
ss << "Failed to post send queue pair, not enough resources, " << strerror(errno) << "(" << errno << ")";
ss << "Send Queue is full or not enough resources to complete this operation, " << strerror(ret) << "(" << ret << ")";
XCEPT_RAISE(ibvla::exception::QueueFull, ss.str());
}
else if (errno == EFAULT)
else if (ret == EFAULT)
{
ss << "Failed to post send queue pair, invalid value in QP, " << strerror(errno) << "(" << errno << ")";
ss << "Invalid value provided in qp, " << strerror(ret) << "(" << ret << ")";
}
else
{
ss << "Not understood error, (errno=" << errno << ", return value=" << ret << ")";
ss << "Not understood error, return value = " << ret;
}
XCEPT_RAISE(ibvla::exception::InternalError, ss.str());
}
......@@ -93,14 +107,30 @@ void ibvla::QueuePair::postSend (ibv_send_wr &wr)
void ibvla::QueuePair::postRecv (ibv_recv_wr &wr)
{
errno = 0;
ibv_recv_wr *bad_wr;
int ret = ibv_post_recv(qp_, &wr, &bad_wr);
if (ret != 0)
{
std::stringstream ss;
ss << "Failed to post recv on queue pair, errno = " << strerror(errno);
ss << "Failed to post recv on queue pair, ";
if (ret == EINVAL)
{
ss << "Invalid value provided in wr, " << strerror(ret) << "(" << ret << ")";
}
else if (ret == ENOMEM)
{
ss << "Receive Queue is full or not enough resources to complete this operation, " << strerror(ret) << "(" << ret << ")";
XCEPT_RAISE(ibvla::exception::QueueFull, ss.str());
}
else if (ret == EFAULT)
{
ss << "Invalid value provided in qp, " << strerror(ret) << "(" << ret << ")";
}
else
{
ss << "Not understood error, return value = " << ret;
}
XCEPT_RAISE(ibvla::exception::Exception, ss.str());
}
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment