references #271: correctly interpreting return values and errno of ib verb functions

ibvla/include/ibvla/version.h

-// $Id$
-
 /*************************************************************************
  * XDAQ Components for Distributed Data Acquisition                      *
- * Copyright (C) 2000-2014, CERN.                                        *
+ * Copyright (C) 2000-2021, CERN.                                        *
  * All rights reserved.                                                  *
- * Authors: L.Orsini, A.Petrucci, A.Forrest								 *
+ * Authors: L.Orsini, A.Petrucci, A.Forrest, D.Simelevicius              *
  * For the licensing terms see LICENSE.                                  *
  * For the list of contributors see CREDITS.                             *
  *************************************************************************/
@@ -18,9 +16,9 @@
 
 #define WORKSUITE_IBVLA_VERSION_MAJOR 2
 #define WORKSUITE_IBVLA_VERSION_MINOR 1
-#define WORKSUITE_IBVLA_VERSION_PATCH 2
+#define WORKSUITE_IBVLA_VERSION_PATCH 3
 // If any previous versions available E.g. #define WORKSUITE_ESOURCE_PREVIOUS_VERSIONS "3.8.0,3.8.1"
-#define WORKSUITE_IBVLA_PREVIOUS_VERSIONS "2.0.0,2.1.0,2.1.1"
+#define WORKSUITE_IBVLA_PREVIOUS_VERSIONS "2.0.0,2.1.0,2.1.1,2.1.2"
 
 //
 // Template macros

ibvla/src/common/Context.cc

-// $Id$
-
 /*************************************************************************
  * XDAQ Components for Distributed Data Acquisition                      *
- * Copyright (C) 2000-2014, CERN.                                        *
+ * Copyright (C) 2000-2021, CERN.                                        *
  * All rights reserved.                                                  *
- * Authors: L.Orsini, A.Petrucci, A.Forrest								 *
+ * Authors: L.Orsini, A.Petrucci, A.Forrest, D.Simelevicius              *
  * For the licensing terms see LICENSE.                                  *
  * For the list of contributors see CREDITS.                             *
  *************************************************************************/
@@ -27,19 +25,25 @@ ibvla::Context::~Context ()
 {
 }
 
-ibv_device_attr ibvla::Context::queryDevice () 
+ibv_device_attr ibvla::Context::queryDevice ()
 {
-	errno = 0;
-
 	ibv_device_attr device_attr;
 	memset(&device_attr, 0, sizeof(device_attr));
 
-	int result = ibv_query_device(context_, &device_attr);
+	int ret = ibv_query_device(context_, &device_attr);
 
-	if (result != 0)
+	if (ret != 0)
 	{
 		std::stringstream ss;
-		ss << "ibv_query_device failed, errno = " << strerror(errno);
+		ss << "Failed to query device: ";
+		if (ret == ENOMEM)
+		{
+			ss << "Out of memory, " << strerror(ret) << "(" << ret << ")";
+		}
+		else
+		{
+			ss << "Not understood error, return value = " << ret;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
 	}
 
@@ -48,17 +52,27 @@ ibv_device_attr ibvla::Context::queryDevice ()
 
 ibv_port_attr ibvla::Context::queryPort (uint8_t port_num) 
 {
-	errno = 0;
-
 	struct ibv_port_attr port_attr;
 	memset(&port_attr, 0, sizeof(port_attr));
 
-	int result = ibv_query_port(context_, port_num, &port_attr);
+	int ret = ibv_query_port(context_, port_num, &port_attr);
 
-	if (result != 0)
+	if (ret != 0)
 	{
 		std::stringstream ss;
-		ss << "ibv_query_port failed, errno = " << strerror(errno);
+		ss << "Failed to query port: ";
+		if (ret == EINVAL)
+		{
+			ss << "port_num is invalid, port_num = " << port_num << ", "<< strerror(ret) << "(" << ret << ")";
+		}
+		else if (ret == ENOMEM)
+		{
+			ss << "Out of memory, " << strerror(ret) << "(" << ret << ")";
+		}
+		else
+		{
+			ss << "Not understood error, return value = " << ret;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
 	}
 
@@ -68,16 +82,24 @@ ibv_port_attr ibvla::Context::queryPort (uint8_t port_num)
 ibv_gid ibvla::Context::queryGID (uint8_t port_num, int index) 
 {
 	errno = 0;
-
+	
 	union ibv_gid gid;
 	memset(&gid, 0, sizeof(gid));
 
-	int result = ibv_query_gid(context_, port_num, index, &gid);
+	int ret = ibv_query_gid(context_, port_num, index, &gid);
 
-	if (result != 0)
+	if (ret != 0)
 	{
 		std::stringstream ss;
-		ss << "ibv_query_gid failed, errno = " << strerror(errno);
+		ss << "Failed to query gid: ";
+		if (errno == EMFILE)
+		{
+			ss << "Too many files are opened by this process, " << strerror(errno) << "(" << errno << ")";
+		}
+		else
+		{
+			ss << "Not understood error, return value = " << ret << ", errno = " << errno;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
 	}
 
@@ -90,12 +112,20 @@ uint16_t ibvla::Context::queryPKey (uint8_t port_num, int index)
 
 	uint16_t pkey;
 
-	int result = ibv_query_pkey(context_, port_num, index, &pkey);
+	int ret = ibv_query_pkey(context_, port_num, index, &pkey);
 
-	if (result != 0)
+	if (ret != 0)
 	{
 		std::stringstream ss;
-		ss << "ibv_query_gid failed, errno = " << strerror(errno);
+		ss << "Failed to query P_Key: ";
+		if (errno == EMFILE)
+		{
+			ss << "Too many files are opened by this process, " << strerror(errno) << "(" << errno << ")";
+		}
+		else
+		{
+			ss << "Not understood error, return value = " << ret << ", errno = " << errno;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
 	}
 
@@ -104,8 +134,6 @@ uint16_t ibvla::Context::queryPKey (uint8_t port_num, int index)
 
 ibvla::ProtectionDomain ibvla::Context::allocateProtectionDomain () 
 {
-	errno = 0;
-
 	ibv_pd* pd = ibv_alloc_pd(context_);
 
 	if (pd == 0)
@@ -120,16 +148,26 @@ ibvla::ProtectionDomain ibvla::Context::allocateProtectionDomain ()
 	return ibvla::ProtectionDomain(*this, pd);
 }
 
-void ibvla::Context::deallocateProtectionDomain (ProtectionDomain & pd) 
+void ibvla::Context::deallocateProtectionDomain (ProtectionDomain & pd)
 {
-	errno = 0;
+	int ret = ibv_dealloc_pd(pd.pd_);
 
-	int i = ibv_dealloc_pd(pd.pd_);
-
-	if (i != 0)
+	if (ret != 0)
 	{
 		std::stringstream ss;
-		ss << "Failed to deallocate protection domain for context device '" << getDeviceName() << "', errno = " << strerror(errno);
+		ss << "Failed to deallocate protection domain for context device '" << getDeviceName() << "': ";
+		if (ret == EINVAL)
+		{
+			ss << "PD context is invalid, " << strerror(ret) << "(" << ret << ")";
+		}
+		else if (ret == EBUSY)
+		{
+			ss << "RDMA resources are still associated with this PD, " << strerror(ret) << "(" << ret << ")";
+		}
+		else
+		{
+			ss << "Not understood error, return value = " << ret;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
 	}
 
@@ -150,7 +188,19 @@ ibvla::CompletionQueue ibvla::Context::createCompletionQueue (int size, void* us
 	if (cq == 0)
 	{
 		std::stringstream ss;
-		ss << "Failed to create completion queue for context device '" << getDeviceName() << "', errno = " << strerror(errno);
+		ss << "Failed to create completion queue for context device '" << getDeviceName() << ": ";
+		if (errno == EINVAL)
+		{
+			ss << "Invalid cqe, channel or comp_vector, " << strerror(errno) << "(" << errno << ")";
+		}
+		else if (errno == ENOMEM)
+		{
+			ss << "Not enough resources to complete this operation, " << strerror(errno) << "(" << errno << ")";
+		}
+		else
+		{
+			ss << "Not understood error, errno = " << errno;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
 	}
 
@@ -166,14 +216,20 @@ ibvla::CompletionQueue ibvla::Context::createCompletionQueue (int size, void* us
 
 void ibvla::Context::destroyCompletionQueue (ibvla::CompletionQueue& cq) 
 {
-	errno = 0;
-
-	int i = ibv_destroy_cq(cq.cq_);
+	int ret = ibv_destroy_cq(cq.cq_);
 
-	if (i != 0)
+	if (ret != 0)
 	{
 		std::stringstream ss;
-		ss << "Failed to destroy completion queue for device '" << getDeviceName() << "', errno = " << strerror(errno);
+		ss << "Failed to destroy completion queue for device '" << getDeviceName() << ": ";
+		if (ret == EBUSY)
+		{
+			ss << "One or more Work Queues is still associated with the CQ, " << strerror(ret) << "(" << ret << ")";
+		}
+		else
+		{
+			ss << "Not understood error, return value = " << ret;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
 	}
 }
@@ -186,12 +242,12 @@ std::string ibvla::Context::getDeviceName ()
 /*
  * Performs a blocking call on the asynchronous event queue, and ACK's the event
  */
-ibv_async_event ibvla::Context::waitAsyncEvent () 
+ibv_async_event ibvla::Context::waitAsyncEvent ()
 {
 	ibv_async_event e;
-	int i = ibv_get_async_event(context_, &e);
+	int ret = ibv_get_async_event(context_, &e);
 
-	if (i != 0)
+	if (ret != 0)
 	{
 		std::stringstream ss;
 		ss << "Failed to get async event for device '" << getDeviceName() << "'";

ibvla/src/common/ProtectionDomain.cc

-// $Id$
-
 /*************************************************************************
  * XDAQ Components for Distributed Data Acquisition                      *
- * Copyright (C) 2000-2014, CERN.                                        *
+ * Copyright (C) 2000-2021, CERN.                                        *
  * All rights reserved.                                                  *
- * Authors: L.Orsini, A.Petrucci, A.Forrest								 *
+ * Authors: L.Orsini, A.Petrucci, A.Forrest, D.Simelevicius              *
  * For the licensing terms see LICENSE.                                  *
  * For the list of contributors see CREDITS.                             *
  *************************************************************************/
@@ -38,22 +36,45 @@ ibvla::MemoryRegion ibvla::ProtectionDomain::registerMemoryRegion (void *addr, s
 	if (mr == 0)
 	{
 		std::stringstream ss;
-		ss << "Failed to register memory region for protection domain '" << getContext().getDeviceName() << "', " << strerror(errno);
+		ss << "Failed to register memory region for protection domain '" << getContext().getDeviceName() << ": ";
+		if (errno == EINVAL)
+		{
+			ss << "Invalid access value, " << strerror(errno) << "(" << errno << ")";
+		}
+		else if (errno == ENOMEM)
+		{
+			ss << "Not enough resources (either in operating system or in RDMA device) to complete this operation, " << strerror(errno) << "(" << errno << ")";
+		}
+		else
+		{
+			ss << "Not understood error, errno = " << errno;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
 	}
 
 	return ibvla::MemoryRegion(*this, mr);
 }
 
-void ibvla::ProtectionDomain::deregisterMemoryRegion (MemoryRegion & mr) 
+void ibvla::ProtectionDomain::deregisterMemoryRegion (MemoryRegion & mr)
 {
-	errno = 0;
 	int ret = ibv_dereg_mr(mr.mr_);
 
 	if (ret != 0)
 	{
 		std::stringstream ss;
-		ss << "Failed to de-register memory region, " << strerror(errno);
+		ss << "Failed to de-register memory region: ";
+		if (ret == EINVAL)
+		{
+			ss << "MR context is invalid, " << strerror(ret) << "(" << ret << ")";
+		}
+		else if (ret == EBUSY)
+		{
+			ss << "MWs are still associated with this MR, " << strerror(ret) << "(" << ret << ")";
+		}
+		else
+		{
+			ss << "Not understood error, return value = " << ret;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
 	}
 }
@@ -87,7 +108,27 @@ ibvla::QueuePair ibvla::ProtectionDomain::createQueuePair (ibvla::CompletionQueu
 	if (qp == 0)
 	{
 		std::stringstream ss;
-		ss << "Failed to create queue pair for device '" << getContext().getDeviceName() << "', errno = " << strerror(errno);
+		ss << "Failed to create queue pair for device '" << getContext().getDeviceName() << "': ";
+		if (errno == EINVAL)
+		{
+			ss << "Invalid pd, send_cq, recv_cq, srq or invalid value provided in max_send_wr, max_recv_wr, max_send_sge, max_recv_sge or in max_inline_data, " << strerror(errno) << "(" << errno << ")";
+		}
+		else if (errno == ENOMEM)
+		{
+			ss << "Not enough resources to complete this operation, " << strerror(errno) << "(" << errno << ")";
+		}
+		else if (errno == ENOSYS)
+		{
+			ss << "QP with this Transport Service Type isn't supported by this RDMA device, " << strerror(errno) << "(" << errno << ")";
+		}
+		else if (errno == EPERM)
+		{
+			ss << "Not enough permissions to create a QP with this Transport Service Type, " << strerror(errno) << "(" << errno << ")";
+		}
+		else
+		{
+			ss << "Not understood error, errno = " << errno;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
 	}
 
@@ -96,15 +137,20 @@ ibvla::QueuePair ibvla::ProtectionDomain::createQueuePair (ibvla::CompletionQueu
 
 void ibvla::ProtectionDomain::destroyQueuePair (ibvla::QueuePair& qp) 
 {
-	int ret = 0;
-	errno = 0;
-
-	ret = ibv_destroy_qp(qp.qp_);
+	int ret = ibv_destroy_qp(qp.qp_);
 
 	if (ret != 0)
 	{
 		std::stringstream ss;
-		ss << "Failed to destroy queue pair for device '" << getContext().getDeviceName() << "', errno = " << strerror(errno);
+		ss << "Failed to destroy queue pair for device '" << getContext().getDeviceName() << "'";
+		if (ret == EBUSY)
+		{
+			ss << "QP is still attached to one or more multicast groups, " << strerror(ret) << "(" << ret << ")";
+		}
+		else
+		{
+			ss << "Not understood error, return value = " << ret;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
 	}
 }

ibvla/src/common/QueuePair.cc

-// $Id$
-
 /*************************************************************************
  * XDAQ Components for Distributed Data Acquisition                      *
- * Copyright (C) 2000-2014, CERN.                                        *
+ * Copyright (C) 2000-2021, CERN.                                        *
  * All rights reserved.                                                  *
- * Authors: L.Orsini, A.Petrucci, A.Forrest								 *
+ * Authors: L.Orsini, A.Petrucci, A.Forrest, D.Simelevicius              *
  * For the licensing terms see LICENSE.                                  *
  * For the list of contributors see CREDITS.                             *
  *************************************************************************/
@@ -28,12 +26,23 @@ ibvla::QueuePair::~QueuePair ()
 
 void ibvla::QueuePair::modify (ibv_qp_attr &attr, int attr_mask) 
 {
-	errno = 0;
 	int ret = ibv_modify_qp(qp_, &attr, attr_mask);
 	if (ret != 0)
 	{
 		std::stringstream ss;
-		ss << "Failed to modify queue pair, errno = " << strerror(errno);
+		ss << "Failed to modify queue pair: ";
+		if (ret == EINVAL)
+		{
+			ss << "Invalid value provided in attr or in attr_mask, " << strerror(ret) << "(" << ret << ")";
+		}
+		else if (ret == ENOMEM)
+		{
+			ss << "Not enough resources to complete this operation, " << strerror(ret) << "(" << ret << ")";
+		}
+		else
+		{
+			ss << "Not understood error, return value = " << ret;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
 	}
 }
@@ -45,7 +54,6 @@ void ibvla::QueuePair::modify (ibv_qp_attr &attr, int attr_mask)
  */
 void ibvla::QueuePair::query (int attr_mask, ibv_qp_attr &attr, ibv_qp_init_attr &init_attr) 
 {
-	errno = 0;
 	memset(&attr, 0, sizeof(ibv_qp_attr));
 	memset(&init_attr, 0, sizeof(ibv_qp_init_attr));
 
@@ -54,38 +62,44 @@ void ibvla::QueuePair::query (int attr_mask, ibv_qp_attr &attr, ibv_qp_init_attr
 	if (ret != 0)
 	{
 		std::stringstream ss;
-		ss << "Failed to query queue pair, errno = " << strerror(errno);
+		ss << "Failed to query queue pair: ";
+		if (ret == ENOMEM)
+		{
+			ss << "Not enough resources to complete this operation, " << strerror(ret) << "(" << ret << ")";
+		}
+		else
+		{
+			ss << "Not understood error, return value = " << ret;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
-		//throw error("ibv_query_qp", result);
 	}
 }
 
 void ibvla::QueuePair::postSend (ibv_send_wr &wr) 
 {
-	errno = 0;
 	ibv_send_wr *bad_wr;
 
 	int ret = ibv_post_send(qp_, &wr, &bad_wr);
 	if (ret != 0)
 	{
 		std::stringstream ss;
-		// warning, errno is not being set correctly by this call
-		if (errno == EINVAL)
+		ss << "Failed to post send queue pair: ";
+		if (ret == EINVAL)
 		{
-			ss << "Failed to post send queue pair with invalid value in WR, " << strerror(errno) << "(" << errno << ")";
+			ss << "Invalid value provided in wr, " << strerror(ret) << "(" << ret << ")";
 		}
-		else if (errno == ENOMEM)
+		else if (ret == ENOMEM)
 		{
-			ss << "Failed to post send queue pair, not enough resources, " << strerror(errno) << "(" << errno << ")";
+			ss << "Send Queue is full or not enough resources to complete this operation, " << strerror(ret) << "(" << ret << ")";
 			XCEPT_RAISE(ibvla::exception::QueueFull, ss.str());
 		}
-		else if (errno == EFAULT)
+		else if (ret == EFAULT)
 		{
-			ss << "Failed to post send queue pair, invalid value in QP, " << strerror(errno) << "(" << errno << ")";
+			ss << "Invalid value provided in qp, " << strerror(ret) << "(" << ret << ")";
 		}
 		else
 		{
-			ss << "Not understood error, (errno=" << errno << ", return value=" << ret << ")";
+			ss << "Not understood error, return value = " << ret;
 		}
 		XCEPT_RAISE(ibvla::exception::InternalError, ss.str());
 	}
@@ -93,14 +107,30 @@ void ibvla::QueuePair::postSend (ibv_send_wr &wr)
 
 void ibvla::QueuePair::postRecv (ibv_recv_wr &wr) 
 {
-	errno = 0;
 	ibv_recv_wr *bad_wr;
 
 	int ret = ibv_post_recv(qp_, &wr, &bad_wr);
 	if (ret != 0)
 	{
 		std::stringstream ss;
-		ss << "Failed to post recv on queue pair, errno = " << strerror(errno);
+		ss << "Failed to post recv on queue pair, ";
+		if (ret == EINVAL)
+		{
+			ss << "Invalid value provided in wr, " << strerror(ret) << "(" << ret << ")";
+		}
+		else if (ret == ENOMEM)
+		{
+			ss << "Receive Queue is full or not enough resources to complete this operation, " << strerror(ret) << "(" << ret << ")";
+			XCEPT_RAISE(ibvla::exception::QueueFull, ss.str());
+		}
+		else if (ret == EFAULT)
+		{
+			ss << "Invalid value provided in qp, " << strerror(ret) << "(" << ret << ")";
+		}
+		else
+		{
+			ss << "Not understood error, return value = " << ret;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
 	}
 }

ibvla/src/common/Utils.cc

@@ -41,7 +41,23 @@ std::list<ibvla::Device> ibvla::getDeviceList ()
 	if (devlist == 0)
 	{
 		std::stringstream ss;
-		ss << "Failure to get device list, errno = " << ::strerror(errno);
+		ss << "Failed to get device list: ";
+		if (errno == EPERM)
+		{
+			ss << "Permission denied, " << strerror(errno) << "(" << errno << ")";
+		}
+		else if (errno == ENOMEM)
+		{
+			ss << "Insufficient memory to complete the operation, " << strerror(errno) << "(" << errno << ")";
+		}
+		else if (errno == ENOSYS)
+		{
+			ss << "No kernel support for RDMA, " << strerror(errno) << "(" << errno << ")";
+		}
+		else
+		{
+			ss << "Not understood error, errno = " << errno;
+		}
 		XCEPT_RAISE(ibvla::exception::Exception, ss.str());
 	}