Commit beb9ad06 authored by Luciano Orsini's avatar Luciano Orsini
Browse files

references #200: preparing skeleton vpi

parent b29e021f
......@@ -78,9 +78,25 @@ DynamicLibrary=ibvla
#ExternalObjects=/usr/lib64/libibverbs.a
TestExecutables= standalone.cc
TestLibraries= config \
ibvla \
ibverbs
TestLibraries= crypt \
config \
peer \
toolbox \
asyncresolv \
log4cplus \
xerces-c \
cgicc \
xcept \
xoap \
xdata \
xgi \
logudpappender \
logxmlappender \
mimetic \
uuid \
tirpc \
elf \
ibvla ibverbs
include $(XDAQ_ROOT)/$(BUILD_SUPPORT)/Makefile.rules
include $(XDAQ_ROOT)/$(BUILD_SUPPORT)/mfRPM.rules
......@@ -36,6 +36,7 @@ namespace ibvla
Context& getContext ();
size_t size () ;
void resize (int cqe) ;
unsigned poll (int num_entries, ibv_wc *wc) ;
......
......@@ -26,6 +26,8 @@ namespace ibvla
public:
ConnectionRequest (int lid, int psn, int qpn, union ibv_gid gid, Acceptor * acceptor, int sockfd, const std::string & ip, const std::string & hostname);
ConnectionRequest (int lid, int psn, int qpn, union ibv_gid gid, Acceptor * acceptor, int sockfd, const std::string & ip, const std::string & hostname, const std::string & network);
int remote_lid_;
int remote_psn_;
......@@ -38,6 +40,7 @@ namespace ibvla
std::string ip_;
std::string hostname_;
std::string network_;
};
}
......
......@@ -104,7 +104,7 @@ bool ibvla::Acceptor::process (toolbox::task::WorkLoop* wl)
//std::cout << "Received connection info from remote address " << ip << ", lid='" << lid << "', qpn='" << qpn << "', psn='" << psn << "', msg='" << msg << std::endl;
listener_->connectionRequest(ibvla::ConnectionRequest(lid, psn, qpn, gid, this, connfd, ip, remoteHostName));
listener_->connectionRequest(ibvla::ConnectionRequest(lid, psn, qpn, gid, this, connfd, ip, remoteHostName, name_));
return true;
}
......
......@@ -29,6 +29,11 @@ ibvla::Context& ibvla::CompletionQueue::getContext ()
return context_;
}
size_t ibvla::CompletionQueue::size ()
{
return cq_->cqe;
}
void ibvla::CompletionQueue::resize (int size)
{
int result = ibv_resize_cq(cq_, size);
......
......@@ -14,6 +14,12 @@
#include "ibvla/Acceptor.h"
ibvla::ConnectionRequest::ConnectionRequest (int lid, int psn, int qpn, union ibv_gid gid, ibvla::Acceptor * acceptor, int sockfd, const std::string & ip, const std::string & hostname)
: remote_lid_(lid), remote_psn_(psn), remote_qpn_(qpn), remote_gid_(gid), acceptor_(acceptor), sockfd_(sockfd), ip_(ip), hostname_(hostname)
: remote_lid_(lid), remote_psn_(psn), remote_qpn_(qpn), remote_gid_(gid), acceptor_(acceptor), sockfd_(sockfd), ip_(ip), hostname_(hostname), network_("unknown")
{
}
ibvla::ConnectionRequest::ConnectionRequest (int lid, int psn, int qpn, union ibv_gid gid, ibvla::Acceptor * acceptor, int sockfd, const std::string & ip, const std::string & hostname, const std::string & network)
: remote_lid_(lid), remote_psn_(psn), remote_qpn_(qpn), remote_gid_(gid), acceptor_(acceptor), sockfd_(sockfd), ip_(ip), hostname_(hostname), network_(network)
{
}
#include <arpa/inet.h>
#include <string.h>
#include <stdio.h>
#include <errno.h>
#include <getopt.h>
#include <stdlib.h>
#include <unistd.h>
#include <rdma/rdma_cma.h>
struct Args {
char *remote_addr;
int is_server;
int port;
};
struct Args args;
int main(int argc, char *argv[])
{
struct rdma_event_channel *evch;
struct rdma_cm_id *server_id;
struct rdma_cm_id *client_id;
struct rdma_cm_event *event;
struct rdma_conn_param conn_param;
struct ibv_pd *pd;
struct ibv_cq *cq;
struct ibv_mr *mr;
struct ibv_send_wr snd_wr;
struct ibv_recv_wr rcv_wr;
struct ibv_sge sge;
struct ibv_wc wc;
struct ibv_qp_init_attr attr = {
.cap = {
.max_send_wr = 32,
.max_recv_wr = 32,
.max_send_sge = 1,
.max_recv_sge = 1,
.max_inline_data = 64
},
.qp_type = IBV_QPT_RC
};
char msg[256] = "Hello World";
int msg_len = strlen(msg) + 1;
struct sockaddr_in sin;
args.remote_addr = "0.0.0.0";
args.is_server = 1;
args.port = 21234;
/* Parameter parsing. */
while (1) {
int c;
c = getopt(argc, argv, "c:p:");
if (c == -1)
break;
switch (c) {
case 'c':
args.is_server = 0;
args.remote_addr = optarg;
break;
case 'p':
args.port = strtol(optarg, NULL, 0);
break;
default:
perror("Invalid option");
exit(-1);
};
}
if (args.is_server) {
if (!(evch = rdma_create_event_channel())) {
perror("rdma_create_event_channel");
exit(-1);
}
if (rdma_create_id(evch, &server_id, NULL, RDMA_PS_TCP)) {
perror("rdma_create_id");
exit(-1);
}
sin.sin_family = AF_INET;
sin.sin_port = htons(args.port);
sin.sin_addr.s_addr = htonl(INADDR_ANY);
if (rdma_bind_addr(server_id, (struct sockaddr *)&sin)) {
perror("rdma_bind_addr");
exit(-1);
}
if (rdma_listen(server_id, 6)) {
perror("rdma_listen");
exit(-1);
}
if (rdma_get_cm_event(evch, &event)
|| event->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
perror("rdma_get_cm_event");
exit(-1);
}
client_id = (struct rdma_cm_id *)event->id;
if (!(pd = ibv_alloc_pd(client_id->verbs))) {
perror("ibv_alloc_pd");
exit(-1);
}
if (!(mr = ibv_reg_mr(pd, msg, 256,
IBV_ACCESS_REMOTE_WRITE |
IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_READ))) {
perror("ibv_reg_mr");
exit(-1);
}
if (!(cq = ibv_create_cq(client_id->verbs, 32, 0, 0, 0))) {
perror("ibv_create_cq");
exit(-1);
}
attr.send_cq = attr.recv_cq = cq;
if (rdma_create_qp(client_id, pd, &attr)) {
perror("rdma_create_qp");
exit(-1);
}
memset(&conn_param, 0, sizeof conn_param);
if (rdma_accept(client_id, &conn_param)) {
perror("rdma_accept");
exit(-1);
}
rdma_ack_cm_event(event);
if (rdma_get_cm_event(evch, &event)
|| event->event != RDMA_CM_EVENT_ESTABLISHED) {
perror("rdma_get_cm_event");
exit(-1);
}
rdma_ack_cm_event(event);
sge.addr = (uint64_t)msg;
sge.length = msg_len;
sge.lkey = mr->lkey;
snd_wr.sg_list = &sge;
snd_wr.num_sge = 1;
snd_wr.opcode = IBV_WR_SEND;
snd_wr.send_flags = IBV_SEND_SIGNALED;
snd_wr.next = NULL;
if (ibv_post_send(client_id->qp, &snd_wr, NULL)) {
perror("ibv_post_send");
exit(-1);
}
while (!ibv_poll_cq(cq, 1, &wc))
;
if (wc.status != IBV_WC_SUCCESS) {
perror("ibv_poll_cq");
exit(-1);
}
}
else {
if (!(evch = rdma_create_event_channel())) {
perror("rdma_create_event_channel");
exit(-1);
}
if (rdma_create_id(evch, &client_id, NULL, RDMA_PS_TCP)) {
perror("rdma_create_id");
exit(-1);
}
sin.sin_family = AF_INET;
sin.sin_port = htons(args.port);
sin.sin_addr.s_addr = inet_addr(args.remote_addr);
if (rdma_resolve_addr
(client_id, NULL, (struct sockaddr *)&sin, 2000)) {
perror("rdma_resolve_addr");
exit(-1);
}
if (rdma_get_cm_event(evch, &event)
|| event->event != RDMA_CM_EVENT_ADDR_RESOLVED) {
perror("rdma_get_cm_event");
exit(-1);
}
rdma_ack_cm_event(event);
if (rdma_resolve_route(client_id, 2000)) {
perror("rdma_resolve_route");
exit(-1);
}
if (rdma_get_cm_event(evch, &event)
|| event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) {
perror("rdma_get_cm_event");
exit(-1);
}
rdma_ack_cm_event(event);
if (!(pd = ibv_alloc_pd(client_id->verbs))) {
perror("ibv_alloc_pd");
exit(-1);
}
if (!(mr = ibv_reg_mr(pd, msg, 256,
IBV_ACCESS_REMOTE_WRITE |
IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_READ))) {
perror("ibv_reg_mr");
exit(-1);
}
if (!(cq = ibv_create_cq(client_id->verbs, 32, 0, 0, 0))) {
perror("ibv_create_cq");
exit(-1);
}
attr.send_cq = attr.recv_cq = cq;
if (rdma_create_qp(client_id, pd, &attr)) {
perror("rdma_create_qp");
exit(-1);
}
sge.addr = (uint64_t)msg;
sge.length = msg_len;
sge.lkey = mr->lkey;
rcv_wr.sg_list = &sge;
rcv_wr.num_sge = 1;
rcv_wr.next = NULL;
if (ibv_post_recv(client_id->qp, &rcv_wr, NULL)) {
perror("ibv_post_recv");
exit(-1);
}
memset(&conn_param, 0, sizeof conn_param);
if (rdma_connect(client_id, &conn_param)) {
perror("rdma_connect");
exit(-1);
}
if (rdma_get_cm_event(evch, &event)
|| event->event != RDMA_CM_EVENT_ESTABLISHED) {
perror("rdma_get_cm_event");
exit(-1);
}
rdma_ack_cm_event(event);
while (!ibv_poll_cq(cq, 1, &wc)) ;
if (wc.status != IBV_WC_SUCCESS) {
perror("ibv_poll_cq");
exit(-1);
}
fprintf(stdout, "Received %s \n", msg);
}
fprintf(stdout, "Done \n");
return 0;
}
//#include <stdio.h>
//#include <infiniband/verbs.h>
#include <stdio.h>
#include <infiniband/verbs.h>
#include "ibvla/Utils.h"
//#include <sys/types.h>
//#include <netinet/in.h>
//#include <inttypes.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <inttypes.h>
//#include <byteswap.h>
//#include "xcept/tools.h"
//#include "ibvla/ProtectionDomain.h"
//#include "ibvla/CompletionQueue.h"
//#include "ibvla/QueuePair.h"
//#include "ibvla/Device.h"
#include <byteswap.h>
#include "xcept/tools.h"
#include "ibvla/ProtectionDomain.h"
#include "ibvla/CompletionQueue.h"
#include "ibvla/QueuePair.h"
#include "ibvla/Device.h"
//#include <errno.h>
//#include <string.h>
#include <errno.h>
#include <string.h>
#include <sstream>
void test ()
{
//////////
/*
try
{
......@@ -54,6 +53,7 @@ void test ()
std::cout << "Going to query ports" << std::endl;
/*
for (int i = 1; i <= num_ports; i++)
{
// for each possible port
......@@ -72,6 +72,7 @@ void test ()
//std::cout << "TEST : Port lid = '0x" << std::hex << ntohs(p_att.lid) << std::dec << "'" << std::endl;
}
*/
std::cout << "TEST : allocate protection domain" << std::endl;
ibvla::ProtectionDomain pd = ibContext.allocateProtectionDomain();
......@@ -80,7 +81,7 @@ void test ()
ibvla::CompletionQueue cq = ibContext.createCompletionQueue(256, 0, 0);
std::cout << "TEST : create queue pair" << std::endl;
ibvla::QueuePair qp = pd.createQueuePair(cq, cq, 8, 8);
ibvla::QueuePair qp = pd.createQueuePair(cq, cq, 8, 8, 0);
ibv_qp_attr qpattr;
memset(&qpattr, 0, sizeof(qpattr));
......@@ -101,7 +102,6 @@ void test ()
//////////
std::cout << "TEST : Client ready" << std::endl;
*/
}
void runOurTest ()
......@@ -141,8 +141,8 @@ void runOurTest ()
//goto out;
}
std::cout << "The next line SHOULD say where in memory the function for calling query_port is" << std::endl;
std::cout << ctx->ops.query_port << std::endl;
//std::cout << "The next line SHOULD say where in memory the function for calling query_port is" << std::endl;
//std::cout << ctx->ops.query_port << std::endl;
rc = ibv_query_device(ctx, &device_attr);
if (rc)
......@@ -165,13 +165,13 @@ void runOurTest ()
//goto out_device;
}
*/
port_attr.link_layer = IBV_LINK_LAYER_UNSPECIFIED;
port_attr.pad = 0;
//port_attr.link_layer = IBV_LINK_LAYER_UNSPECIFIED;
//port_attr.pad = 0;
std::cout << "doign it the ib way" << std::endl;
ctx->ops.query_port(ctx, j, &port_attr);
//ctx->ops.query_port(ctx, j, &port_attr);
printf("RDMA device %s, port %d state %s active\n", ibv_get_device_name(ctx->device), j, (port_attr.state == IBV_PORT_ACTIVE) ? "is" : "isn't");
//printf("RDMA device %s, port %d state %s active\n", ibv_get_device_name(ctx->device), j, (port_attr.state == IBV_PORT_ACTIVE) ? "is" : "isn't");
}
rc = ibv_close_device(ctx);
......@@ -187,8 +187,8 @@ void runOurTest ()
void test2 ()
{
//runOurTest();
ibvla::runTest();
runOurTest();
//ibvla::runTest();
std::cout << "TEST : END" << std::endl;
}
......@@ -196,7 +196,7 @@ void test2 ()
int main (void)
{
test2();
test();
return 0;
}
# $Id$
#########################################################################
# XDAQ Components for Distributed Data Acquisition #
# Copyright (C) 2000-2020, CERN. #
# All rights reserved. #
# Authors: L. Orsini and D. Simelevicius #
# #
# For the licensing terms see LICENSE. #
# For the list of contributors see CREDITS. #
#########################################################################
##
#
# Project level Makefile
#
##
BUILD_HOME:=$(shell pwd)
ifndef PACKAGES
PACKAGES= \
extern/oracle \
extern/dim \
extern/smi \
extern/yui \
extern/caen/a2818 \
extern/caen/a3818 \
extern/caen/CAENUSBdrvB \
extern/caen/caenvmelib \
extern/caen/caencomm \
extern/caen/caenbridgeupgrade \
extern/caen/caenupgrader \
xdaq2rc \
interface/evb \
interface/shared \
interface/bril \
evb \
jobcontrol \
xpci/drv \
xpci \
hal/generic \
hal/utilities \
hal/busAdapter/dummy \
hal/busAdapter/caen \
hal/busAdapter/pci \
hal/PyHAL \
pheaps/drv/cmem_rcc \
pheaps \
jal/jtagChain \
jal/jtagController \
jal/jtagSVFSequencer \
ttc/utils \
ttc/monitoring \
ttc/ltc \
ttc/ttcci \
tts/ttsbase \
tts/atts \
tts/cpcibase \
tts/fmmtd \
tts/ipcutils \
tts/fmm \
tts/fmmcontroller \
tts/fmmdbi \
tts/fmmtester \
d2s/utils \
d2s/fedemulator \
d2s/gtpe \
d2s/gtpecontroller \
d2s/firmwareloader \
fedstreamer \
fedkit \
ferol \
ferol40 \
psx/sapi \
psx/mapi \
psx \
psx/watchdog \
sentinel \
sentinel/utils \
sentinel/probe \
sentinel/sentineld \
sentinel/tester \
sentinel/bridge2g \
sentinel/spotlight2g \
sentinel/spotlightocci \
tstore/utils \
tstore/client \
tstore \
tstore/api \
xmas/admin \
xmas/heartbeat \
xmas/heartbeat/probe \
xmas/heartbeat/heartbeatd \
xmas/utils \
xmas/sensord \
xmas/probe \
xmas/tester \
xmas/bridge2g \
xmas/slash2g \
xmas/store2g \
xmas/smarthub \
ibvla \
pt/ibv \
gevb2g \
elastic/api \
elastic/timestream \
amc13controller \
dipbridge \
oms
endif
export PACKAGES
BUILD_SUPPORT=build
export BUILD_SUPPORT
PROJECT_NAME=worksuite
export PROJECT_NAME
include $(XDAQ_ROOT)/$(BUILD_SUPPORT)/mfAutoconf.rules
Project=$(PROJECT_NAME)
include $(XDAQ_ROOT)/$(BUILD_SUPPORT)/mfDefs.$(XDAQ_OS)
Packages=$(PACKAGES)
include $(XDAQ_ROOT)/$(BUILD_SUPPORT)/Makefile.rules
include $(XDAQ_ROOT)/$(BUILD_SUPPORT)/mfRPM.rules
// $Id$
/*************************************************************************
* XDAQ Components for Distributed Data Acquisition *
* Copyright (C) 2000-2020, CERN. *
* All rights reserved. *
* Authors: L.Orsini,D. Siemelevicius *
* For the licensing terms see LICENSE. *
* For the list of contributors see CREDITS. *
*************************************************************************/
#ifndef _pt_vpi_Address_h