--- a/components/open-fabrics/librdmacm/patches/base.patch Fri Jul 11 14:21:49 2014 -0700
+++ b/components/open-fabrics/librdmacm/patches/base.patch Mon Jul 14 18:21:18 2014 -0700
@@ -1,6 +1,6 @@
diff -r -u /tmp/librdmacm-1.0.14.1/configure librdmacm-1.0.14.1/configure
--- /tmp/librdmacm-1.0.14.1/configure Tue Feb 15 17:12:14 2011
-+++ librdmacm-1.0.14.1/configure Thu Feb 24 08:39:24 2011
++++ librdmacm-1.0.14.1/configure Wed Apr 30 10:52:05 2014
@@ -7625,6 +7625,7 @@
esac ;;
esac
@@ -11,7 +11,7 @@
sunos4*)
diff -r -u /tmp/librdmacm-1.0.14.1/Makefile.in librdmacm-1.0.14.1/Makefile.in
--- /tmp/librdmacm-1.0.14.1/Makefile.in Tue Feb 15 17:12:13 2011
-+++ librdmacm-1.0.14.1/Makefile.in Mon Mar 28 16:49:13 2011
++++ librdmacm-1.0.14.1/Makefile.in Wed Apr 30 10:52:05 2014
@@ -69,7 +69,7 @@
"$(DESTDIR)$(man1dir)" "$(DESTDIR)$(man3dir)" \
"$(DESTDIR)$(man7dir)" "$(DESTDIR)$(infinibandincludedir)" \
@@ -32,7 +32,7 @@
examples_mckey_OBJECTS = $(am_examples_mckey_OBJECTS)
diff -r -u /tmp/librdmacm-1.0.14.1/src/cma.h librdmacm-1.0.14.1/src/cma.h
--- /tmp/librdmacm-1.0.14.1/src/cma.h Mon Oct 4 17:00:18 2010
-+++ librdmacm-1.0.14.1/src/cma.h Fri Feb 11 04:08:57 2011
++++ librdmacm-1.0.14.1/src/cma.h Tue May 6 11:32:19 2014
@@ -40,8 +40,10 @@
#include <stdlib.h>
@@ -59,9 +59,18 @@
#define min(a, b) (a < b ? a : b)
static inline int ERR(int err)
+@@ -74,7 +68,7 @@
+ return -1;
+ }
+
+-int ucma_init();
++int ucma_init(void);
+ extern int af_ib_support;
+
+ #define RAI_ROUTEONLY 0x01000000
diff -r -u /tmp/librdmacm-1.0.14.1/src/cma.c librdmacm-1.0.14.1/src/cma.c
--- /tmp/librdmacm-1.0.14.1/src/cma.c Fri Dec 10 12:05:34 2010
-+++ librdmacm-1.0.14.1/src/cma.c Mon Mar 28 16:44:55 2011
++++ librdmacm-1.0.14.1/src/cma.c Tue May 6 11:30:54 2014
@@ -46,12 +46,18 @@
#include <poll.h>
#include <unistd.h>
@@ -81,7 +90,217 @@
#include <infiniband/driver.h>
#include <infiniband/marshall.h>
#include <rdma/rdma_cma.h>
-@@ -354,9 +360,18 @@
+@@ -100,6 +106,8 @@
+ struct ibv_pd *pd;
+ uint64_t guid;
+ int port_cnt;
++ int refcnt;
++ int max_qpsize;
+ uint8_t max_initiator_depth;
+ uint8_t max_responder_resources;
+ };
+@@ -143,6 +151,7 @@
+
+ static struct cma_device *cma_dev_array;
+ static int cma_dev_cnt;
++static int cma_init_cnt;
+ static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
+ static int abi_ver = RDMA_USER_CM_MAX_ABI_VERSION;
+ int af_ib_support;
+@@ -156,12 +165,16 @@
+
+ if (cma_dev_cnt) {
+ while (cma_dev_cnt--) {
+- ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
++ if (!cma_dev_array[cma_dev_cnt].verbs)
++ continue;
++
++ if (cma_dev_array[cma_dev_cnt].refcnt)
++ ibv_dealloc_pd(cma_dev_array[cma_dev_cnt].pd);
+ ibv_close_device(cma_dev_array[cma_dev_cnt].verbs);
++ cma_init_cnt--;
+ }
+
+ free(cma_dev_array);
+- cma_dev_cnt = 0;
+ }
+ }
+
+@@ -228,9 +241,7 @@
+ int ucma_init(void)
+ {
+ struct ibv_device **dev_list = NULL;
+- struct cma_device *cma_dev;
+- struct ibv_device_attr attr;
+- int i, ret, dev_cnt, ib;
++ int i, ret, dev_cnt;
+
+ /* Quick check without lock to see if we're already initialized */
+ if (cma_dev_cnt)
+@@ -253,46 +264,20 @@
+ goto err1;
+ }
+
+- cma_dev_array = malloc(sizeof *cma_dev * dev_cnt);
++ if (!dev_cnt) {
++ printf("CMA: no RDMA devices found\n");
++ ret = ERR(ENODEV);
++ goto err2;
++ }
++ cma_dev_array = calloc(dev_cnt, sizeof *cma_dev_array);
+ if (!cma_dev_array) {
+ ret = ERR(ENOMEM);
+ goto err2;
+ }
+
+- for (i = 0, ib = 0; dev_list[i];) {
+- cma_dev = &cma_dev_array[i];
++ for (i = 0; dev_list[i]; i++)
++ cma_dev_array[i].guid = ibv_get_device_guid(dev_list[i]);
+
+- cma_dev->guid = ibv_get_device_guid(dev_list[i]);
+- cma_dev->verbs = ibv_open_device(dev_list[i]);
+- if (!cma_dev->verbs) {
+- printf("CMA: unable to open RDMA device\n");
+- ret = ERR(ENODEV);
+- goto err3;
+- }
+-
+- cma_dev->pd = ibv_alloc_pd(cma_dev->verbs);
+- if (!cma_dev->pd) {
+- ibv_close_device(cma_dev->verbs);
+- ret = ERR(ENOMEM);
+- goto err3;
+- }
+-
+- i++;
+- ret = ibv_query_device(cma_dev->verbs, &attr);
+- if (ret) {
+- printf("CMA: unable to query RDMA device\n");
+- ret = ERR(ret);
+- goto err3;
+- }
+-
+- cma_dev->port_cnt = attr.phys_port_cnt;
+- cma_dev->max_initiator_depth = (uint8_t) attr.max_qp_init_rd_atom;
+- cma_dev->max_responder_resources = (uint8_t) attr.max_qp_rd_atom;
+- ib += (cma_dev->verbs->device->transport_type == IBV_TRANSPORT_IB);
+- }
+-
+- if (ib)
+- ucma_ib_init();
+ cma_dev_cnt = dev_cnt;
+ ucma_set_af_ib_support();
+ pthread_mutex_unlock(&mut);
+@@ -299,12 +284,6 @@
+ ibv_free_device_list(dev_list);
+ return 0;
+
+-err3:
+- while (i--) {
+- ibv_dealloc_pd(cma_dev_array[i].pd);
+- ibv_close_device(cma_dev_array[i].verbs);
+- }
+- free(cma_dev_array);
+ err2:
+ ibv_free_device_list(dev_list);
+ err1:
+@@ -312,12 +291,93 @@
+ return ret;
+ }
+
++static struct ibv_context *ucma_open_device(uint64_t guid)
++{
++ struct ibv_device **dev_list;
++ struct ibv_context *verbs = NULL;
++ int i;
++
++ dev_list = ibv_get_device_list(NULL);
++ if (!dev_list) {
++ fprintf(stderr, PFX "Fatal: unable to get RDMA device list\n");
++ return NULL;
++ }
++
++ for (i = 0; dev_list[i]; i++) {
++ if (ibv_get_device_guid(dev_list[i]) == guid) {
++ verbs = ibv_open_device(dev_list[i]);
++ break;
++ }
++ }
++
++ if (!verbs)
++ fprintf(stderr, PFX "Fatal: unable to open RDMA device\n");
++
++ ibv_free_device_list(dev_list);
++ return verbs;
++}
++
++static int ucma_init_device(struct cma_device *cma_dev)
++{
++ struct ibv_device_attr attr;
++ int ret;
++
++ if (cma_dev->verbs)
++ return 0;
++
++ cma_dev->verbs = ucma_open_device(cma_dev->guid);
++ if (!cma_dev->verbs)
++ return ERR(ENODEV);
++
++ ret = ibv_query_device(cma_dev->verbs, &attr);
++ if (ret) {
++ fprintf(stderr, PFX "Fatal: unable to query RDMA device\n");
++ ret = ERR(ret);
++ goto err;
++ }
++
++ cma_dev->port_cnt = attr.phys_port_cnt;
++ cma_dev->max_qpsize = attr.max_qp_wr;
++ cma_dev->max_initiator_depth = (uint8_t) attr.max_qp_init_rd_atom;
++ cma_dev->max_responder_resources = (uint8_t) attr.max_qp_rd_atom;
++ cma_init_cnt++;
++ return 0;
++
++err:
++ ibv_close_device(cma_dev->verbs);
++ cma_dev->verbs = NULL;
++ return ret;
++}
++
++static int ucma_init_all(void)
++{
++ int i, ret = 0;
++
++ if (!cma_dev_cnt) {
++ ret = ucma_init();
++ if (ret)
++ return ret;
++ }
++
++ if (cma_init_cnt == cma_dev_cnt)
++ return 0;
++
++ pthread_mutex_lock(&mut);
++ for (i = 0; i < cma_dev_cnt; i++) {
++ ret = ucma_init_device(&cma_dev_array[i]);
++ if (ret)
++ break;
++ }
++ pthread_mutex_unlock(&mut);
++ return ret;
++}
++
+ struct ibv_context **rdma_get_devices(int *num_devices)
+ {
+ struct ibv_context **devs = NULL;
+ int i;
+
+- if (ucma_init())
++ if (ucma_init_all())
+ goto out;
+
+ devs = malloc(sizeof *devs * (cma_dev_cnt + 1));
+@@ -354,9 +414,18 @@
if (!channel)
return NULL;
@@ -100,7 +319,48 @@
goto err;
}
return channel;
-@@ -1186,6 +1201,10 @@
+@@ -374,18 +443,33 @@
+ static int ucma_get_device(struct cma_id_private *id_priv, uint64_t guid)
+ {
+ struct cma_device *cma_dev;
+- int i;
++ int i, ret;
+
+ for (i = 0; i < cma_dev_cnt; i++) {
+ cma_dev = &cma_dev_array[i];
+- if (cma_dev->guid == guid) {
+- id_priv->cma_dev = cma_dev;
+- id_priv->id.verbs = cma_dev->verbs;
+- return 0;
+- }
++ if (cma_dev->guid == guid)
++ goto match;
+ }
+-
++
+ return ERR(ENODEV);
++match:
++ pthread_mutex_lock(&mut);
++ if ((ret = ucma_init_device(cma_dev)))
++ goto out;
++
++ if (!cma_dev->refcnt++) {
++ cma_dev->pd = ibv_alloc_pd(cma_dev->verbs);
++ if (!cma_dev->pd) {
++ cma_dev->refcnt--;
++ ret = ERR(ENOMEM);
++ goto out;
++ }
++ }
++ id_priv->cma_dev = cma_dev;
++ id_priv->id.verbs = cma_dev->verbs;
++out:
++ pthread_mutex_unlock(&mut);
++ return ret;
+ }
+
+ static void ucma_free_id(struct cma_id_private *id_priv)
+@@ -1186,6 +1270,10 @@
if (ret)
return ret;
@@ -111,7 +371,7 @@
qp = ibv_create_qp(pd, qp_init_attr);
if (!qp) {
ret = ERR(ENOMEM);
-@@ -1787,6 +1806,9 @@
+@@ -1787,6 +1875,9 @@
CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_ACCEPT, size);
cmd->id = id_priv->handle;
@@ -121,7 +381,7 @@
ret = write(id_priv->id.channel->fd, msg, size);
if (ret != size) {
-@@ -2051,8 +2073,20 @@
+@@ -2051,8 +2142,20 @@
CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_MIGRATE_ID, size);
cmd->id = id_priv->handle;
@@ -144,7 +404,7 @@
return (ret >= 0) ? ERR(ENODATA) : -1;
diff -r -u /tmp/librdmacm-1.0.14.1/man/rdma_create_id.3 librdmacm-1.0.14.1/man/rdma_create_id.3
--- /tmp/librdmacm-1.0.14.1/man/rdma_create_id.3 Mon Oct 4 17:00:18 2010
-+++ librdmacm-1.0.14.1/man/rdma_create_id.3 Mon Mar 28 03:11:48 2011
++++ librdmacm-1.0.14.1/man/rdma_create_id.3 Wed Apr 30 10:52:04 2014
@@ -31,9 +31,7 @@
explicitly binding to a specified RDMA device before communication
can occur, and most operations are asynchronous in nature. Asynchronous
@@ -158,7 +418,7 @@
.P
diff -r -u /tmp/librdmacm-1.0.14.1/man/rdma_create_qp.3 librdmacm-1.0.14.1/man/rdma_create_qp.3
--- /tmp/librdmacm-1.0.14.1/man/rdma_create_qp.3 Fri Dec 10 12:05:34 2010
-+++ librdmacm-1.0.14.1/man/rdma_create_qp.3 Mon Mar 28 03:11:48 2011
++++ librdmacm-1.0.14.1/man/rdma_create_qp.3 Wed Apr 30 11:48:07 2014
@@ -33,8 +33,7 @@
the rdma_cm_id will be created using a default protection domain. One
default protection domain is allocated per RDMA device.
@@ -171,7 +431,7 @@
channels. Completion channels and CQ data created by the rdma_cm are
diff -r -u /tmp/librdmacm-1.0.14.1/man/rdma_cm.7 librdmacm-1.0.14.1/man/rdma_cm.7
--- /tmp/librdmacm-1.0.14.1/man/rdma_cm.7 Mon Oct 4 17:00:18 2010
-+++ librdmacm-1.0.14.1/man/rdma_cm.7 Mon Mar 28 03:11:47 2011
++++ librdmacm-1.0.14.1/man/rdma_cm.7 Wed Apr 30 11:48:07 2014
@@ -19,7 +19,7 @@
API defined by the libibverbs library. The libibverbs library provides the
underlying interfaces needed to send and receive data.
@@ -238,7 +498,7 @@
rdma_resolve_route(3),
diff -r -u /tmp/librdmacm-1.0.14.1/include/infiniband/ib.h librdmacm-1.0.14.1/include/infiniband/ib.h
--- /tmp/librdmacm-1.0.14.1/include/infiniband/ib.h Mon Oct 4 17:00:18 2010
-+++ librdmacm-1.0.14.1/include/infiniband/ib.h Fri Feb 11 04:08:56 2011
++++ librdmacm-1.0.14.1/include/infiniband/ib.h Wed Apr 30 10:52:05 2014
@@ -33,7 +33,11 @@
#if !defined(_RDMA_IB_H)
#define _RDMA_IB_H
@@ -253,7 +513,7 @@
#ifndef AF_IB
diff -r -u /tmp/librdmacm-1.0.14.1/include/rdma/rdma_cma_abi.h librdmacm-1.0.14.1/include/rdma/rdma_cma_abi.h
--- /tmp/librdmacm-1.0.14.1/include/rdma/rdma_cma_abi.h Mon Oct 4 17:00:18 2010
-+++ librdmacm-1.0.14.1/include/rdma/rdma_cma_abi.h Fri Feb 11 04:08:48 2011
++++ librdmacm-1.0.14.1/include/rdma/rdma_cma_abi.h Wed Apr 30 10:52:05 2014
@@ -104,6 +104,9 @@
__u64 response;
struct sockaddr_in6 addr;
@@ -276,7 +536,7 @@
struct ucma_abi_join_mcast {
diff -r -u /tmp/librdmacm-1.0.14.1/examples/udaddy.c librdmacm-1.0.14.1/examples/udaddy.c
--- /tmp/librdmacm-1.0.14.1/examples/udaddy.c Mon Oct 4 17:00:18 2010
-+++ librdmacm-1.0.14.1/examples/udaddy.c Fri Feb 11 04:08:48 2011
++++ librdmacm-1.0.14.1/examples/udaddy.c Wed Apr 30 11:48:05 2014
@@ -40,7 +40,9 @@
#include <netinet/in.h>
#include <sys/socket.h>
@@ -289,7 +549,7 @@
#include <rdma/rdma_cma.h>
diff -r -u /tmp/librdmacm-1.0.14.1/examples/mckey.c librdmacm-1.0.14.1/examples/mckey.c
--- /tmp/librdmacm-1.0.14.1/examples/mckey.c Mon Oct 4 17:00:18 2010
-+++ librdmacm-1.0.14.1/examples/mckey.c Fri Feb 11 04:08:48 2011
++++ librdmacm-1.0.14.1/examples/mckey.c Wed Apr 30 11:48:05 2014
@@ -41,7 +41,9 @@
#include <arpa/inet.h>
#include <sys/socket.h>
@@ -348,7 +608,7 @@
diff -r -u /tmp/librdmacm-1.0.14.1/examples/cmatose.c librdmacm-1.0.14.1/examples/cmatose.c
--- /tmp/librdmacm-1.0.14.1/examples/cmatose.c Mon Oct 4 17:00:18 2010
-+++ librdmacm-1.0.14.1/examples/cmatose.c Fri Feb 11 04:08:48 2011
++++ librdmacm-1.0.14.1/examples/cmatose.c Wed Apr 30 11:48:03 2014
@@ -40,7 +40,9 @@
#include <netinet/in.h>
#include <sys/socket.h>
@@ -361,7 +621,7 @@
#include <rdma/rdma_cma.h>
diff -r -u /tmp/librdmacm-1.0.14.1/examples/rping.c librdmacm-1.0.14.1/examples/rping.c
--- /tmp/librdmacm-1.0.14.1/examples/rping.c Tue Feb 15 17:10:48 2011
-+++ librdmacm-1.0.14.1/examples/rping.c Wed May 25 14:46:30 2011
++++ librdmacm-1.0.14.1/examples/rping.c Wed Apr 30 11:48:04 2014
@@ -40,11 +40,17 @@
#include <netinet/in.h>
#include <sys/socket.h>