components/open-fabrics/librdmacm/patches/base.patch
changeset 369 cc8c00719da9
child 1022 6291fa546414
equal deleted inserted replaced
368:9a01d3a61f01 369:cc8c00719da9
       
     1 diff -r -u /tmp/librdmacm-1.0.14.1/configure librdmacm-1.0.14.1/configure
       
     2 --- /tmp/librdmacm-1.0.14.1/configure	Tue Feb 15 17:12:14 2011
       
     3 +++ librdmacm-1.0.14.1/configure	Thu Feb 24 08:39:24 2011
       
     4 @@ -7625,6 +7625,7 @@
       
     5   	esac ;;
       
     6        esac
       
     7        link_all_deplibs=yes
       
     8 +      hardcode_libdir_flag_spec=
       
     9        ;;
       
    10  
       
    11      sunos4*)
       
    12 diff -r -u /tmp/librdmacm-1.0.14.1/Makefile.in librdmacm-1.0.14.1/Makefile.in
       
    13 --- /tmp/librdmacm-1.0.14.1/Makefile.in	Tue Feb 15 17:12:13 2011
       
    14 +++ librdmacm-1.0.14.1/Makefile.in	Mon Mar 28 16:49:13 2011
       
    15 @@ -69,7 +69,7 @@
       
    16  	"$(DESTDIR)$(man1dir)" "$(DESTDIR)$(man3dir)" \
       
    17  	"$(DESTDIR)$(man7dir)" "$(DESTDIR)$(infinibandincludedir)" \
       
    18  	"$(DESTDIR)$(librdmacmincludedir)"
       
    19 -libLTLIBRARIES_INSTALL = $(INSTALL)
       
    20 +libLTLIBRARIES_INSTALL = $(INSTALL) -m 755
       
    21  LTLIBRARIES = $(lib_LTLIBRARIES)
       
    22  src_librdmacm_la_LIBADD =
       
    23  am_src_librdmacm_la_OBJECTS = src_librdmacm_la-cma.lo \
       
    24 @@ -76,7 +76,7 @@
       
    25  	src_librdmacm_la-addrinfo.lo src_librdmacm_la-acm.lo
       
    26  src_librdmacm_la_OBJECTS = $(am_src_librdmacm_la_OBJECTS)
       
    27  am__dirstamp = $(am__leading_dot)dirstamp
       
    28 -binPROGRAMS_INSTALL = $(INSTALL_PROGRAM)
       
    29 +binPROGRAMS_INSTALL = $(INSTALL_PROGRAM) -m 755
       
    30  PROGRAMS = $(bin_PROGRAMS)
       
    31  am_examples_mckey_OBJECTS = mckey.$(OBJEXT)
       
    32  examples_mckey_OBJECTS = $(am_examples_mckey_OBJECTS)
       
    33 diff -r -u /tmp/librdmacm-1.0.14.1/src/cma.h librdmacm-1.0.14.1/src/cma.h
       
    34 --- /tmp/librdmacm-1.0.14.1/src/cma.h	Mon Oct  4 17:00:18 2010
       
    35 +++ librdmacm-1.0.14.1/src/cma.h	Fri Feb 11 04:08:57 2011
       
    36 @@ -40,8 +40,10 @@
       
    37  
       
    38  #include <stdlib.h>
       
    39  #include <errno.h>
       
    40 +#if !(defined(__SVR4) && defined(__sun))
       
    41  #include <endian.h>
       
    42  #include <byteswap.h>
       
    43 +#endif
       
    44  
       
    45  #include <rdma/rdma_cma.h>
       
    46  
       
    47 @@ -58,14 +60,6 @@
       
    48  
       
    49  #define PFX "librdmacm: "
       
    50  
       
    51 -#if __BYTE_ORDER == __LITTLE_ENDIAN
       
    52 -static inline uint64_t htonll(uint64_t x) { return bswap_64(x); }
       
    53 -static inline uint64_t ntohll(uint64_t x) { return bswap_64(x); }
       
    54 -#else
       
    55 -static inline uint64_t htonll(uint64_t x) { return x; }
       
    56 -static inline uint64_t ntohll(uint64_t x) { return x; }
       
    57 -#endif
       
    58 -
       
    59  #define min(a, b) (a < b ? a : b)
       
    60  
       
    61  static inline int ERR(int err)
       
    62 diff -r -u /tmp/librdmacm-1.0.14.1/src/cma.c librdmacm-1.0.14.1/src/cma.c
       
    63 --- /tmp/librdmacm-1.0.14.1/src/cma.c	Fri Dec 10 12:05:34 2010
       
    64 +++ librdmacm-1.0.14.1/src/cma.c	Mon Mar 28 16:44:55 2011
       
    65 @@ -46,12 +46,15 @@
       
    66  #include <poll.h>
       
    67  #include <unistd.h>
       
    68  #include <pthread.h>
       
    69 +#if !(defined(__SVR4) && defined(__sun))
       
    70  #include <endian.h>
       
    71  #include <byteswap.h>
       
    72 +#endif
       
    73  #include <stddef.h>
       
    74  #include <netdb.h>
       
    75  
       
    76  #include "cma.h"
       
    77 +#include <infiniband/arch.h>
       
    78  #include <infiniband/driver.h>
       
    79  #include <infiniband/marshall.h>
       
    80  #include <rdma/rdma_cma.h>
       
    81 @@ -354,9 +357,18 @@
       
    82  	if (!channel)
       
    83  		return NULL;
       
    84  
       
    85 +#if defined(__SVR4) && defined(__sun)
       
    86 +	channel->fd = open("/dev/infiniband/ofs/rdma_cm", O_RDWR);
       
    87 +#else
       
    88  	channel->fd = open("/dev/infiniband/rdma_cm", O_RDWR);
       
    89 +#endif
       
    90 +
       
    91  	if (channel->fd < 0) {
       
    92 +#if defined(__SVR4) && defined(__sun)
       
    93 +		printf("CMA: unable to open /dev/infiniband/ofs/rdma_cm\n");
       
    94 +#else
       
    95  		printf("CMA: unable to open /dev/infiniband/rdma_cm\n");
       
    96 +#endif
       
    97  		goto err;
       
    98  	}
       
    99  	return channel;
       
   100 @@ -1186,6 +1198,10 @@
       
   101  	if (ret)
       
   102  		return ret;
       
   103  
       
   104 +#if defined(__SVR4) && defined(__sun)
       
   105 +	qp_init_attr->sq_sig_all |= LIB_RDMACM_QP_BIT;
       
   106 +#endif
       
   107 +
       
   108  	qp = ibv_create_qp(pd, qp_init_attr);
       
   109  	if (!qp) {
       
   110  		ret = ERR(ENOMEM);
       
   111 @@ -1787,6 +1803,9 @@
       
   112  
       
   113  	CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_ACCEPT, size);
       
   114  	cmd->id = id_priv->handle;
       
   115 +#if defined(__SVR4) && defined(__sun)
       
   116 +	cmd->conn_param.qp_num = ((id_priv->id).qp)->qp_num;
       
   117 +#endif
       
   118  
       
   119  	ret = write(id_priv->id.channel->fd, msg, size);
       
   120  	if (ret != size) {
       
   121 diff -r -u /tmp/librdmacm-1.0.14.1/man/rdma_create_id.3 librdmacm-1.0.14.1/man/rdma_create_id.3
       
   122 --- /tmp/librdmacm-1.0.14.1/man/rdma_create_id.3	Mon Oct  4 17:00:18 2010
       
   123 +++ librdmacm-1.0.14.1/man/rdma_create_id.3	Mon Mar 28 03:11:48 2011
       
   124 @@ -31,9 +31,7 @@
       
   125  explicitly binding to a specified RDMA device before communication
       
   126  can occur, and most operations are asynchronous in nature.  Asynchronous
       
   127  communication events on an rdma_cm_id are reported through the associated
       
   128 -event channel.  If the channel parameter is NULL, the rdma_cm_id will
       
   129 -be placed into synchronous operation.  While operating synchronously,
       
   130 -calls that result in an event will block until the operation completes.
       
   131 +event channel.
       
   132  The event will be returned to the user through the rdma_cm_id structure,
       
   133  and be available for access until another rdma_cm call is made.
       
   134  .P
       
   135 diff -r -u /tmp/librdmacm-1.0.14.1/man/rdma_create_qp.3 librdmacm-1.0.14.1/man/rdma_create_qp.3
       
   136 --- /tmp/librdmacm-1.0.14.1/man/rdma_create_qp.3	Fri Dec 10 12:05:34 2010
       
   137 +++ librdmacm-1.0.14.1/man/rdma_create_qp.3	Mon Mar 28 03:11:48 2011
       
   138 @@ -33,8 +33,7 @@
       
   139  the rdma_cm_id will be created using a default protection domain.  One
       
   140  default protection domain is allocated per RDMA device.
       
   141  .P
       
   142 -The initial QP attributes are specified by the qp_init_attr parameter.  The
       
   143 -send_cq and recv_cq fields in the ibv_qp_init_attr are optional.  If
       
   144 +The initial QP attributes are specified by the qp_init_attr parameter.  If
       
   145  a send or receive completion queue is not specified, then a CQ will be
       
   146  allocated by the rdma_cm for the QP, along with corresponding completion
       
   147  channels.  Completion channels and CQ data created by the rdma_cm are
       
   148 diff -r -u /tmp/librdmacm-1.0.14.1/man/rdma_cm.7 librdmacm-1.0.14.1/man/rdma_cm.7
       
   149 --- /tmp/librdmacm-1.0.14.1/man/rdma_cm.7	Mon Oct  4 17:00:18 2010
       
   150 +++ librdmacm-1.0.14.1/man/rdma_cm.7	Mon Mar 28 03:11:47 2011
       
   151 @@ -19,7 +19,7 @@
       
   152  API defined by the libibverbs library.  The libibverbs library provides the
       
   153  underlying interfaces needed to send and receive data.
       
   154  .P
       
   155 -The RDMA CM can operate asynchronously or synchronously.  The mode of
       
   156 +The RDMA CM operates asynchronously.  The mode of
       
   157  operation is controlled by the user through the use of the rdma_cm event channel
       
   158  parameter in specific calls.  If an event channel is provided, an rdma_cm identifier
       
   159  will report its event data (results of connecting, for example), on that channel.
       
   160 @@ -63,12 +63,7 @@
       
   161  .SH "CLIENT OPERATION"
       
   162  This section provides a general overview of the basic operation for the active,
       
   163  or client, side of communication.  This flow assume asynchronous operation with
       
   164 -low level call details shown.  For
       
   165 -synchronous operation, calls to rdma_create_event_channel, rdma_get_cm_event,
       
   166 -rdma_ack_cm_event, and rdma_destroy_event_channel
       
   167 -would be eliminated.  Abstracted calls, such as rdma_create_ep encapsulate
       
   168 -serveral of these calls under a single API.
       
   169 -Users may also refer to the example applications for
       
   170 +low level call details shown.  Users may also refer to the example applications for
       
   171  code samples.  A general connection flow would be:
       
   172  .IP rdma_getaddrinfo
       
   173  retrieve address information of the destination
       
   174 @@ -178,12 +173,9 @@
       
   175  rdma_ack_cm_event(3),
       
   176  rdma_bind_addr(3),
       
   177  rdma_connect(3),
       
   178 -rdma_create_ep(3),
       
   179  rdma_create_event_channel(3),
       
   180  rdma_create_id(3),
       
   181  rdma_create_qp(3),
       
   182 -rdma_dereg_mr(3),
       
   183 -rdma_destroy_ep(3),
       
   184  rdma_destroy_event_channel(3),
       
   185  rdma_destroy_id(3),
       
   186  rdma_destroy_qp(3),
       
   187 @@ -196,27 +188,11 @@
       
   188  rdma_get_dst_port(3),
       
   189  rdma_get_local_addr(3),
       
   190  rdma_get_peer_addr(3),
       
   191 -rdma_get_recv_comp(3),
       
   192 -rdma_get_request(3),
       
   193 -rdma_get_send_comp(3),
       
   194  rdma_get_src_port(3),
       
   195  rdma_join_multicast(3),
       
   196  rdma_leave_multicast(3),
       
   197  rdma_listen(3),
       
   198 -rdma_migrate_id(3),
       
   199  rdma_notify(3),
       
   200 -rdma_post_read(3)
       
   201 -rdma_post_readv(3),
       
   202 -rdma_post_recv(3),
       
   203 -rdma_post_recvv(3),
       
   204 -rdma_post_send(3),
       
   205 -rdma_post_sendv(3),
       
   206 -rdma_post_ud_send(3),
       
   207 -rdma_post_write(3),
       
   208 -rdma_post_writev(3),
       
   209 -rdma_reg_msgs(3),
       
   210 -rdma_reg_read(3),
       
   211 -rdma_reg_write(3),
       
   212  rdma_reject(3),
       
   213  rdma_resolve_addr(3),
       
   214  rdma_resolve_route(3),
       
   215 diff -r -u /tmp/librdmacm-1.0.14.1/include/infiniband/ib.h librdmacm-1.0.14.1/include/infiniband/ib.h
       
   216 --- /tmp/librdmacm-1.0.14.1/include/infiniband/ib.h	Mon Oct  4 17:00:18 2010
       
   217 +++ librdmacm-1.0.14.1/include/infiniband/ib.h	Fri Feb 11 04:08:56 2011
       
   218 @@ -33,7 +33,11 @@
       
   219  #if !defined(_RDMA_IB_H)
       
   220  #define _RDMA_IB_H
       
   221  
       
   222 +#if !(defined(__SVR4) && defined(__sun))
       
   223  #include <linux/types.h>
       
   224 +#else
       
   225 +#include <infiniband/ofa_solaris.h>
       
   226 +#endif
       
   227  #include <string.h>
       
   228  
       
   229  #ifndef AF_IB
       
   230 diff -r -u /tmp/librdmacm-1.0.14.1/include/rdma/rdma_cma_abi.h librdmacm-1.0.14.1/include/rdma/rdma_cma_abi.h
       
   231 --- /tmp/librdmacm-1.0.14.1/include/rdma/rdma_cma_abi.h	Mon Oct  4 17:00:18 2010
       
   232 +++ librdmacm-1.0.14.1/include/rdma/rdma_cma_abi.h	Fri Feb 11 04:08:48 2011
       
   233 @@ -104,6 +104,9 @@
       
   234  	__u64 response;
       
   235  	struct sockaddr_in6 addr;
       
   236  	__u32 id;
       
   237 +#if defined(__SVR4) && defined(__sun)
       
   238 +	uint32_t	reserved;
       
   239 +#endif
       
   240  };
       
   241  
       
   242  struct ucma_abi_bind {
       
   243 @@ -243,6 +246,9 @@
       
   244  	__u64 uid;
       
   245  	struct sockaddr_in6 addr;
       
   246  	__u32 id;
       
   247 +#if defined(__SVR4) && defined(__sun)
       
   248 +	uint32_t	reserved;
       
   249 +#endif
       
   250  };
       
   251  
       
   252  struct ucma_abi_join_mcast {
       
   253 diff -r -u /tmp/librdmacm-1.0.14.1/examples/udaddy.c librdmacm-1.0.14.1/examples/udaddy.c
       
   254 --- /tmp/librdmacm-1.0.14.1/examples/udaddy.c	Mon Oct  4 17:00:18 2010
       
   255 +++ librdmacm-1.0.14.1/examples/udaddy.c	Fri Feb 11 04:08:48 2011
       
   256 @@ -40,7 +40,9 @@
       
   257  #include <netinet/in.h>
       
   258  #include <sys/socket.h>
       
   259  #include <netdb.h>
       
   260 +#if !(defined(__SVR4) && defined(__sun))
       
   261  #include <byteswap.h>
       
   262 +#endif
       
   263  #include <getopt.h>
       
   264  
       
   265  #include <rdma/rdma_cma.h>
       
   266 diff -r -u /tmp/librdmacm-1.0.14.1/examples/mckey.c librdmacm-1.0.14.1/examples/mckey.c
       
   267 --- /tmp/librdmacm-1.0.14.1/examples/mckey.c	Mon Oct  4 17:00:18 2010
       
   268 +++ librdmacm-1.0.14.1/examples/mckey.c	Fri Feb 11 04:08:48 2011
       
   269 @@ -41,7 +41,9 @@
       
   270  #include <arpa/inet.h>
       
   271  #include <sys/socket.h>
       
   272  #include <netdb.h>
       
   273 +#if !(defined(__SVR4) && defined(__sun))
       
   274  #include <byteswap.h>
       
   275 +#endif
       
   276  #include <unistd.h>
       
   277  #include <getopt.h>
       
   278  
       
   279 @@ -329,6 +331,16 @@
       
   280  
       
   281  	while (1) {
       
   282  		ret = rdma_get_cm_event(test.channel, &event);
       
   283 +
       
   284 +/* 
       
   285 + * Solaris returns EBADF if we close the channel while we're waiting
       
   286 + * for any events to occur. It is safe to ignore EBADF here.
       
   287 + */
       
   288 +#if defined(__SVR4) && defined(__sun)
       
   289 +		if (ret && (errno == EBADF))
       
   290 +			break;
       
   291 +#endif
       
   292 +
       
   293  		if (ret) {
       
   294  			perror("rdma_get_cm_event");
       
   295  			break;
       
   296 @@ -461,6 +473,7 @@
       
   297  	return ret;
       
   298  }
       
   299  
       
   300 +#if !(defined(__SVR4) && defined(__sun))
       
   301  static int get_dst_addr(char *dst, struct sockaddr *addr)
       
   302  {
       
   303  	struct sockaddr_ib *sib;
       
   304 @@ -474,6 +487,7 @@
       
   305  	inet_pton(AF_INET6, dst, &sib->sib_addr);
       
   306  	return 0;
       
   307  }
       
   308 +#endif
       
   309  
       
   310  static int run(void)
       
   311  {
       
   312 @@ -486,7 +500,12 @@
       
   313  			return ret;
       
   314  	}
       
   315  
       
   316 +/* Solaris does not yet support family AF_IB */
       
   317 +#if defined(__SVR4) && defined(__sun)
       
   318 +	ret = get_addr(dst_addr, (struct sockaddr *) &test.dst_in);
       
   319 +#else
       
   320  	ret = get_dst_addr(dst_addr, (struct sockaddr *) &test.dst_in);
       
   321 +#endif
       
   322  	if (ret)
       
   323  		return ret;
       
   324  
       
   325 diff -r -u /tmp/librdmacm-1.0.14.1/examples/cmatose.c librdmacm-1.0.14.1/examples/cmatose.c
       
   326 --- /tmp/librdmacm-1.0.14.1/examples/cmatose.c	Mon Oct  4 17:00:18 2010
       
   327 +++ librdmacm-1.0.14.1/examples/cmatose.c	Fri Feb 11 04:08:48 2011
       
   328 @@ -40,7 +40,9 @@
       
   329  #include <netinet/in.h>
       
   330  #include <sys/socket.h>
       
   331  #include <netdb.h>
       
   332 +#if !(defined(__SVR4) && defined(__sun))
       
   333  #include <byteswap.h>
       
   334 +#endif
       
   335  #include <getopt.h>
       
   336  
       
   337  #include <rdma/rdma_cma.h>
       
   338 diff -r -u /tmp/librdmacm-1.0.14.1/examples/rping.c librdmacm-1.0.14.1/examples/rping.c
       
   339 --- /tmp/librdmacm-1.0.14.1/examples/rping.c	Tue Feb 15 17:10:48 2011
       
   340 +++ librdmacm-1.0.14.1/examples/rping.c	Wed May 25 14:46:30 2011
       
   341 @@ -40,11 +40,17 @@
       
   342  #include <netinet/in.h>
       
   343  #include <sys/socket.h>
       
   344  #include <netdb.h>
       
   345 +#if !(defined(__SVR4) && defined(__sun))
       
   346  #include <byteswap.h>
       
   347 +#endif
       
   348  #include <semaphore.h>
       
   349  #include <arpa/inet.h>
       
   350  #include <pthread.h>
       
   351  #include <inttypes.h>
       
   352 +#if defined(__SVR4) && defined(__sun)
       
   353 +#include <unistd.h>
       
   354 +#include <libgen.h>
       
   355 +#endif
       
   356  
       
   357  #include <rdma/rdma_cma.h>
       
   358  #include <infiniband/arch.h>
       
   359 @@ -85,6 +91,13 @@
       
   360  	ERROR
       
   361  };
       
   362  
       
   363 +enum disconnect_state {
       
   364 +	DISCONNECT_NONE,
       
   365 +	CALLING_DISCONNECT = 1,
       
   366 +	DISCONNECT_CALLED,
       
   367 +	DISCONNECT_DONE
       
   368 +};
       
   369 +
       
   370  struct rping_rdma_info {
       
   371  	uint64_t buf;
       
   372  	uint32_t rkey;
       
   373 @@ -143,6 +156,9 @@
       
   374  	enum test_state state;		/* used for cond/signalling */
       
   375  	sem_t sem;
       
   376  
       
   377 +	enum disconnect_state  dis_state;
       
   378 +	sem_t dis_sem;
       
   379 +
       
   380  	struct sockaddr_storage sin;
       
   381  	uint16_t port;			/* dst port in NBO */
       
   382  	int verbose;			/* verbose logging */
       
   383 @@ -218,6 +234,8 @@
       
   384  		fprintf(stderr, "%s DISCONNECT EVENT...\n",
       
   385  			cb->server ? "server" : "client");
       
   386  		sem_post(&cb->sem);
       
   387 +		cb->dis_state = DISCONNECT_DONE;
       
   388 +		sem_post(&cb->dis_sem);
       
   389  		break;
       
   390  
       
   391  	case RDMA_CM_EVENT_DEVICE_REMOVAL:
       
   392 @@ -285,6 +303,29 @@
       
   393  					"cq completion failed status %d\n",
       
   394  					wc.status);
       
   395  				ret = -1;
       
   396 +			} else {
       
   397 +				/*
       
   398 +				 * FLUSH Error can be polled before RDMA-CM
       
   399 +				 * DISCONNECT is notified. Ensure that cb_state
       
   400 +				 * is set appropriately in such a case.
       
   401 +				 * sleep for sometime if Disconnect has not
       
   402 +				 * been called. The FLUSH WR can be because
       
   403 +				 * the remote end initiated the disconnect.
       
   404 +				 */
       
   405 +				if (cb->dis_state == DISCONNECT_NONE)
       
   406 +					sleep(2);
       
   407 +
       
   408 +				if (cb->dis_state == DISCONNECT_DONE)
       
   409 +					return (0);
       
   410 +
       
   411 +				/* Wait if disconnect is called. */
       
   412 +				if (cb->dis_state == DISCONNECT_CALLED) {
       
   413 +					sem_wait(&cb->dis_sem);
       
   414 +					if (cb->dis_state == DISCONNECT_DONE)
       
   415 +						return (0);
       
   416 +					else
       
   417 +						goto error;
       
   418 +				}
       
   419  			}
       
   420  			goto error;
       
   421  		}
       
   422 @@ -571,9 +612,15 @@
       
   423  
       
   424  	while (1) {
       
   425  		ret = rdma_get_cm_event(cb->cm_channel, &event);
       
   426 -		if (ret) {
       
   427 +		/*
       
   428 +		 * If the retry of read() syscall returned EBADF, as the
       
   429 +		 * file was closed on process exit. Ignore this error.
       
   430 +		 */
       
   431 +		if (ret && errno != EBADF) {
       
   432  			perror("rdma_get_cm_event");
       
   433  			exit(ret);
       
   434 +		} else if (ret && errno == EBADF) {
       
   435 +			exit(0);
       
   436  		}
       
   437  		ret = rping_cma_event_handler(event->id, event);
       
   438  		rdma_ack_cm_event(event);
       
   439 @@ -595,8 +642,14 @@
       
   440  		pthread_testcancel();
       
   441  
       
   442  		ret = ibv_get_cq_event(cb->channel, &ev_cq, &ev_ctx);
       
   443 -		if (ret) {
       
   444 +		/*
       
   445 +		 * If the retry of write() syscall returned EBADF, as the
       
   446 +		 * file was closed on process exit. Ignore this error.
       
   447 +		 */
       
   448 +		if (ret && errno != EBADF) {
       
   449  			fprintf(stderr, "Failed to get cq event!\n");
       
   450 +			 pthread_exit(NULL);
       
   451 +		} else if (ret && errno == EBADF) {
       
   452  			pthread_exit(NULL);
       
   453  		}
       
   454  		if (ev_cq != cb->cq) {
       
   455 @@ -801,11 +854,13 @@
       
   456  	}
       
   457  
       
   458  	rping_test_server(cb);
       
   459 +	cb->dis_state = CALLING_DISCONNECT;
       
   460 +	sem_post(&cb->dis_sem);
       
   461  	rdma_disconnect(cb->child_cm_id);
       
   462 -	rping_free_buffers(cb);
       
   463 -	rping_free_qp(cb);
       
   464  	pthread_cancel(cb->cqthread);
       
   465  	pthread_join(cb->cqthread, NULL);
       
   466 +	rping_free_buffers(cb);
       
   467 +	rping_free_qp(cb);
       
   468  	rdma_destroy_id(cb->child_cm_id);
       
   469  	free_cb(cb);
       
   470  	return NULL;
       
   471 @@ -889,6 +944,8 @@
       
   472  	}
       
   473  
       
   474  	rping_test_server(cb);
       
   475 +	cb->dis_state = CALLING_DISCONNECT;
       
   476 +	sem_post(&cb->dis_sem);
       
   477  	rdma_disconnect(cb->child_cm_id);
       
   478  	rdma_destroy_id(cb->child_cm_id);
       
   479  err2:
       
   480 @@ -1056,6 +1113,8 @@
       
   481  	}
       
   482  
       
   483  	rping_test_client(cb);
       
   484 +	cb->dis_state = CALLING_DISCONNECT;
       
   485 +	sem_post(&cb->dis_sem);
       
   486  	rdma_disconnect(cb->cm_id);
       
   487  err2:
       
   488  	rping_free_buffers(cb);
       
   489 @@ -1123,6 +1182,7 @@
       
   490  	cb->sin.ss_family = PF_INET;
       
   491  	cb->port = htons(7174);
       
   492  	sem_init(&cb->sem, 0, 0);
       
   493 +	sem_init(&cb->dis_sem, 0, 0);
       
   494  
       
   495  	opterr = 0;
       
   496  	while ((op=getopt(argc, argv, "a:Pp:C:S:t:scvVd")) != -1) {