6948085 socket close should not return until all references to the socket are closed
authoragiri <Giri.Adari@Sun.COM>
Thu, 06 May 2010 09:54:45 -0700
changeset 12320 4d40a60c2314
parent 12319 450834a921de
child 12321 c2943f5c6eb9
6948085 socket close should not return until all references to the socket are closed 6947648 missing conn->c_cm_lock exit on errors in rdsv3_ib_cm_handle_connect()/ib_cm.c 6940747 rdsv3_wake_sk_sleep is called without checking if any threads are waiting to be woken 6948603 Change all L0 and L1 debug messages to L2
usr/src/uts/common/io/ib/clients/rdsv3/af_rds.c
usr/src/uts/common/io/ib/clients/rdsv3/cong.c
usr/src/uts/common/io/ib/clients/rdsv3/connection.c
usr/src/uts/common/io/ib/clients/rdsv3/ib_cm.c
usr/src/uts/common/io/ib/clients/rdsv3/ib_rdma.c
usr/src/uts/common/io/ib/clients/rdsv3/ib_recv.c
usr/src/uts/common/io/ib/clients/rdsv3/ib_send.c
usr/src/uts/common/io/ib/clients/rdsv3/message.c
usr/src/uts/common/io/ib/clients/rdsv3/rdma_transport.c
usr/src/uts/common/io/ib/clients/rdsv3/rds_recv.c
usr/src/uts/common/io/ib/clients/rdsv3/rdsv3_ddi.c
usr/src/uts/common/io/ib/clients/rdsv3/rdsv3_impl.c
usr/src/uts/common/io/ib/clients/rdsv3/send.c
usr/src/uts/common/io/ib/clients/rdsv3/threads.c
usr/src/uts/common/sys/ib/clients/rdsv3/rdsv3.h
usr/src/uts/common/sys/ib/clients/rdsv3/rdsv3_debug.h
usr/src/uts/common/sys/ib/clients/rdsv3/rdsv3_impl.h
--- a/usr/src/uts/common/io/ib/clients/rdsv3/af_rds.c	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/io/ib/clients/rdsv3/af_rds.c	Thu May 06 09:54:45 2010 -0700
@@ -85,7 +85,6 @@
 kmutex_t rdsv3_sock_lock;
 static unsigned long rdsv3_sock_count;
 list_t rdsv3_sock_list;
-rdsv3_wait_queue_t rdsv3_poll_waitq;
 
 /*
  * This is called as the final descriptor referencing this socket is closed.
@@ -127,6 +126,12 @@
 	rdsv3_sock_count--;
 	mutex_exit(&rdsv3_sock_lock);
 
+	while (sk->sk_refcount > 1) {
+		/* wait for 1 sec and try again */
+		delay(drv_usectohz(1000000));
+	}
+
+	/* this will free the rs and sk */
 	rdsv3_sk_sock_put(sk);
 
 	RDSV3_DPRINTF4("rdsv3_release", "Return (rds: %p)", rs);
--- a/usr/src/uts/common/io/ib/clients/rdsv3/cong.c	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/io/ib/clients/rdsv3/cong.c	Thu May 06 09:54:45 2010 -0700
@@ -276,11 +276,6 @@
 	if (waitqueue_active(&map->m_waitq))
 #endif
 		rdsv3_wake_up(&map->m_waitq);
-#if 0
-XXX
-	if (waitqueue_active(&rds_poll_waitq))
-#endif
-		rdsv3_wake_up_all(&rdsv3_poll_waitq);
 
 	if (portmask && !list_is_empty(&rdsv3_cong_monitor)) {
 		struct rdsv3_sock *rs;
@@ -415,7 +410,7 @@
 rdsv3_cong_wait(struct rdsv3_cong_map *map, uint16_be_t port, int nonblock,
     struct rdsv3_sock *rs)
 {
-	int	ret = 0;
+	int ret = 0;
 
 	RDSV3_DPRINTF4("rdsv3_cong_wait", "Enter(rs: %p, mode: %d)",
 	    rs, nonblock);
@@ -448,17 +443,26 @@
 	RDSV3_DPRINTF3("rdsv3_cong_wait", "waiting on map %p for port %u",
 	    map, ntohs(port));
 
+#if 0
+	ret = rdsv3_wait_sig(&map->m_waitq, !rdsv3_cong_test_bit(map, port));
+	if (ret == 0)
+		return (-ERESTART);
+	return (0);
+#else
 	mutex_enter(&map->m_waitq.waitq_mutex);
+	map->m_waitq.waitq_waiters++;
 	while (rdsv3_cong_test_bit(map, port)) {
-		if (cv_wait_sig(&map->m_waitq.waitq_cv,
-		    &map->m_waitq.waitq_mutex) == 0) {
+		ret = cv_wait_sig(&map->m_waitq.waitq_cv,
+		    &map->m_waitq.waitq_mutex);
+		if (ret == 0) {
 			ret = -ERESTART;
 			break;
 		}
 	}
+	map->m_waitq.waitq_waiters--;
 	mutex_exit(&map->m_waitq.waitq_mutex);
-
 	return (ret);
+#endif
 }
 
 void
--- a/usr/src/uts/common/io/ib/clients/rdsv3/connection.c	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/io/ib/clients/rdsv3/connection.c	Thu May 06 09:54:45 2010 -0700
@@ -495,7 +495,7 @@
 	    sizeof (struct rdsv3_connection), 0, rdsv3_conn_constructor,
 	    rdsv3_conn_destructor, NULL, NULL, NULL, 0);
 	if (rdsv3_conn_slab == NULL) {
-		RDSV3_DPRINTF1("rdsv3_conn_init",
+		RDSV3_DPRINTF2("rdsv3_conn_init",
 		    "kmem_cache_create(rdsv3_conn_slab) failed");
 		return (-1);
 	}
--- a/usr/src/uts/common/io/ib/clients/rdsv3/ib_cm.c	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/io/ib/clients/rdsv3/ib_cm.c	Thu May 06 09:54:45 2010 -0700
@@ -126,7 +126,7 @@
 	attr->min_rnr_timer = IB_RNR_TIMER_000_32;
 	ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER);
 	if (ret)
-		RDSV3_DPRINTF0("rdsv3_ib_tune_rnr",
+		RDSV3_DPRINTF2("rdsv3_ib_tune_rnr",
 		    "ib_modify_qp(IB_QP_MIN_RNR_TIMER): err=%d", -ret);
 }
 
@@ -186,14 +186,14 @@
 	qp_attr.qp_state = IB_QPS_RTS;
 	err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE);
 	if (err)
-		RDSV3_DPRINTF0("rdsv3_ib_cm_connect_complete",
+		RDSV3_DPRINTF2("rdsv3_ib_cm_connect_complete",
 		    "ib_modify_qp(IB_QP_STATE, RTS): err=%d", err);
 
 	/* update ib_device with this local ipaddr & conn */
 	rds_ibdev = ib_get_client_data(ic->i_cm_id->device, &rdsv3_ib_client);
 	err = rdsv3_ib_update_ipaddr(rds_ibdev, conn->c_laddr);
 	if (err)
-		RDSV3_DPRINTF0("rdsv3_ib_cm_connect_complete",
+		RDSV3_DPRINTF2("rdsv3_ib_cm_connect_complete",
 		    "rdsv3_ib_update_ipaddr failed (%d)", err);
 	rdsv3_ib_add_conn(rds_ibdev, conn);
 
@@ -331,7 +331,7 @@
 	 */
 	rds_ibdev = ib_get_client_data(dev, &rdsv3_ib_client);
 	if (rds_ibdev == NULL) {
-		RDSV3_DPRINTF0("rdsv3_ib_setup_qp",
+		RDSV3_DPRINTF2("rdsv3_ib_setup_qp",
 		    "RDS/IB: No client_data for device %s", dev->name);
 		return (-EOPNOTSUPP);
 	}
@@ -505,7 +505,7 @@
 		while ((common >>= 1) != 0)
 			version++;
 	} else {
-		RDSV3_DPRINTF0("rdsv3_ib_protocol_compatible",
+		RDSV3_DPRINTF2("rdsv3_ib_protocol_compatible",
 		    "RDS: Connection from %u.%u.%u.%u using "
 		    "incompatible protocol version %u.%u\n",
 		    NIPQUAD(dp->dp_saddr),
@@ -623,6 +623,7 @@
 	if (err) {
 		RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect",
 		    "rdsv3_ib_setup_qp failed (%d)", err);
+		mutex_exit(&conn->c_cm_lock);
 		rdsv3_conn_drop(conn);
 		goto out;
 	}
@@ -801,7 +802,7 @@
 			    ib_get_ibt_channel_hdl(ic->i_cm_id));
 
 			/* wait until all WRs are flushed */
-			rdsv3_wait_event(rdsv3_ib_ring_empty_wait,
+			rdsv3_wait_event(&rdsv3_ib_ring_empty_wait,
 			    rdsv3_ib_ring_empty(&ic->i_send_ring) &&
 			    rdsv3_ib_ring_empty(&ic->i_recv_ring));
 
--- a/usr/src/uts/common/io/ib/clients/rdsv3/ib_rdma.c	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/io/ib/clients/rdsv3/ib_rdma.c	Thu May 06 09:54:45 2010 -0700
@@ -401,7 +401,7 @@
 		    "Return: ibmr: %p umem_cookie %p", ibmr, ibmr->umem_cookie);
 		return (ibmr);
 	} else { /* error return */
-		RDSV3_DPRINTF1("rdsv3_ib_get_mr", "map_fmr failed (errno=%d)\n",
+		RDSV3_DPRINTF2("rdsv3_ib_get_mr", "map_fmr failed (errno=%d)\n",
 		    ret);
 		ddi_umem_unlock(umem_cookie);
 		kmem_free((void *)ibmr, sizeof (*ibmr));
--- a/usr/src/uts/common/io/ib/clients/rdsv3/ib_recv.c	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/io/ib/clients/rdsv3/ib_recv.c	Thu May 06 09:54:45 2010 -0700
@@ -300,7 +300,7 @@
 	while ((prefill || rdsv3_conn_up(conn)) &&
 	    rdsv3_ib_ring_alloc(&ic->i_recv_ring, 1, &pos)) {
 		if (pos >= ic->i_recv_ring.w_nr) {
-			RDSV3_DPRINTF0("rdsv3_ib_recv_refill",
+			RDSV3_DPRINTF2("rdsv3_ib_recv_refill",
 			    "Argh - ring alloc returned pos=%u",
 			    pos);
 			ret = -EINVAL;
--- a/usr/src/uts/common/io/ib/clients/rdsv3/ib_send.c	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/io/ib/clients/rdsv3/ib_send.c	Thu May 06 09:54:45 2010 -0700
@@ -276,7 +276,7 @@
 				break;
 			default:
 #ifndef __lock_lint
-				RDSV3_DPRINTF0("rdsv3_ib_send_cq_comp_handler",
+				RDSV3_DPRINTF2("rdsv3_ib_send_cq_comp_handler",
 				    "RDS/IB: %s: unexpected opcode "
 				    "0x%x in WR!",
 				    __func__, send->s_opcode);
@@ -826,7 +826,7 @@
 		if (0) {
 			struct rdsv3_header *hdr = &ic->i_send_hdrs[pos];
 
-			RDSV3_DPRINTF0("rdsv3_ib_xmit",
+			RDSV3_DPRINTF2("rdsv3_ib_xmit",
 			    "send WR dport=%u flags=0x%x len=%d",
 			    ntohs(hdr->h_dport),
 			    hdr->h_flags,
@@ -872,12 +872,12 @@
 	ret = ibt_post_send(ib_get_ibt_channel_hdl(ic->i_cm_id),
 	    ic->i_send_wrs, i, &posted);
 	if (posted != i) {
-		RDSV3_DPRINTF1("rdsv3_ib_xmit",
+		RDSV3_DPRINTF2("rdsv3_ib_xmit",
 		    "ic %p first %p nwr: %d ret %d:%d",
 		    ic, first, i, ret, posted);
 	}
 	if (ret) {
-		RDSV3_DPRINTF0("rdsv3_ib_xmit",
+		RDSV3_DPRINTF2("rdsv3_ib_xmit",
 		    "RDS/IB: ib_post_send to %u.%u.%u.%u "
 		    "returned %d\n", NIPQUAD(conn->c_faddr), ret);
 		rdsv3_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
@@ -1126,7 +1126,7 @@
 	status = ibt_post_send(ib_get_ibt_channel_hdl(ic->i_cm_id),
 	    ic->i_send_wrs, k, &posted);
 	if (status != IBT_SUCCESS) {
-		RDSV3_DPRINTF0("rdsv3_ib_xmit_rdma",
+		RDSV3_DPRINTF2("rdsv3_ib_xmit_rdma",
 		    "RDS/IB: rdma ib_post_send returned %d", status);
 		rdsv3_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
 	}
--- a/usr/src/uts/common/io/ib/clients/rdsv3/message.c	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/io/ib/clients/rdsv3/message.c	Thu May 06 09:54:45 2010 -0700
@@ -461,7 +461,7 @@
 void
 rdsv3_message_wait(struct rdsv3_message *rm)
 {
-	rdsv3_wait_event(rdsv3_message_flush_waitq,
+	rdsv3_wait_event(&rdsv3_message_flush_waitq,
 	    !test_bit(RDSV3_MSG_MAPPED, &rm->m_flags));
 }
 
--- a/usr/src/uts/common/io/ib/clients/rdsv3/rdma_transport.c	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/io/ib/clients/rdsv3/rdma_transport.c	Thu May 06 09:54:45 2010 -0700
@@ -141,7 +141,7 @@
 		    "RDS/RDMA: DISCONNECT event - dropping connection "
 		    "cm_id: %p", cm_id);
 		if (conn) {
-			RDSV3_DPRINTF0("rdsv3_rdma_cm_event_handler",
+			RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler",
 			    "RDS/RDMA: DISCONNECT event - dropping connection "
 			    "%u.%u.%u.%u ->%u.%u.%u.%u", NIPQUAD(conn->c_laddr),
 			    NIPQUAD(conn->c_faddr));
@@ -151,7 +151,7 @@
 
 	default:
 		/* things like device disconnect? */
-		RDSV3_DPRINTF0("rdsv3_rdma_cm_event_handler",
+		RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler",
 		    "unknown event %u\n", event->event);
 		RDSV3_PANIC();
 		break;
@@ -187,7 +187,7 @@
 	cm_id = rdma_create_id(rdsv3_rdma_cm_event_handler, NULL, RDMA_PS_TCP);
 	if (IS_ERR(cm_id)) {
 		ret = PTR_ERR(cm_id);
-		RDSV3_DPRINTF0("rdsv3_rdma_listen_init",
+		RDSV3_DPRINTF2("rdsv3_rdma_listen_init",
 		    "RDS/RDMA: failed to setup listener, "
 		    "rdma_create_id() returned %d", ret);
 		goto out;
@@ -203,7 +203,7 @@
 	 */
 	ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin);
 	if (ret) {
-		RDSV3_DPRINTF0("rdsv3_rdma_listen_init",
+		RDSV3_DPRINTF2("rdsv3_rdma_listen_init",
 		    "RDS/RDMA: failed to setup listener, "
 		    "rdma_bind_addr() returned %d", ret);
 		goto out;
@@ -211,7 +211,7 @@
 
 	ret = rdma_listen(cm_id, 128);
 	if (ret) {
-		RDSV3_DPRINTF0("rdsv3_rdma_listen_init",
+		RDSV3_DPRINTF2("rdsv3_rdma_listen_init",
 		    "RDS/RDMA: failed to setup listener, "
 		    "rdma_listen() returned %d", ret);
 		goto out;
--- a/usr/src/uts/common/io/ib/clients/rdsv3/rds_recv.c	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/io/ib/clients/rdsv3/rds_recv.c	Thu May 06 09:54:45 2010 -0700
@@ -520,25 +520,23 @@
 			RDSV3_DPRINTF3("rdsv3_recvmsg",
 			    "Before wait (rs: %p)", rs);
 
+#if 0
+			ret = rdsv3_wait_sig(sk->sk_sleep,
+			    !(list_is_empty(&rs->rs_notify_queue) &&
+			    !rs->rs_cong_notify &&
+			    !rdsv3_next_incoming(rs, &inc)));
+			if (ret == 0) {
+				/* signal/timeout pending */
+				RDSV3_DPRINTF2("rdsv3_recvmsg",
+				    "woke due to signal");
+				ret = -ERESTART;
+			}
+#else
 			mutex_enter(&sk->sk_sleep->waitq_mutex);
+			sk->sk_sleep->waitq_waiters++;
 			while ((list_is_empty(&rs->rs_notify_queue) &&
 			    !rs->rs_cong_notify &&
 			    !rdsv3_next_incoming(rs, &inc))) {
-#if 0
-				ret = cv_timedwait_sig(&sk->sk_sleep->waitq_cv,
-				    &sk->sk_sleep->waitq_mutex,
-				    timeo * drv_usectohz(1000000) +
-				    ddi_get_lbolt());
-				if (ret <= 0) {
-					/* signal/timeout pending */
-					RDSV3_DPRINTF2("rdsv3_recvmsg",
-					    "woke due to signal/timeout: %d",
-					    ret);
-					ret = (ret == 0) ? -ERESTART :
-					    -ETIMEDOUT;
-					break;
-				}
-#else
 				ret = cv_wait_sig(&sk->sk_sleep->waitq_cv,
 				    &sk->sk_sleep->waitq_mutex);
 				if (ret == 0) {
@@ -548,9 +546,10 @@
 					ret = -ERESTART;
 					break;
 				}
+			}
+			sk->sk_sleep->waitq_waiters--;
+			mutex_exit(&sk->sk_sleep->waitq_mutex);
 #endif
-			}
-			mutex_exit(&sk->sk_sleep->waitq_mutex);
 
 			RDSV3_DPRINTF5("rdsv3_recvmsg",
 			    "recvmsg woke rs: %p inc %p ret %d",
--- a/usr/src/uts/common/io/ib/clients/rdsv3/rdsv3_ddi.c	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/io/ib/clients/rdsv3/rdsv3_ddi.c	Thu May 06 09:54:45 2010 -0700
@@ -66,7 +66,7 @@
 	    sizeof (struct rsock) + sizeof (struct rdsv3_sock), 0, NULL,
 	    NULL, NULL, NULL, NULL, 0);
 	if (rdsv3_alloc_cache == NULL) {
-		RDSV3_DPRINTF1("rdsv3_alloc_cache",
+		RDSV3_DPRINTF2("rdsv3_alloc_cache",
 		    "kmem_cache_create(rdsv3_alloc_cache) failed");
 		return (-1);
 	}
@@ -107,7 +107,7 @@
 		return (DDI_FAILURE);
 
 	if (rdsv3_dev_info != NULL) {
-		RDSV3_DPRINTF1("rdsv3_attach", "Multiple RDS instances are"
+		RDSV3_DPRINTF2("rdsv3_attach", "Multiple RDS instances are"
 		    " not supported (rdsv3_dev_info: 0x%p)", rdsv3_dev_info);
 		return (DDI_FAILURE);
 	}
@@ -119,7 +119,7 @@
 	rdsv3_trans_init();
 	ret = rdsv3_init();
 	if (ret) {
-		RDSV3_DPRINTF1("rdsv3_attach", "rdsv3_init failed: %d", ret);
+		RDSV3_DPRINTF2("rdsv3_attach", "rdsv3_init failed: %d", ret);
 		rdsv3_trans_exit();
 		mutex_destroy(&rdsv3_rdma_listen_id_lock);
 		rdsv3_dev_info = NULL;
--- a/usr/src/uts/common/io/ib/clients/rdsv3/rdsv3_impl.c	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/io/ib/clients/rdsv3/rdsv3_impl.c	Thu May 06 09:54:45 2010 -0700
@@ -773,14 +773,14 @@
 	rdsv3_taskq = ddi_taskq_create(rdsv3_dev_info, name,
 	    RDSV3_NUM_TASKQ_THREADS, TASKQ_DEFAULTPRI, 0);
 	if (rdsv3_taskq == NULL) {
-		RDSV3_DPRINTF1(__FILE__,
+		RDSV3_DPRINTF2(__FILE__,
 		    "ddi_taskq_create failed for rdsv3_taskq");
 		return (NULL);
 	}
 
 	wq = kmem_zalloc(sizeof (rdsv3_workqueue_struct_t), KM_NOSLEEP);
 	if (wq == NULL) {
-		RDSV3_DPRINTF1(__FILE__, "kmem_zalloc failed for wq");
+		RDSV3_DPRINTF2(__FILE__, "kmem_zalloc failed for wq");
 		ddi_taskq_destroy(rdsv3_taskq);
 		return (NULL);
 	}
@@ -865,27 +865,6 @@
 	sk->sk_rcvbuf = RDSV3_RECV_HIWATER;
 }
 
-/* XXX - not complete */
-void
-rdsv3_poll_wait(struct rsock *sk, rdsv3_wait_queue_t *waitq, short events)
-{
-	struct rdsv3_sock *rs = rdsv3_sk_to_rs(sk);
-
-	if (events & POLLIN) {
-		rw_enter(&rs->rs_recv_lock, RW_READER);
-		while (list_is_empty(&rs->rs_recv_queue) &&
-		    list_is_empty(&rs->rs_notify_queue)) {
-			rw_exit(&rs->rs_recv_lock);
-			mutex_enter(&waitq->waitq_mutex);
-			(void) cv_wait_sig(&waitq->waitq_cv,
-			    &waitq->waitq_mutex);
-			mutex_exit(&waitq->waitq_mutex);
-			rw_enter(&rs->rs_recv_lock, RW_READER);
-		}
-		rw_exit(&rs->rs_recv_lock);
-	}
-}
-
 /*
  * Connection cache
  */
--- a/usr/src/uts/common/io/ib/clients/rdsv3/send.c	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/io/ib/clients/rdsv3/send.c	Thu May 06 09:54:45 2010 -0700
@@ -1009,7 +1009,7 @@
 
 	if ((rm->m_rdma_cookie || rm->m_rdma_op) &&
 	    conn->c_trans->xmit_rdma == NULL) {
-		RDSV3_DPRINTF0("rdsv3_sendmsg", "rdma_op %p conn xmit_rdma %p",
+		RDSV3_DPRINTF2("rdsv3_sendmsg", "rdma_op %p conn xmit_rdma %p",
 		    rm->m_rdma_op, conn->c_trans->xmit_rdma);
 		ret = -EOPNOTSUPP;
 		goto out;
@@ -1051,37 +1051,37 @@
 			goto out;
 		}
 
+#if 0
+		ret = rdsv3_wait_sig(sk->sk_sleep,
+		    (rdsv3_send_queue_rm(rs, conn, rm, rs->rs_bound_port,
+		    dport, &queued)));
+		if (ret == 0) {
+			/* signal/timeout pending */
+			RDSV3_DPRINTF2("rdsv3_sendmsg",
+			    "woke due to signal: %d", ret);
+			ret = -ERESTART;
+			goto out;
+		}
+#else
 		mutex_enter(&sk->sk_sleep->waitq_mutex);
+		sk->sk_sleep->waitq_waiters++;
 		while (!rdsv3_send_queue_rm(rs, conn, rm, rs->rs_bound_port,
 		    dport, &queued)) {
-#if 0
-			ret = cv_timedwait_sig(&sk->sk_sleep->waitq_cv,
-			    &sk->sk_sleep->waitq_mutex,
-			    timeo * drv_usectohz(1000000) + ddi_get_lbolt());
-			if (ret <= 0) {
-				/* signal/timeout pending */
-				RDSV3_DPRINTF2("rdsv3_sendmsg",
-				    "woke due to signal/timeout: %d",
-				    ret);
-				ret = (ret == 0) ? -ERESTART : -ETIMEDOUT;
-				mutex_exit(&sk->sk_sleep->waitq_mutex);
-				goto out;
-			}
-#else
 			ret = cv_wait_sig(&sk->sk_sleep->waitq_cv,
 			    &sk->sk_sleep->waitq_mutex);
 			if (ret == 0) {
 				/* signal/timeout pending */
 				RDSV3_DPRINTF2("rdsv3_sendmsg",
-				    "woke due to signal: %d",
-				    ret);
+				    "woke due to signal: %d", ret);
 				ret = -ERESTART;
+				sk->sk_sleep->waitq_waiters--;
 				mutex_exit(&sk->sk_sleep->waitq_mutex);
 				goto out;
 			}
+		}
+		sk->sk_sleep->waitq_waiters--;
+		mutex_exit(&sk->sk_sleep->waitq_mutex);
 #endif
-		}
-		mutex_exit(&sk->sk_sleep->waitq_mutex);
 
 		RDSV3_DPRINTF5("rdsv3_sendmsg", "sendmsg woke queued %d",
 		    queued);
--- a/usr/src/uts/common/io/ib/clients/rdsv3/threads.c	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/io/ib/clients/rdsv3/threads.c	Thu May 06 09:54:45 2010 -0700
@@ -103,7 +103,7 @@
 	if (!rdsv3_conn_transition(conn, RDSV3_CONN_CONNECTING,
 	    RDSV3_CONN_UP)) {
 #ifndef __lock_lint
-		RDSV3_DPRINTF0("rdsv3_connect_complete",
+		RDSV3_DPRINTF2("rdsv3_connect_complete",
 		    "%s: Cannot transition to state UP, "
 		    "current state is %d",
 		    __func__,
--- a/usr/src/uts/common/sys/ib/clients/rdsv3/rdsv3.h	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/sys/ib/clients/rdsv3/rdsv3.h	Thu May 06 09:54:45 2010 -0700
@@ -580,8 +580,6 @@
 void rdsv3_wake_sk_sleep(struct rdsv3_sock *rs);
 void __rdsv3_wake_sk_sleep(struct rsock *sk);
 
-extern rdsv3_wait_queue_t rdsv3_poll_waitq;
-
 /* bind.c */
 int rdsv3_bind(sock_lower_handle_t proto_handle, struct sockaddr *sa,
     socklen_t len, cred_t *cr);
--- a/usr/src/uts/common/sys/ib/clients/rdsv3/rdsv3_debug.h	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/sys/ib/clients/rdsv3/rdsv3_debug.h	Thu May 06 09:54:45 2010 -0700
@@ -82,9 +82,6 @@
 #define	RDSV3_DPRINTF5		rdsv3_dprintf5
 #define	RDSV3_DPRINTF4		rdsv3_dprintf4
 #define	RDSV3_DPRINTF3		rdsv3_dprintf3
-#define	RDSV3_DPRINTF2		rdsv3_dprintf2
-#define	RDSV3_DPRINTF1		rdsv3_dprintf1
-#define	RDSV3_DPRINTF0		rdsv3_dprintf0
 
 void rdsv3_dprintf_intr(
 		char		*name,
@@ -98,6 +95,17 @@
 void rdsv3_dprintf3(
 		char		*name,
 		char		*fmt, ...);
+#else
+#define	RDSV3_DPRINTF_INTR	0 &&
+#define	RDSV3_DPRINTF5		0 &&
+#define	RDSV3_DPRINTF4		0 &&
+#define	RDSV3_DPRINTF3		0 &&
+#endif
+
+#define	RDSV3_DPRINTF2		rdsv3_dprintf2
+#define	RDSV3_DPRINTF1		rdsv3_dprintf1
+#define	RDSV3_DPRINTF0		rdsv3_dprintf0
+
 void rdsv3_dprintf2(
 		char		*name,
 		char		*fmt, ...);
@@ -107,15 +115,6 @@
 void rdsv3_dprintf0(
 		char		*name,
 		char		*fmt, ...);
-#else
-#define	RDSV3_DPRINTF_INTR	0 &&
-#define	RDSV3_DPRINTF5		0 &&
-#define	RDSV3_DPRINTF4		0 &&
-#define	RDSV3_DPRINTF3		0 &&
-#define	RDSV3_DPRINTF2		0 &&
-#define	RDSV3_DPRINTF1		0 &&
-#define	RDSV3_DPRINTF0		0 &&
-#endif
 
 void rdsv3_trace(
 		char		*name,
--- a/usr/src/uts/common/sys/ib/clients/rdsv3/rdsv3_impl.h	Thu May 06 09:06:40 2010 -0700
+++ b/usr/src/uts/common/sys/ib/clients/rdsv3/rdsv3_impl.h	Thu May 06 09:54:45 2010 -0700
@@ -159,61 +159,64 @@
 typedef struct rdsv3_wait_queue_s {
 	kmutex_t	waitq_mutex;
 	kcondvar_t	waitq_cv;
+	uint_t		waitq_waiters;
 } rdsv3_wait_queue_t;
 
 #define	rdsv3_init_waitqueue(waitqp)					\
 	mutex_init(&(waitqp)->waitq_mutex, NULL, MUTEX_DRIVER, NULL);	\
-	cv_init(&(waitqp)->waitq_cv, NULL, CV_DRIVER, NULL)
+	cv_init(&(waitqp)->waitq_cv, NULL, CV_DRIVER, NULL);		\
+	(waitqp)->waitq_waiters = 0
 
 #define	rdsv3_exit_waitqueue(waitqp)					\
+	ASSERT((waitqp)->waitq_waiters == 0);				\
 	mutex_destroy(&(waitqp)->waitq_mutex);				\
 	cv_destroy(&(waitqp)->waitq_cv)
 
 #define	rdsv3_wake_up(waitqp)	{					\
 	mutex_enter(&(waitqp)->waitq_mutex);				\
-	cv_signal(&(waitqp)->waitq_cv);					\
+	if ((waitqp)->waitq_waiters)					\
+		cv_signal(&(waitqp)->waitq_cv);				\
 	mutex_exit(&(waitqp)->waitq_mutex);				\
 	}
 
 #define	rdsv3_wake_up_all(waitqp)	{				\
 	mutex_enter(&(waitqp)->waitq_mutex);				\
-	cv_broadcast(&(waitqp)->waitq_cv);				\
+	if ((waitqp)->waitq_waiters)					\
+		cv_broadcast(&(waitqp)->waitq_cv);			\
 	mutex_exit(&(waitqp)->waitq_mutex);				\
 	}
 
+/* analogous to cv_wait */
 #define	rdsv3_wait_event(waitq, condition)				\
 {									\
-	mutex_enter(&(waitq).waitq_mutex);				\
+	mutex_enter(&(waitq)->waitq_mutex);				\
+	(waitq)->waitq_waiters++;					\
 	while (!(condition)) {						\
-		cv_wait(&(waitq).waitq_cv, &(waitq).waitq_mutex);	\
+		cv_wait(&(waitq)->waitq_cv, &(waitq)->waitq_mutex);	\
 	}								\
-	mutex_exit(&(waitq).waitq_mutex);				\
-}									\
+	(waitq)->waitq_waiters--;					\
+	mutex_exit(&(waitq)->waitq_mutex);				\
+}
 
-#ifndef __lock_lint
-#define	rdsv3_wait_event_interruptible_timeout(waitq, condition, timeo)	\
+/* analogous to cv_wait_sig */
+#define	rdsv3_wait_sig(waitqp, condition)				\
 (									\
 {									\
-	long cv_return;							\
-	mutex_enter(&((waitq).waitq_mutex));				\
-	cv_return = condition;						\
-	while (!(cv_return)) {						\
-		cv_return = cv_timedwait_sig(&((waitq).waitq_cv),	\
-		    &((waitq).waitq_mutex),				\
-		    timeo * drv_usectohz(1000000) + ddi_get_lbolt());	\
+	int cv_return = 1;						\
+	mutex_enter(&(waitqp)->waitq_mutex);				\
+	(waitqp)->waitq_waiters++;					\
+	while (!(condition)) {						\
+		cv_return = cv_wait_sig(&(waitqp)->waitq_cv,		\
+		    &(waitqp)->waitq_mutex);				\
 		if (cv_return == 0) {					\
 			break;						\
 		}							\
-		cv_return = condition;					\
 	}								\
-	mutex_exit(&((waitq).waitq_mutex));				\
+	(waitqp)->waitq_waiters--;					\
+	mutex_exit(&(waitqp)->waitq_mutex);				\
 	cv_return;							\
 }									\
 )
-#else
-#define	rdsv3_wait_event_interruptible(waitq, condition)		0
-#define	rdsv3_wait_event_interruptible_timeout(waitq, condition, timeo)	0
-#endif
 
 #define	SOCK_DEAD	1ul
 
@@ -224,7 +227,7 @@
 
 	kmutex_t		sk_lock;
 	ulong_t			sk_flag;
-	rdsv3_wait_queue_t	*sk_sleep;
+	rdsv3_wait_queue_t	*sk_sleep; /* Also protected by rs_recv_lock */
 	int			sk_sndbuf;
 	int			sk_rcvbuf;
 	atomic_t		sk_refcount;
@@ -350,7 +353,6 @@
 struct rsock *rdsv3_sk_alloc();
 void rdsv3_sock_init_data(struct rsock *sk);
 void rdsv3_sock_exit_data(struct rsock *sk);
-void rdsv3_poll_wait(struct rsock *sk, rdsv3_wait_queue_t *waitq, short events);
 void rdsv3_destroy_task_workqueue(rdsv3_workqueue_struct_t *wq);
 rdsv3_workqueue_struct_t *rdsv3_create_task_workqueue(char *name);
 int rdsv3_conn_constructor(void *buf, void *arg, int kmflags);