1 /*

     2  * CDDL HEADER START

     3  *

     4  * The contents of this file are subject to the terms of the

     5  * Common Development and Distribution License (the "License").

     6  * You may not use this file except in compliance with the License.

     7  *

     8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

     9  * or http://www.opensolaris.org/os/licensing.

    10  * See the License for the specific language governing permissions

    11  * and limitations under the License.

    12  *

    13  * When distributing Covered Code, include this CDDL HEADER in each

    14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.

    15  * If applicable, add the following below this CDDL HEADER, with the

    16  * fields enclosed by brackets "[]" replaced with your own identifying

    17  * information: Portions Copyright [yyyy] [name of copyright owner]

    18  *

    19  * CDDL HEADER END

    20  */

    21 /*

    22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.

    23  * Use is subject to license terms.

    24  */

    25 

    26 #ifndef _ISER_IB_H

    27 #define	_ISER_IB_H

    28 

    29 #ifdef	__cplusplus

    30 extern "C" {

    31 #endif

    32 

    33 #include <sys/types.h>

    34 #include <sys/socket.h>

    35 #include <netinet/in.h>

    36 #include <sys/ib/ibtl/ibti.h>

    37 #include <sys/iscsi_protocol.h>

    38 

    39 /*

    40  * iser_ib.h

    41  *	Definitions and macros related to iSER InfiniBand transport,

    42  * 	and the use of the Solaris IBTI (InfiniBand Transport Interface).

    43  */

    44 

    45 struct iser_state_s;

    46 

    47 extern struct iser_state_s	*iser_state;

    48 extern ddi_taskq_t	*iser_taskq;

    49 

    50 /*

    51  * iser_hca_s holds all the information about the Infinband HCAs in use.

    52  */

    53 typedef struct iser_hca_s {

    54 	list_node_t		hca_node;

    55 	boolean_t		hca_failed;

    56 	ibt_clnt_hdl_t		hca_clnt_hdl;

    57 	ibt_hca_hdl_t		hca_hdl;

    58 	ibt_hca_attr_t		hca_attr;

    59 	ibt_pd_hdl_t		hca_pdhdl;

    60 	ib_guid_t		hca_guid;

    61 	uint_t			hca_num_ports;

    62 	ibt_hca_portinfo_t	*hca_port_info;

    63 	uint_t			hca_port_info_sz;

    64 

    65 	/* Per PD (per HCA) message and data buffer caches */

    66 	struct iser_vmem_mr_pool_s *hca_msg_pool; /* Use iser_msg_cache */

    67 	kmem_cache_t		*iser_msg_cache;

    68 	struct iser_vmem_mr_pool_s *hca_buf_pool; /* Use iser_buf_cache */

    69 	kmem_cache_t		*iser_buf_cache;

    70 } iser_hca_t;

    71 

    72 /* RQ low water mark percentage */

    73 #define	ISER_IB_RQ_LWM_PCT	80

    74 

    75 /* Maximum number of WRs to post on the RQ at a time */

    76 #define	ISER_IB_RQ_POST_MAX	64

    77 

    78 /* Maximum number of SCQ WCs to poll at a time */

    79 #define	ISER_IB_SCQ_POLL_MAX	8

    80 

    81 /*

    82  * iser_qp_t encodes data related to a Queue Pair (QP) in use by

    83  * iSER. Each QP consists of two Work Queues (WQs), one Send Queue

    84  * (SQ) and on Receive Queue (RQ). Most of the data in the QP

    85  * handle relates to monitoring the posted depth of the RQ.

    86  *

    87  * Note that we are explicitly using slightly less than a power-of-2

    88  * number for our queue sizes.  The HCA driver will round up for us,

    89  * and this affords us some headroom.

    90  */

    91 #ifdef _LP64

    92 #define	ISER_IB_RECVQ_SIZE	400

    93 #else

    94 /* Memory is very limited on 32-bit kernels */

    95 #define	ISER_IB_RECVQ_SIZE	100

    96 #endif

    97 #define	ISER_IB_SENDQ_SIZE	2000

    98 #define	ISER_IB_SGLIST_SIZE	1

    99 

   100 #define	ISER_IB_DEFAULT_IRD	2

   101 #define	ISER_IB_DEFAULT_ORD	4

   102 

   103 typedef struct iser_qp_s {

   104 	kmutex_t	qp_lock;

   105 	uint_t		sq_size;

   106 	uint_t		rq_size;

   107 	uint32_t	rq_depth;

   108 	uint32_t	rq_level;

   109 	uint32_t	rq_min_post_level;

   110 	uint32_t	rq_lwm;

   111 	boolean_t	rq_taskqpending;

   112 } iser_qp_t;

   113 

   114 /*

   115  * iSER RC channel information

   116  */

   117 typedef struct iser_chan_s {

   118 	kmutex_t		ic_lock;

   119 

   120 	/* IBT channel handle */

   121 	ibt_channel_hdl_t	ic_chanhdl;

   122 

   123 	/* local and remote IP addresses and port numbers */

   124 	ibt_ip_addr_t		ic_localip;

   125 	ibt_ip_addr_t		ic_remoteip;

   126 	in_port_t		ic_lport;

   127 	in_port_t		ic_rport;

   128 

   129 	/*

   130 	 * The local HCA GUID, the service ID, Destination GID, Source GID

   131 	 * the primary hca port on which the channel is connected is

   132 	 * stored in ic_ibt_path

   133 	 */

   134 	ibt_path_info_t		ic_ibt_path;

   135 

   136 	/*

   137 	 * Information related to the HCA handle and the queues.

   138 	 */

   139 	iser_hca_t		*ic_hca;

   140 	ibt_cq_hdl_t		ic_sendcq;

   141 	ibt_cq_hdl_t		ic_recvcq;

   142 	uint_t			ic_sendcq_sz;

   143 	uint_t			ic_recvcq_sz;

   144 	iser_qp_t		ic_qp;

   145 

   146 	/* Used to track the number of WRs posted on the SQ */

   147 	kmutex_t		ic_sq_post_lock;

   148 	uint_t			ic_sq_post_count;

   149 	uint_t			ic_sq_max_post_count;

   150 

   151 	/*

   152 	 * To help identify the channel end point and some connection

   153 	 * specifics, maintain a pointer to the connection on which

   154 	 * this channel originated

   155 	 */

   156 	struct iser_conn_s	*ic_conn;

   157 } iser_chan_t;

   158 

   159 int iser_ib_init(void);

   160 

   161 int iser_ib_fini(void);

   162 

   163 int iser_ib_register_service(idm_svc_t *idm_svc);

   164 

   165 int iser_ib_bind_service(idm_svc_t *idm_svc);

   166 

   167 void iser_ib_unbind_service(idm_svc_t *idm_svc);

   168 

   169 void iser_ib_deregister_service(idm_svc_t *idm_svc);

   170 

   171 void iser_ib_conv_sockaddr2ibtaddr(idm_sockaddr_t *saddr,

   172     ibt_ip_addr_t *ibt_addr);

   173 

   174 void iser_ib_conv_ibtaddr2sockaddr(struct sockaddr_storage *ss,

   175     ibt_ip_addr_t *ibt_addr, in_port_t port);

   176 

   177 int iser_ib_get_paths(

   178     ibt_ip_addr_t *local_ip, ibt_ip_addr_t *remote_ip, ibt_path_info_t *path,

   179     ibt_path_ip_src_t *path_src_ip);

   180 

   181 iser_chan_t *iser_ib_alloc_rc_channel(ibt_ip_addr_t *local_ip,

   182     ibt_ip_addr_t *remote_ip);

   183 

   184 int iser_ib_open_rc_channel(iser_chan_t *chan);

   185 

   186 void iser_ib_close_rc_channel(iser_chan_t *chan);

   187 

   188 void iser_ib_free_rc_channel(iser_chan_t *chan);

   189 

   190 void iser_ib_post_recv(void *arg);

   191 

   192 void iser_ib_recvcq_handler(ibt_cq_hdl_t cq_hdl, void *arg);

   193 

   194 void iser_ib_sendcq_handler(ibt_cq_hdl_t cq_hdl, void *arg);

   195 

   196 void iser_ib_async_handler(void *clntp, ibt_hca_hdl_t hdl,

   197     ibt_async_code_t code, ibt_async_event_t *event);

   198 

   199 #ifdef	__cplusplus

   200 }

   201 #endif

   202 

   203 #endif /* _ISER_IB_H */