|
1 /* |
|
2 * CDDL HEADER START |
|
3 * |
|
4 * The contents of this file are subject to the terms of the |
|
5 * Common Development and Distribution License (the "License"). |
|
6 * You may not use this file except in compliance with the License. |
|
7 * |
|
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE |
|
9 * or http://www.opensolaris.org/os/licensing. |
|
10 * See the License for the specific language governing permissions |
|
11 * and limitations under the License. |
|
12 * |
|
13 * When distributing Covered Code, include this CDDL HEADER in each |
|
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. |
|
15 * If applicable, add the following below this CDDL HEADER, with the |
|
16 * fields enclosed by brackets "[]" replaced with your own identifying |
|
17 * information: Portions Copyright [yyyy] [name of copyright owner] |
|
18 * |
|
19 * CDDL HEADER END |
|
20 */ |
|
21 /* |
|
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. |
|
23 * Use is subject to license terms. |
|
24 */ |
|
25 |
|
26 #ifndef _ISER_RESOURCE_H |
|
27 #define _ISER_RESOURCE_H |
|
28 |
|
29 #ifdef __cplusplus |
|
30 extern "C" { |
|
31 #endif |
|
32 |
|
33 #include <sys/types.h> |
|
34 #include <sys/ib/ibtl/ibti.h> |
|
35 #include <sys/ib/ibtl/ibtl_types.h> |
|
36 #include <sys/iscsi_protocol.h> |
|
37 |
|
38 #define ISER_CACHE_NAMELEN 31 /* KMEM_CACHE_NAMELEN */ |
|
39 |
|
40 /* Default message lengths */ |
|
41 #define ISER_MAX_CTRLPDU_LEN 0x4000 |
|
42 #define ISER_MAX_TEXTPDU_LEN 0x4000 |
|
43 |
|
44 /* Default data buffer length */ |
|
45 #define ISER_DEFAULT_BUFLEN 0x20000 |
|
46 |
|
47 /* |
|
48 * iser_resource.h |
|
49 * Definitions and functions related to set up buffer allocation from |
|
50 * IBT memory regions and managment of work requessts. |
|
51 */ |
|
52 |
|
53 struct iser_hca_s; |
|
54 |
|
55 /* |
|
56 * Memory regions |
|
57 */ |
|
58 typedef struct iser_mr_s { |
|
59 ibt_mr_hdl_t is_mrhdl; |
|
60 ib_vaddr_t is_mrva; |
|
61 ib_memlen_t is_mrlen; |
|
62 ibt_lkey_t is_mrlkey; |
|
63 ibt_rkey_t is_mrrkey; |
|
64 avl_node_t is_avl_ln; |
|
65 } iser_mr_t; |
|
66 |
|
67 typedef struct iser_vmem_mr_pool_s { |
|
68 iser_hca_t *ivmp_hca; |
|
69 ibt_mr_flags_t ivmp_mr_flags; |
|
70 ib_memlen_t ivmp_chunksize; |
|
71 vmem_t *ivmp_vmem; |
|
72 uint64_t ivmp_total_size; |
|
73 uint64_t ivmp_max_total_size; |
|
74 avl_tree_t ivmp_mr_list; |
|
75 kmutex_t ivmp_mutex; |
|
76 } iser_vmem_mr_pool_t; |
|
77 |
|
78 #define ISER_MR_QUANTSIZE 0x400 |
|
79 #define ISER_MIN_CHUNKSIZE 0x100000 /* 1MB */ |
|
80 |
|
81 #ifdef _LP64 |
|
82 #define ISER_BUF_MR_CHUNKSIZE 0x8000000 /* 128MB */ |
|
83 #define ISER_BUF_POOL_MAX 0x40000000 /* 1GB */ |
|
84 #else |
|
85 /* Memory is very limited on 32-bit kernels */ |
|
86 #define ISER_BUF_MR_CHUNKSIZE 0x400000 /* 4MB */ |
|
87 #define ISER_BUF_POOL_MAX 0x4000000 /* 64MB */ |
|
88 #endif |
|
89 #define ISER_BUF_MR_FLAGS IBT_MR_ENABLE_LOCAL_WRITE | \ |
|
90 IBT_MR_ENABLE_REMOTE_READ | IBT_MR_ENABLE_REMOTE_WRITE |
|
91 #ifdef _LP64 |
|
92 #define ISER_MSG_MR_CHUNKSIZE 0x2000000 /* 32MB */ |
|
93 #define ISER_MSG_POOL_MAX 0x10000000 /* 256MB */ |
|
94 #else |
|
95 #define ISER_MSG_MR_CHUNKSIZE 0x100000 /* 1MB */ |
|
96 #define ISER_MSG_POOL_MAX 0x2000000 /* 32MB */ |
|
97 #endif |
|
98 #define ISER_MSG_MR_FLAGS IBT_MR_ENABLE_LOCAL_WRITE |
|
99 |
|
100 iser_vmem_mr_pool_t *iser_vmem_create(const char *name, iser_hca_t *hca, |
|
101 ib_memlen_t chunksize, uint64_t max_total_size, |
|
102 ibt_mr_flags_t arena_mr_flags); |
|
103 void iser_vmem_destroy(iser_vmem_mr_pool_t *vmr_pool); |
|
104 void *iser_vmem_alloc(iser_vmem_mr_pool_t *vmr_pool, size_t size); |
|
105 void iser_vmem_free(iser_vmem_mr_pool_t *vmr_pool, void *vaddr, size_t size); |
|
106 idm_status_t iser_vmem_mr(iser_vmem_mr_pool_t *vmr_pool, |
|
107 void *vaddr, size_t size, iser_mr_t *mr); |
|
108 |
|
109 /* |
|
110 * iSER work request structure encodes an iSER Send Queue work request |
|
111 * context, with pointers to relevant resources related to the work request. |
|
112 * We hold a pointer to either an IDM PDU handle, an iSER message handle |
|
113 * or an IDM buffer handle. These are allocated from a kmem_cache when |
|
114 * we post send WR's, and freed back when the completion is polled. |
|
115 */ |
|
116 typedef enum { |
|
117 ISER_WR_SEND, |
|
118 ISER_WR_RDMAW, |
|
119 ISER_WR_RDMAR, |
|
120 ISER_WR_UNDEFINED |
|
121 } iser_wr_type_t; |
|
122 |
|
123 typedef struct iser_wr_s { |
|
124 iser_wr_type_t iw_type; |
|
125 struct iser_msg_s *iw_msg; |
|
126 struct idm_buf_s *iw_buf; |
|
127 struct idm_pdu_s *iw_pdu; |
|
128 } iser_wr_t; |
|
129 |
|
130 int iser_wr_cache_constructor(void *mr, void *arg, int flags); |
|
131 void iser_wr_cache_destructor(void *mr, void *arg); |
|
132 iser_wr_t *iser_wr_get(); |
|
133 void iser_wr_free(iser_wr_t *iser_wr); |
|
134 |
|
135 /* |
|
136 * iSER message structure for iSCSI Control PDUs, constructor and |
|
137 * destructor routines, and utility routines for allocating and |
|
138 * freeing message handles. |
|
139 */ |
|
140 typedef struct iser_msg_s { |
|
141 struct iser_msg_s *nextp; /* for building lists */ |
|
142 kmem_cache_t *cache; /* back pointer for cleanup */ |
|
143 ibt_wr_ds_t msg_ds; /* SGEs for hdr and text */ |
|
144 ibt_mr_hdl_t mrhdl[2]; /* MR handles for each SGE */ |
|
145 } iser_msg_t; |
|
146 |
|
147 int iser_msg_cache_constructor(void *mr, void *arg, int flags); |
|
148 void iser_msg_cache_destructor(void *mr, void *arg); |
|
149 iser_msg_t *iser_msg_get(iser_hca_t *hca, int num, int *ret); |
|
150 void iser_msg_free(iser_msg_t *msg); |
|
151 |
|
152 /* |
|
153 * iSER data buffer structure for iSER RDMA operations, constructor and |
|
154 * destructor routines, and utility routines for allocating and freeing |
|
155 * buffer handles. |
|
156 */ |
|
157 typedef struct iser_buf_s { |
|
158 kmem_cache_t *cache; /* back pointer for cleanup */ |
|
159 void *buf; /* buffer */ |
|
160 uint64_t buflen; |
|
161 iser_mr_t *iser_mr; /* MR handle for this buffer */ |
|
162 ibt_wr_ds_t buf_ds; /* SGE for this buffer */ |
|
163 ibt_send_wr_t buf_wr; /* DEBUG, copy of wr from request */ |
|
164 ibt_wc_t buf_wc; /* DEBUG, copy of wc from completion */ |
|
165 timespec_t buf_constructed; |
|
166 timespec_t buf_destructed; |
|
167 } iser_buf_t; |
|
168 |
|
169 int iser_buf_cache_constructor(void *mr, void *arg, int flags); |
|
170 void iser_buf_cache_destructor(void *mr, void *arg); |
|
171 |
|
172 void iser_init_hca_caches(struct iser_hca_s *hca); |
|
173 void iser_fini_hca_caches(struct iser_hca_s *hca); |
|
174 |
|
175 /* Routines to register in-place memory passed on an existing idb */ |
|
176 int iser_reg_rdma_mem(struct iser_hca_s *hca, idm_buf_t *idb); |
|
177 void iser_dereg_rdma_mem(struct iser_hca_s *hca, idm_buf_t *idb); |
|
178 |
|
179 #ifdef __cplusplus |
|
180 } |
|
181 #endif |
|
182 |
|
183 #endif /* _ISER_RESOURCE_H */ |