17 * information: Portions Copyright [yyyy] [name of copyright owner] |
17 * information: Portions Copyright [yyyy] [name of copyright owner] |
18 * |
18 * |
19 * CDDL HEADER END |
19 * CDDL HEADER END |
20 */ |
20 */ |
21 /* |
21 /* |
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. |
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. |
23 * Use is subject to license terms. |
23 * Use is subject to license terms. |
24 */ |
24 */ |
25 |
25 |
26 /* |
26 /* |
27 * Copyright (c) 2007, The Ohio State University. All rights reserved. |
27 * Copyright (c) 2008, The Ohio State University. All rights reserved. |
28 * |
28 * |
29 * Portions of this source code is developed by the team members of |
29 * Portions of this source code is developed by the team members of |
30 * The Ohio State University's Network-Based Computing Laboratory (NBCL), |
30 * The Ohio State University's Network-Based Computing Laboratory (NBCL), |
31 * headed by Professor Dhabaleswar K. (DK) Panda. |
31 * headed by Professor Dhabaleswar K. (DK) Panda. |
32 * |
32 * |
51 * Globals |
51 * Globals |
52 */ |
52 */ |
53 int rdma_modloaded = 0; /* flag to load RDMA plugin modules */ |
53 int rdma_modloaded = 0; /* flag to load RDMA plugin modules */ |
54 int rdma_dev_available = 0; /* if any RDMA device is loaded */ |
54 int rdma_dev_available = 0; /* if any RDMA device is loaded */ |
55 kmutex_t rdma_modload_lock; /* protects rdma_modloaded flag */ |
55 kmutex_t rdma_modload_lock; /* protects rdma_modloaded flag */ |
|
56 |
|
57 rdma_svc_wait_t rdma_wait; |
|
58 |
56 rdma_registry_t *rdma_mod_head = NULL; /* head for RDMA modules */ |
59 rdma_registry_t *rdma_mod_head = NULL; /* head for RDMA modules */ |
57 krwlock_t rdma_lock; /* protects rdma_mod_head list */ |
60 krwlock_t rdma_lock; /* protects rdma_mod_head list */ |
58 ldi_ident_t rpcmod_li = NULL; /* identifies us with ldi_ framework */ |
61 ldi_ident_t rpcmod_li = NULL; /* identifies us with ldi_ framework */ |
59 |
62 |
60 kmem_cache_t *clist_cache = NULL; |
63 kmem_cache_t *clist_cache = NULL; |
61 |
64 |
62 /* |
65 /* |
63 * Statics |
66 * Statics |
64 */ |
67 */ |
65 static ldi_handle_t rpcib_handle = NULL; |
68 ldi_handle_t rpcib_handle = NULL; |
66 |
69 |
67 /* |
70 /* |
68 * Externs |
71 * Externs |
69 */ |
72 */ |
70 extern kstat_named_t *rdmarcstat_ptr; |
73 extern kstat_named_t *rdmarcstat_ptr; |
94 */ |
97 */ |
95 mp = &rdma_mod_head; |
98 mp = &rdma_mod_head; |
96 while (*mp != NULL) { |
99 while (*mp != NULL) { |
97 if (strncmp((*mp)->r_mod->rdma_api, mod->rdma_api, |
100 if (strncmp((*mp)->r_mod->rdma_api, mod->rdma_api, |
98 KNC_STRSIZE) == 0) { |
101 KNC_STRSIZE) == 0) { |
|
102 if ((*mp)->r_mod_state == RDMA_MOD_INACTIVE) { |
|
103 (*mp)->r_mod_state = RDMA_MOD_ACTIVE; |
|
104 (*mp)->r_mod->rdma_ops = mod->rdma_ops; |
|
105 (*mp)->r_mod->rdma_count = mod->rdma_count; |
|
106 goto announce_hca; |
|
107 } |
99 rw_exit(&rdma_lock); |
108 rw_exit(&rdma_lock); |
100 return (RDMA_REG_EXIST); |
109 return (RDMA_REG_EXIST); |
101 } |
110 } |
102 mp = &((*mp)->r_next); |
111 mp = &((*mp)->r_next); |
103 } |
112 } |
110 *m->r_mod = *mod; |
119 *m->r_mod = *mod; |
111 m->r_next = NULL; |
120 m->r_next = NULL; |
112 m->r_mod->rdma_api = kmem_zalloc(KNC_STRSIZE, KM_SLEEP); |
121 m->r_mod->rdma_api = kmem_zalloc(KNC_STRSIZE, KM_SLEEP); |
113 (void) strncpy(m->r_mod->rdma_api, mod->rdma_api, KNC_STRSIZE); |
122 (void) strncpy(m->r_mod->rdma_api, mod->rdma_api, KNC_STRSIZE); |
114 m->r_mod->rdma_api[KNC_STRSIZE - 1] = '\0'; |
123 m->r_mod->rdma_api[KNC_STRSIZE - 1] = '\0'; |
|
124 m->r_mod_state = RDMA_MOD_ACTIVE; |
115 *mp = m; |
125 *mp = m; |
|
126 |
|
127 announce_hca: |
116 rw_exit(&rdma_lock); |
128 rw_exit(&rdma_lock); |
|
129 /* |
|
130 * Start the nfs service on the rdma xprts. |
|
131 * (this notification mechanism will need to change when we support |
|
132 * multiple hcas and have support for multiple rdma plugins). |
|
133 */ |
|
134 mutex_enter(&rdma_wait.svc_lock); |
|
135 rdma_wait.svc_stat = RDMA_HCA_ATTACH; |
|
136 cv_signal(&rdma_wait.svc_cv); |
|
137 mutex_exit(&rdma_wait.svc_lock); |
117 |
138 |
118 return (RDMA_SUCCESS); |
139 return (RDMA_SUCCESS); |
119 } |
140 } |
120 |
141 |
121 /* |
142 /* |
138 continue; |
159 continue; |
139 } |
160 } |
140 /* |
161 /* |
141 * Check if any device attached, if so return error |
162 * Check if any device attached, if so return error |
142 */ |
163 */ |
143 if ((*m)->r_mod->rdma_count != 0) { |
164 if (mod->rdma_count != 0) { |
144 rw_exit(&rdma_lock); |
165 rw_exit(&rdma_lock); |
145 return (RDMA_FAILED); |
166 return (RDMA_FAILED); |
146 } |
167 } |
147 /* |
168 /* |
148 * Found entry. Now remove it. |
169 * Found entry. Mark it inactive. |
149 */ |
170 */ |
150 mmod = *m; |
171 mmod = *m; |
151 *m = (*m)->r_next; |
172 mmod->r_mod->rdma_count = 0; |
152 kmem_free(mmod->r_mod->rdma_api, KNC_STRSIZE); |
173 mmod->r_mod_state = RDMA_MOD_INACTIVE; |
153 kmem_free(mmod->r_mod, sizeof (rdma_mod_t)); |
174 break; |
154 kmem_free(mmod, sizeof (rdma_registry_t)); |
175 } |
155 rw_exit(&rdma_lock); |
176 |
156 return (RDMA_SUCCESS); |
177 rdma_modloaded = 0; |
157 } |
178 rdma_dev_available = 0; |
|
179 rw_exit(&rdma_lock); |
|
180 |
|
181 /* |
|
182 * Stop the nfs service running on the rdma xprts. |
|
183 * (this notification mechanism will need to change when we support |
|
184 * multiple hcas and have support for multiple rdma plugins). |
|
185 */ |
|
186 mutex_enter(&rdma_wait.svc_lock); |
|
187 rdma_wait.svc_stat = RDMA_HCA_DETACH; |
|
188 cv_signal(&rdma_wait.svc_cv); |
|
189 mutex_exit(&rdma_wait.svc_lock); |
158 |
190 |
159 /* |
191 /* |
160 * Not found. |
192 * Not found. |
161 */ |
193 */ |
162 rw_exit(&rdma_lock); |
194 return (RDMA_SUCCESS); |
163 return (RDMA_FAILED); |
|
164 } |
195 } |
165 |
196 |
166 struct clist * |
197 struct clist * |
167 clist_alloc(void) |
198 clist_alloc(void) |
168 { |
199 { |
421 return (EPROTONOSUPPORT); |
452 return (EPROTONOSUPPORT); |
422 |
453 |
423 status = ldi_open_by_name("/devices/ib/rpcib@0:rpcib", |
454 status = ldi_open_by_name("/devices/ib/rpcib@0:rpcib", |
424 FREAD | FWRITE, kcred, |
455 FREAD | FWRITE, kcred, |
425 &rpcib_handle, rpcmod_li); |
456 &rpcib_handle, rpcmod_li); |
|
457 |
426 if (status != 0) |
458 if (status != 0) |
427 return (EPROTONOSUPPORT); |
459 return (EPROTONOSUPPORT); |
428 |
460 |
429 /* success */ |
461 |
430 rdma_kstat_init(); |
462 /* |
431 |
463 * We will need to reload the plugin module after it was unregistered |
432 clist_cache = kmem_cache_create("rdma_clist", |
464 * but the resources below need to allocated only the first time. |
433 sizeof (struct clist), _POINTER_ALIGNMENT, NULL, |
465 */ |
434 NULL, NULL, NULL, 0, 0); |
466 if (!clist_cache) { |
|
467 clist_cache = kmem_cache_create("rdma_clist", |
|
468 sizeof (struct clist), _POINTER_ALIGNMENT, NULL, |
|
469 NULL, NULL, NULL, 0, 0); |
|
470 rdma_kstat_init(); |
|
471 } |
|
472 |
|
473 (void) ldi_close(rpcib_handle, FREAD|FWRITE, kcred); |
435 |
474 |
436 return (0); |
475 return (0); |
437 } |
476 } |
438 |
477 |
439 void |
478 void |