author | rk129064 |
Mon, 29 Oct 2007 14:07:16 -0700 | |
changeset 5360 | 96c808cd96bc |
parent 3932 | efce29b04ab4 |
child 5753 | d64b1f799526 |
permissions | -rw-r--r-- |
0 | 1 |
/* |
2 |
* CDDL HEADER START |
|
3 |
* |
|
4 |
* The contents of this file are subject to the terms of the |
|
2497
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
5 |
* Common Development and Distribution License (the "License"). |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
6 |
* You may not use this file except in compliance with the License. |
0 | 7 |
* |
8 |
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE |
|
9 |
* or http://www.opensolaris.org/os/licensing. |
|
10 |
* See the License for the specific language governing permissions |
|
11 |
* and limitations under the License. |
|
12 |
* |
|
13 |
* When distributing Covered Code, include this CDDL HEADER in each |
|
14 |
* file and include the License file at usr/src/OPENSOLARIS.LICENSE. |
|
15 |
* If applicable, add the following below this CDDL HEADER, with the |
|
16 |
* fields enclosed by brackets "[]" replaced with your own identifying |
|
17 |
* information: Portions Copyright [yyyy] [name of copyright owner] |
|
18 |
* |
|
19 |
* CDDL HEADER END |
|
20 |
*/ |
|
21 |
/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ |
|
22 |
/* All Rights Reserved */ |
|
23 |
||
24 |
||
25 |
/* |
|
3448 | 26 |
* Copyright 2007 Sun Microsystems, Inc. All rights reserved. |
0 | 27 |
* Use is subject to license terms. |
28 |
*/ |
|
29 |
||
30 |
#pragma ident "%Z%%M% %I% %E% SMI" |
|
31 |
||
32 |
#include <sys/types.h> |
|
33 |
#include <sys/sysmacros.h> |
|
34 |
#include <sys/param.h> |
|
35 |
#include <sys/errno.h> |
|
36 |
#include <sys/signal.h> |
|
37 |
#include <sys/proc.h> |
|
38 |
#include <sys/conf.h> |
|
39 |
#include <sys/cred.h> |
|
40 |
#include <sys/user.h> |
|
41 |
#include <sys/vnode.h> |
|
42 |
#include <sys/file.h> |
|
43 |
#include <sys/session.h> |
|
44 |
#include <sys/stream.h> |
|
45 |
#include <sys/strsubr.h> |
|
46 |
#include <sys/stropts.h> |
|
47 |
#include <sys/poll.h> |
|
48 |
#include <sys/systm.h> |
|
49 |
#include <sys/cpuvar.h> |
|
50 |
#include <sys/uio.h> |
|
51 |
#include <sys/cmn_err.h> |
|
52 |
#include <sys/priocntl.h> |
|
53 |
#include <sys/procset.h> |
|
54 |
#include <sys/vmem.h> |
|
55 |
#include <sys/bitmap.h> |
|
56 |
#include <sys/kmem.h> |
|
57 |
#include <sys/siginfo.h> |
|
58 |
#include <sys/vtrace.h> |
|
59 |
#include <sys/callb.h> |
|
60 |
#include <sys/debug.h> |
|
61 |
#include <sys/modctl.h> |
|
62 |
#include <sys/vmsystm.h> |
|
63 |
#include <vm/page.h> |
|
64 |
#include <sys/atomic.h> |
|
65 |
#include <sys/suntpi.h> |
|
66 |
#include <sys/strlog.h> |
|
67 |
#include <sys/promif.h> |
|
68 |
#include <sys/project.h> |
|
69 |
#include <sys/vm.h> |
|
70 |
#include <sys/taskq.h> |
|
71 |
#include <sys/sunddi.h> |
|
72 |
#include <sys/sunldi_impl.h> |
|
73 |
#include <sys/strsun.h> |
|
74 |
#include <sys/isa_defs.h> |
|
75 |
#include <sys/multidata.h> |
|
76 |
#include <sys/pattr.h> |
|
77 |
#include <sys/strft.h> |
|
560
fab7db0411ac
6322179 strfreectty() passed a stream that was not yet a ctty
meem
parents:
235
diff
changeset
|
78 |
#include <sys/fs/snode.h> |
0 | 79 |
#include <sys/zone.h> |
3448 | 80 |
#include <sys/open.h> |
81 |
#include <sys/sunldi.h> |
|
82 |
#include <sys/sad.h> |
|
83 |
#include <sys/netstack.h> |
|
0 | 84 |
|
85 |
#define O_SAMESTR(q) (((q)->q_next) && \ |
|
86 |
(((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR))) |
|
87 |
||
88 |
/* |
|
89 |
* WARNING: |
|
90 |
* The variables and routines in this file are private, belonging |
|
91 |
* to the STREAMS subsystem. These should not be used by modules |
|
92 |
* or drivers. Compatibility will not be guaranteed. |
|
93 |
*/ |
|
94 |
||
95 |
/* |
|
96 |
* Id value used to distinguish between different multiplexor links. |
|
97 |
*/ |
|
98 |
static int32_t lnk_id = 0; |
|
99 |
||
100 |
#define STREAMS_LOPRI MINCLSYSPRI |
|
101 |
static pri_t streams_lopri = STREAMS_LOPRI; |
|
102 |
||
103 |
#define STRSTAT(x) (str_statistics.x.value.ui64++) |
|
104 |
typedef struct str_stat { |
|
105 |
kstat_named_t sqenables; |
|
106 |
kstat_named_t stenables; |
|
107 |
kstat_named_t syncqservice; |
|
108 |
kstat_named_t freebs; |
|
109 |
kstat_named_t qwr_outer; |
|
110 |
kstat_named_t rservice; |
|
111 |
kstat_named_t strwaits; |
|
112 |
kstat_named_t taskqfails; |
|
113 |
kstat_named_t bufcalls; |
|
114 |
kstat_named_t qhelps; |
|
115 |
kstat_named_t qremoved; |
|
116 |
kstat_named_t sqremoved; |
|
117 |
kstat_named_t bcwaits; |
|
118 |
kstat_named_t sqtoomany; |
|
119 |
} str_stat_t; |
|
120 |
||
121 |
static str_stat_t str_statistics = { |
|
122 |
{ "sqenables", KSTAT_DATA_UINT64 }, |
|
123 |
{ "stenables", KSTAT_DATA_UINT64 }, |
|
124 |
{ "syncqservice", KSTAT_DATA_UINT64 }, |
|
125 |
{ "freebs", KSTAT_DATA_UINT64 }, |
|
126 |
{ "qwr_outer", KSTAT_DATA_UINT64 }, |
|
127 |
{ "rservice", KSTAT_DATA_UINT64 }, |
|
128 |
{ "strwaits", KSTAT_DATA_UINT64 }, |
|
129 |
{ "taskqfails", KSTAT_DATA_UINT64 }, |
|
130 |
{ "bufcalls", KSTAT_DATA_UINT64 }, |
|
131 |
{ "qhelps", KSTAT_DATA_UINT64 }, |
|
132 |
{ "qremoved", KSTAT_DATA_UINT64 }, |
|
133 |
{ "sqremoved", KSTAT_DATA_UINT64 }, |
|
134 |
{ "bcwaits", KSTAT_DATA_UINT64 }, |
|
135 |
{ "sqtoomany", KSTAT_DATA_UINT64 }, |
|
136 |
}; |
|
137 |
||
138 |
static kstat_t *str_kstat; |
|
139 |
||
140 |
/* |
|
141 |
* qrunflag was used previously to control background scheduling of queues. It |
|
142 |
* is not used anymore, but kept here in case some module still wants to access |
|
143 |
* it via qready() and setqsched macros. |
|
144 |
*/ |
|
145 |
char qrunflag; /* Unused */ |
|
146 |
||
147 |
/* |
|
148 |
* Most of the streams scheduling is done via task queues. Task queues may fail |
|
149 |
* for non-sleep dispatches, so there are two backup threads servicing failed |
|
150 |
* requests for queues and syncqs. Both of these threads also service failed |
|
151 |
* dispatches freebs requests. Queues are put in the list specified by `qhead' |
|
152 |
* and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs |
|
153 |
* requests are put into `freebs_list' which has no tail pointer. All three |
|
154 |
* lists are protected by a single `service_queue' lock and use |
|
155 |
* `services_to_run' condition variable for signaling background threads. Use of |
|
156 |
* a single lock should not be a problem because it is only used under heavy |
|
157 |
* loads when task queues start to fail and at that time it may be a good idea |
|
158 |
* to throttle scheduling requests. |
|
159 |
* |
|
160 |
* NOTE: queues and syncqs should be scheduled by two separate threads because |
|
161 |
* queue servicing may be blocked waiting for a syncq which may be also |
|
162 |
* scheduled for background execution. This may create a deadlock when only one |
|
163 |
* thread is used for both. |
|
164 |
*/ |
|
165 |
||
166 |
static taskq_t *streams_taskq; /* Used for most STREAMS scheduling */ |
|
167 |
||
168 |
static kmutex_t service_queue; /* protects all of servicing vars */ |
|
169 |
static kcondvar_t services_to_run; /* wake up background service thread */ |
|
170 |
static kcondvar_t syncqs_to_run; /* wake up background service thread */ |
|
171 |
||
172 |
/* |
|
173 |
* List of queues scheduled for background processing dueue to lack of resources |
|
174 |
* in the task queues. Protected by service_queue lock; |
|
175 |
*/ |
|
176 |
static struct queue *qhead; |
|
177 |
static struct queue *qtail; |
|
178 |
||
179 |
/* |
|
180 |
* Same list for syncqs |
|
181 |
*/ |
|
182 |
static syncq_t *sqhead; |
|
183 |
static syncq_t *sqtail; |
|
184 |
||
185 |
static mblk_t *freebs_list; /* list of buffers to free */ |
|
186 |
||
187 |
/* |
|
188 |
* Backup threads for servicing queues and syncqs |
|
189 |
*/ |
|
190 |
kthread_t *streams_qbkgrnd_thread; |
|
191 |
kthread_t *streams_sqbkgrnd_thread; |
|
192 |
||
193 |
/* |
|
194 |
* Bufcalls related variables. |
|
195 |
*/ |
|
196 |
struct bclist strbcalls; /* list of waiting bufcalls */ |
|
197 |
kmutex_t strbcall_lock; /* protects bufcall list (strbcalls) */ |
|
198 |
kcondvar_t strbcall_cv; /* Signaling when a bufcall is added */ |
|
199 |
kmutex_t bcall_monitor; /* sleep/wakeup style monitor */ |
|
200 |
kcondvar_t bcall_cv; /* wait 'till executing bufcall completes */ |
|
201 |
kthread_t *bc_bkgrnd_thread; /* Thread to service bufcall requests */ |
|
202 |
||
203 |
kmutex_t strresources; /* protects global resources */ |
|
204 |
kmutex_t muxifier; /* single-threads multiplexor creation */ |
|
205 |
||
3448 | 206 |
static void *str_stack_init(netstackid_t stackid, netstack_t *ns); |
207 |
static void str_stack_shutdown(netstackid_t stackid, void *arg); |
|
208 |
static void str_stack_fini(netstackid_t stackid, void *arg); |
|
209 |
||
0 | 210 |
extern void time_to_wait(clock_t *, clock_t); |
211 |
||
212 |
/* |
|
213 |
* run_queues is no longer used, but is kept in case some 3-d party |
|
214 |
* module/driver decides to use it. |
|
215 |
*/ |
|
216 |
int run_queues = 0; |
|
217 |
||
218 |
/* |
|
219 |
* sq_max_size is the depth of the syncq (in number of messages) before |
|
220 |
* qfill_syncq() starts QFULL'ing destination queues. As its primary |
|
221 |
* consumer - IP is no longer D_MTPERMOD, but there may be other |
|
222 |
* modules/drivers depend on this syncq flow control, we prefer to |
|
223 |
* choose a large number as the default value. For potential |
|
224 |
* performance gain, this value is tunable in /etc/system. |
|
225 |
*/ |
|
226 |
int sq_max_size = 10000; |
|
227 |
||
228 |
/* |
|
229 |
* the number of ciputctrl structures per syncq and stream we create when |
|
230 |
* needed. |
|
231 |
*/ |
|
232 |
int n_ciputctrl; |
|
233 |
int max_n_ciputctrl = 16; |
|
234 |
/* |
|
235 |
* if n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache. |
|
236 |
*/ |
|
237 |
int min_n_ciputctrl = 2; |
|
238 |
||
239 |
/* |
|
240 |
* Per-driver/module syncqs |
|
241 |
* ======================== |
|
242 |
* |
|
243 |
* For drivers/modules that use PERMOD or outer syncqs we keep a list of |
|
244 |
* perdm structures, new entries being added (and new syncqs allocated) when |
|
245 |
* setq() encounters a module/driver with a streamtab that it hasn't seen |
|
246 |
* before. |
|
247 |
* The reason for this mechanism is that some modules and drivers share a |
|
248 |
* common streamtab and it is necessary for those modules and drivers to also |
|
249 |
* share a common PERMOD syncq. |
|
250 |
* |
|
251 |
* perdm_list --> dm_str == streamtab_1 |
|
252 |
* dm_sq == syncq_1 |
|
253 |
* dm_ref |
|
254 |
* dm_next --> dm_str == streamtab_2 |
|
255 |
* dm_sq == syncq_2 |
|
256 |
* dm_ref |
|
257 |
* dm_next --> ... NULL |
|
258 |
* |
|
259 |
* The dm_ref field is incremented for each new driver/module that takes |
|
260 |
* a reference to the perdm structure and hence shares the syncq. |
|
261 |
* References are held in the fmodsw_impl_t structure for each STREAMS module |
|
262 |
* or the dev_impl array (indexed by device major number) for each driver. |
|
263 |
* |
|
264 |
* perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL |
|
265 |
* ^ ^ ^ ^ |
|
266 |
* | ______________/ | | |
|
267 |
* | / | | |
|
268 |
* dev_impl: ...|x|y|... module A module B |
|
269 |
* |
|
270 |
* When a module/driver is unloaded the reference count is decremented and, |
|
271 |
* when it falls to zero, the perdm structure is removed from the list and |
|
272 |
* the syncq is freed (see rele_dm()). |
|
273 |
*/ |
|
274 |
perdm_t *perdm_list = NULL; |
|
275 |
static krwlock_t perdm_rwlock; |
|
276 |
cdevsw_impl_t *devimpl; |
|
277 |
||
278 |
extern struct qinit strdata; |
|
279 |
extern struct qinit stwdata; |
|
280 |
||
281 |
static void runservice(queue_t *); |
|
282 |
static void streams_bufcall_service(void); |
|
283 |
static void streams_qbkgrnd_service(void); |
|
284 |
static void streams_sqbkgrnd_service(void); |
|
285 |
static syncq_t *new_syncq(void); |
|
286 |
static void free_syncq(syncq_t *); |
|
287 |
static void outer_insert(syncq_t *, syncq_t *); |
|
288 |
static void outer_remove(syncq_t *, syncq_t *); |
|
289 |
static void write_now(syncq_t *); |
|
290 |
static void clr_qfull(queue_t *); |
|
291 |
static void enable_svc(queue_t *); |
|
292 |
static void runbufcalls(void); |
|
293 |
static void sqenable(syncq_t *); |
|
294 |
static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)()); |
|
295 |
static void wait_q_syncq(queue_t *); |
|
235
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
296 |
static void backenable_insertedq(queue_t *); |
0 | 297 |
|
298 |
static void queue_service(queue_t *); |
|
299 |
static void stream_service(stdata_t *); |
|
300 |
static void syncq_service(syncq_t *); |
|
301 |
static void qwriter_outer_service(syncq_t *); |
|
302 |
static void mblk_free(mblk_t *); |
|
303 |
#ifdef DEBUG |
|
304 |
static int qprocsareon(queue_t *); |
|
305 |
#endif |
|
306 |
||
307 |
static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *); |
|
308 |
static void reset_nfsrv_ptr(queue_t *, queue_t *); |
|
309 |
||
310 |
static void sq_run_events(syncq_t *); |
|
311 |
static int propagate_syncq(queue_t *); |
|
312 |
||
313 |
static void blocksq(syncq_t *, ushort_t, int); |
|
314 |
static void unblocksq(syncq_t *, ushort_t, int); |
|
315 |
static int dropsq(syncq_t *, uint16_t); |
|
316 |
static void emptysq(syncq_t *); |
|
317 |
static sqlist_t *sqlist_alloc(struct stdata *, int); |
|
318 |
static void sqlist_free(sqlist_t *); |
|
319 |
static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t); |
|
320 |
static void sqlist_insert(sqlist_t *, syncq_t *); |
|
321 |
static void sqlist_insertall(sqlist_t *, queue_t *); |
|
322 |
||
323 |
static void strsetuio(stdata_t *); |
|
324 |
||
325 |
struct kmem_cache *stream_head_cache; |
|
326 |
struct kmem_cache *queue_cache; |
|
327 |
struct kmem_cache *syncq_cache; |
|
328 |
struct kmem_cache *qband_cache; |
|
329 |
struct kmem_cache *linkinfo_cache; |
|
330 |
struct kmem_cache *ciputctrl_cache = NULL; |
|
331 |
||
332 |
static linkinfo_t *linkinfo_list; |
|
333 |
||
3932
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
334 |
/* global esballoc throttling queue */ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
335 |
static esb_queue_t system_esbq; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
336 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
337 |
/* |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
338 |
* esballoc tunable parameters. |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
339 |
*/ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
340 |
int esbq_max_qlen = 0x16; /* throttled queue length */ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
341 |
clock_t esbq_timeout = 0x8; /* timeout to process esb queue */ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
342 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
343 |
/* |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
344 |
* routines to handle esballoc queuing. |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
345 |
*/ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
346 |
static void esballoc_process_queue(esb_queue_t *); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
347 |
static void esballoc_enqueue_mblk(mblk_t *); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
348 |
static void esballoc_timer(void *); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
349 |
static void esballoc_set_timer(esb_queue_t *, clock_t); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
350 |
static void esballoc_mblk_free(mblk_t *); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
351 |
|
0 | 352 |
/* |
353 |
* Qinit structure and Module_info structures |
|
354 |
* for passthru read and write queues |
|
355 |
*/ |
|
356 |
||
357 |
static void pass_wput(queue_t *, mblk_t *); |
|
358 |
static queue_t *link_addpassthru(stdata_t *); |
|
359 |
static void link_rempassthru(queue_t *); |
|
360 |
||
361 |
struct module_info passthru_info = { |
|
362 |
0, |
|
363 |
"passthru", |
|
364 |
0, |
|
365 |
INFPSZ, |
|
366 |
STRHIGH, |
|
367 |
STRLOW |
|
368 |
}; |
|
369 |
||
370 |
struct qinit passthru_rinit = { |
|
371 |
(int (*)())putnext, |
|
372 |
NULL, |
|
373 |
NULL, |
|
374 |
NULL, |
|
375 |
NULL, |
|
376 |
&passthru_info, |
|
377 |
NULL |
|
378 |
}; |
|
379 |
||
380 |
struct qinit passthru_winit = { |
|
381 |
(int (*)()) pass_wput, |
|
382 |
NULL, |
|
383 |
NULL, |
|
384 |
NULL, |
|
385 |
NULL, |
|
386 |
&passthru_info, |
|
387 |
NULL |
|
388 |
}; |
|
389 |
||
390 |
/* |
|
391 |
* Special form of assertion: verify that X implies Y i.e. when X is true Y |
|
392 |
* should also be true. |
|
393 |
*/ |
|
394 |
#define IMPLY(X, Y) ASSERT(!(X) || (Y)) |
|
395 |
||
396 |
/* |
|
397 |
* Logical equivalence. Verify that both X and Y are either TRUE or FALSE. |
|
398 |
*/ |
|
399 |
#define EQUIV(X, Y) { IMPLY(X, Y); IMPLY(Y, X); } |
|
400 |
||
401 |
/* |
|
402 |
* Verify correctness of list head/tail pointers. |
|
403 |
*/ |
|
404 |
#define LISTCHECK(head, tail, link) { \ |
|
405 |
EQUIV(head, tail); \ |
|
406 |
IMPLY(tail != NULL, tail->link == NULL); \ |
|
407 |
} |
|
408 |
||
409 |
/* |
|
410 |
* Enqueue a list element `el' in the end of a list denoted by `head' and `tail' |
|
411 |
* using a `link' field. |
|
412 |
*/ |
|
413 |
#define ENQUEUE(el, head, tail, link) { \ |
|
414 |
ASSERT(el->link == NULL); \ |
|
415 |
LISTCHECK(head, tail, link); \ |
|
416 |
if (head == NULL) \ |
|
417 |
head = el; \ |
|
418 |
else \ |
|
419 |
tail->link = el; \ |
|
420 |
tail = el; \ |
|
421 |
} |
|
422 |
||
423 |
/* |
|
424 |
* Dequeue the first element of the list denoted by `head' and `tail' pointers |
|
425 |
* using a `link' field and put result into `el'. |
|
426 |
*/ |
|
427 |
#define DQ(el, head, tail, link) { \ |
|
428 |
LISTCHECK(head, tail, link); \ |
|
429 |
el = head; \ |
|
430 |
if (head != NULL) { \ |
|
431 |
head = head->link; \ |
|
432 |
if (head == NULL) \ |
|
433 |
tail = NULL; \ |
|
434 |
el->link = NULL; \ |
|
435 |
} \ |
|
436 |
} |
|
437 |
||
438 |
/* |
|
439 |
* Remove `el' from the list using `chase' and `curr' pointers and return result |
|
440 |
* in `succeed'. |
|
441 |
*/ |
|
442 |
#define RMQ(el, head, tail, link, chase, curr, succeed) { \ |
|
443 |
LISTCHECK(head, tail, link); \ |
|
444 |
chase = NULL; \ |
|
445 |
succeed = 0; \ |
|
446 |
for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \ |
|
447 |
chase = curr; \ |
|
448 |
if (curr != NULL) { \ |
|
449 |
succeed = 1; \ |
|
450 |
ASSERT(curr == el); \ |
|
451 |
if (chase != NULL) \ |
|
452 |
chase->link = curr->link; \ |
|
453 |
else \ |
|
454 |
head = curr->link; \ |
|
455 |
curr->link = NULL; \ |
|
456 |
if (curr == tail) \ |
|
457 |
tail = chase; \ |
|
458 |
} \ |
|
459 |
LISTCHECK(head, tail, link); \ |
|
460 |
} |
|
461 |
||
462 |
/* Handling of delayed messages on the inner syncq. */ |
|
463 |
||
464 |
/* |
|
465 |
* DEBUG versions should use function versions (to simplify tracing) and |
|
466 |
* non-DEBUG kernels should use macro versions. |
|
467 |
*/ |
|
468 |
||
469 |
/* |
|
470 |
* Put a queue on the syncq list of queues. |
|
471 |
* Assumes SQLOCK held. |
|
472 |
*/ |
|
473 |
#define SQPUT_Q(sq, qp) \ |
|
474 |
{ \ |
|
475 |
ASSERT(MUTEX_HELD(SQLOCK(sq))); \ |
|
476 |
if (!(qp->q_sqflags & Q_SQQUEUED)) { \ |
|
477 |
/* The queue should not be linked anywhere */ \ |
|
478 |
ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \ |
|
479 |
/* Head and tail may only be NULL simultaneously */ \ |
|
480 |
EQUIV(sq->sq_head, sq->sq_tail); \ |
|
481 |
/* Queue may be only enqueyed on its syncq */ \ |
|
482 |
ASSERT(sq == qp->q_syncq); \ |
|
483 |
/* Check the correctness of SQ_MESSAGES flag */ \ |
|
484 |
EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \ |
|
485 |
/* Sanity check first/last elements of the list */ \ |
|
486 |
IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\ |
|
487 |
IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\ |
|
488 |
/* \ |
|
489 |
* Sanity check of priority field: empty queue should \ |
|
490 |
* have zero priority \ |
|
491 |
* and nqueues equal to zero. \ |
|
492 |
*/ \ |
|
493 |
IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \ |
|
494 |
/* Sanity check of sq_nqueues field */ \ |
|
495 |
EQUIV(sq->sq_head, sq->sq_nqueues); \ |
|
496 |
if (sq->sq_head == NULL) { \ |
|
497 |
sq->sq_head = sq->sq_tail = qp; \ |
|
498 |
sq->sq_flags |= SQ_MESSAGES; \ |
|
499 |
} else if (qp->q_spri == 0) { \ |
|
500 |
qp->q_sqprev = sq->sq_tail; \ |
|
501 |
sq->sq_tail->q_sqnext = qp; \ |
|
502 |
sq->sq_tail = qp; \ |
|
503 |
} else { \ |
|
504 |
/* \ |
|
505 |
* Put this queue in priority order: higher \ |
|
506 |
* priority gets closer to the head. \ |
|
507 |
*/ \ |
|
508 |
queue_t **qpp = &sq->sq_tail; \ |
|
509 |
queue_t *qnext = NULL; \ |
|
510 |
\ |
|
511 |
while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \ |
|
512 |
qnext = *qpp; \ |
|
513 |
qpp = &(*qpp)->q_sqprev; \ |
|
514 |
} \ |
|
515 |
qp->q_sqnext = qnext; \ |
|
516 |
qp->q_sqprev = *qpp; \ |
|
517 |
if (*qpp != NULL) { \ |
|
518 |
(*qpp)->q_sqnext = qp; \ |
|
519 |
} else { \ |
|
520 |
sq->sq_head = qp; \ |
|
521 |
sq->sq_pri = sq->sq_head->q_spri; \ |
|
522 |
} \ |
|
523 |
*qpp = qp; \ |
|
524 |
} \ |
|
525 |
qp->q_sqflags |= Q_SQQUEUED; \ |
|
526 |
qp->q_sqtstamp = lbolt; \ |
|
527 |
sq->sq_nqueues++; \ |
|
528 |
} \ |
|
529 |
} |
|
530 |
||
531 |
/* |
|
532 |
* Remove a queue from the syncq list |
|
533 |
* Assumes SQLOCK held. |
|
534 |
*/ |
|
535 |
#define SQRM_Q(sq, qp) \ |
|
536 |
{ \ |
|
537 |
ASSERT(MUTEX_HELD(SQLOCK(sq))); \ |
|
538 |
ASSERT(qp->q_sqflags & Q_SQQUEUED); \ |
|
539 |
ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \ |
|
540 |
ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \ |
|
541 |
/* Check that the queue is actually in the list */ \ |
|
542 |
ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \ |
|
543 |
ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \ |
|
544 |
ASSERT(sq->sq_nqueues != 0); \ |
|
545 |
if (qp->q_sqprev == NULL) { \ |
|
546 |
/* First queue on list, make head q_sqnext */ \ |
|
547 |
sq->sq_head = qp->q_sqnext; \ |
|
548 |
} else { \ |
|
549 |
/* Make prev->next == next */ \ |
|
550 |
qp->q_sqprev->q_sqnext = qp->q_sqnext; \ |
|
551 |
} \ |
|
552 |
if (qp->q_sqnext == NULL) { \ |
|
553 |
/* Last queue on list, make tail sqprev */ \ |
|
554 |
sq->sq_tail = qp->q_sqprev; \ |
|
555 |
} else { \ |
|
556 |
/* Make next->prev == prev */ \ |
|
557 |
qp->q_sqnext->q_sqprev = qp->q_sqprev; \ |
|
558 |
} \ |
|
559 |
/* clear out references on this queue */ \ |
|
560 |
qp->q_sqprev = qp->q_sqnext = NULL; \ |
|
561 |
qp->q_sqflags &= ~Q_SQQUEUED; \ |
|
562 |
/* If there is nothing queued, clear SQ_MESSAGES */ \ |
|
563 |
if (sq->sq_head != NULL) { \ |
|
564 |
sq->sq_pri = sq->sq_head->q_spri; \ |
|
565 |
} else { \ |
|
566 |
sq->sq_flags &= ~SQ_MESSAGES; \ |
|
567 |
sq->sq_pri = 0; \ |
|
568 |
} \ |
|
569 |
sq->sq_nqueues--; \ |
|
570 |
ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \ |
|
571 |
(sq->sq_flags & SQ_QUEUED) == 0); \ |
|
572 |
} |
|
573 |
||
574 |
/* Hide the definition from the header file. */ |
|
575 |
#ifdef SQPUT_MP |
|
576 |
#undef SQPUT_MP |
|
577 |
#endif |
|
578 |
||
579 |
/* |
|
580 |
* Put a message on the queue syncq. |
|
581 |
* Assumes QLOCK held. |
|
582 |
*/ |
|
583 |
#define SQPUT_MP(qp, mp) \ |
|
584 |
{ \ |
|
585 |
ASSERT(MUTEX_HELD(QLOCK(qp))); \ |
|
586 |
ASSERT(qp->q_sqhead == NULL || \ |
|
587 |
(qp->q_sqtail != NULL && \ |
|
588 |
qp->q_sqtail->b_next == NULL)); \ |
|
589 |
qp->q_syncqmsgs++; \ |
|
590 |
ASSERT(qp->q_syncqmsgs != 0); /* Wraparound */ \ |
|
591 |
if (qp->q_sqhead == NULL) { \ |
|
592 |
qp->q_sqhead = qp->q_sqtail = mp; \ |
|
593 |
} else { \ |
|
594 |
qp->q_sqtail->b_next = mp; \ |
|
595 |
qp->q_sqtail = mp; \ |
|
596 |
} \ |
|
597 |
ASSERT(qp->q_syncqmsgs > 0); \ |
|
598 |
} |
|
599 |
||
600 |
#define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \ |
|
601 |
ASSERT(MUTEX_HELD(SQLOCK(sq))); \ |
|
602 |
if ((sq)->sq_ciputctrl != NULL) { \ |
|
603 |
int i; \ |
|
604 |
int nlocks = (sq)->sq_nciputctrl; \ |
|
605 |
ciputctrl_t *cip = (sq)->sq_ciputctrl; \ |
|
606 |
ASSERT((sq)->sq_type & SQ_CIPUT); \ |
|
607 |
for (i = 0; i <= nlocks; i++) { \ |
|
608 |
ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ |
|
609 |
cip[i].ciputctrl_count |= SQ_FASTPUT; \ |
|
610 |
} \ |
|
611 |
} \ |
|
612 |
} |
|
613 |
||
614 |
||
615 |
#define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \ |
|
616 |
ASSERT(MUTEX_HELD(SQLOCK(sq))); \ |
|
617 |
if ((sq)->sq_ciputctrl != NULL) { \ |
|
618 |
int i; \ |
|
619 |
int nlocks = (sq)->sq_nciputctrl; \ |
|
620 |
ciputctrl_t *cip = (sq)->sq_ciputctrl; \ |
|
621 |
ASSERT((sq)->sq_type & SQ_CIPUT); \ |
|
622 |
for (i = 0; i <= nlocks; i++) { \ |
|
623 |
ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ |
|
624 |
cip[i].ciputctrl_count &= ~SQ_FASTPUT; \ |
|
625 |
} \ |
|
626 |
} \ |
|
627 |
} |
|
628 |
||
629 |
/* |
|
630 |
* Run service procedures for all queues in the stream head. |
|
631 |
*/ |
|
632 |
#define STR_SERVICE(stp, q) { \ |
|
633 |
ASSERT(MUTEX_HELD(&stp->sd_qlock)); \ |
|
634 |
while (stp->sd_qhead != NULL) { \ |
|
635 |
DQ(q, stp->sd_qhead, stp->sd_qtail, q_link); \ |
|
636 |
ASSERT(stp->sd_nqueues > 0); \ |
|
637 |
stp->sd_nqueues--; \ |
|
638 |
ASSERT(!(q->q_flag & QINSERVICE)); \ |
|
639 |
mutex_exit(&stp->sd_qlock); \ |
|
640 |
queue_service(q); \ |
|
641 |
mutex_enter(&stp->sd_qlock); \ |
|
642 |
} \ |
|
643 |
ASSERT(stp->sd_nqueues == 0); \ |
|
644 |
ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL)); \ |
|
645 |
} |
|
646 |
||
647 |
/* |
|
648 |
* constructor/destructor routines for the stream head cache |
|
649 |
*/ |
|
650 |
/* ARGSUSED */ |
|
651 |
static int |
|
652 |
stream_head_constructor(void *buf, void *cdrarg, int kmflags) |
|
653 |
{ |
|
654 |
stdata_t *stp = buf; |
|
655 |
||
656 |
mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL); |
|
657 |
mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL); |
|
658 |
mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL); |
|
659 |
cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL); |
|
660 |
cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL); |
|
166
519e5268bee7
4657000 releasestr() should only wakeup waiters when sq_refcnt is zero.
xy158873
parents:
0
diff
changeset
|
661 |
cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL); |
0 | 662 |
cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL); |
663 |
cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL); |
|
664 |
stp->sd_wrq = NULL; |
|
665 |
||
666 |
return (0); |
|
667 |
} |
|
668 |
||
669 |
/* ARGSUSED */ |
|
670 |
static void |
|
671 |
stream_head_destructor(void *buf, void *cdrarg) |
|
672 |
{ |
|
673 |
stdata_t *stp = buf; |
|
674 |
||
675 |
mutex_destroy(&stp->sd_lock); |
|
676 |
mutex_destroy(&stp->sd_reflock); |
|
677 |
mutex_destroy(&stp->sd_qlock); |
|
678 |
cv_destroy(&stp->sd_monitor); |
|
679 |
cv_destroy(&stp->sd_iocmonitor); |
|
166
519e5268bee7
4657000 releasestr() should only wakeup waiters when sq_refcnt is zero.
xy158873
parents:
0
diff
changeset
|
680 |
cv_destroy(&stp->sd_refmonitor); |
0 | 681 |
cv_destroy(&stp->sd_qcv); |
682 |
cv_destroy(&stp->sd_zcopy_wait); |
|
683 |
} |
|
684 |
||
685 |
/* |
|
686 |
* constructor/destructor routines for the queue cache |
|
687 |
*/ |
|
688 |
/* ARGSUSED */ |
|
689 |
static int |
|
690 |
queue_constructor(void *buf, void *cdrarg, int kmflags) |
|
691 |
{ |
|
692 |
queinfo_t *qip = buf; |
|
693 |
queue_t *qp = &qip->qu_rqueue; |
|
694 |
queue_t *wqp = &qip->qu_wqueue; |
|
695 |
syncq_t *sq = &qip->qu_syncq; |
|
696 |
||
697 |
qp->q_first = NULL; |
|
698 |
qp->q_link = NULL; |
|
699 |
qp->q_count = 0; |
|
700 |
qp->q_mblkcnt = 0; |
|
701 |
qp->q_sqhead = NULL; |
|
702 |
qp->q_sqtail = NULL; |
|
703 |
qp->q_sqnext = NULL; |
|
704 |
qp->q_sqprev = NULL; |
|
705 |
qp->q_sqflags = 0; |
|
706 |
qp->q_rwcnt = 0; |
|
707 |
qp->q_spri = 0; |
|
708 |
||
709 |
mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL); |
|
710 |
cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL); |
|
711 |
||
712 |
wqp->q_first = NULL; |
|
713 |
wqp->q_link = NULL; |
|
714 |
wqp->q_count = 0; |
|
715 |
wqp->q_mblkcnt = 0; |
|
716 |
wqp->q_sqhead = NULL; |
|
717 |
wqp->q_sqtail = NULL; |
|
718 |
wqp->q_sqnext = NULL; |
|
719 |
wqp->q_sqprev = NULL; |
|
720 |
wqp->q_sqflags = 0; |
|
721 |
wqp->q_rwcnt = 0; |
|
722 |
wqp->q_spri = 0; |
|
723 |
||
724 |
mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL); |
|
725 |
cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL); |
|
726 |
||
727 |
sq->sq_head = NULL; |
|
728 |
sq->sq_tail = NULL; |
|
729 |
sq->sq_evhead = NULL; |
|
730 |
sq->sq_evtail = NULL; |
|
731 |
sq->sq_callbpend = NULL; |
|
732 |
sq->sq_outer = NULL; |
|
733 |
sq->sq_onext = NULL; |
|
734 |
sq->sq_oprev = NULL; |
|
735 |
sq->sq_next = NULL; |
|
736 |
sq->sq_svcflags = 0; |
|
737 |
sq->sq_servcount = 0; |
|
738 |
sq->sq_needexcl = 0; |
|
739 |
sq->sq_nqueues = 0; |
|
740 |
sq->sq_pri = 0; |
|
741 |
||
742 |
mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); |
|
743 |
cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); |
|
744 |
cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); |
|
745 |
||
746 |
return (0); |
|
747 |
} |
|
748 |
||
749 |
/* ARGSUSED */ |
|
750 |
static void |
|
751 |
queue_destructor(void *buf, void *cdrarg) |
|
752 |
{ |
|
753 |
queinfo_t *qip = buf; |
|
754 |
queue_t *qp = &qip->qu_rqueue; |
|
755 |
queue_t *wqp = &qip->qu_wqueue; |
|
756 |
syncq_t *sq = &qip->qu_syncq; |
|
757 |
||
758 |
ASSERT(qp->q_sqhead == NULL); |
|
759 |
ASSERT(wqp->q_sqhead == NULL); |
|
760 |
ASSERT(qp->q_sqnext == NULL); |
|
761 |
ASSERT(wqp->q_sqnext == NULL); |
|
762 |
ASSERT(qp->q_rwcnt == 0); |
|
763 |
ASSERT(wqp->q_rwcnt == 0); |
|
764 |
||
765 |
mutex_destroy(&qp->q_lock); |
|
766 |
cv_destroy(&qp->q_wait); |
|
767 |
||
768 |
mutex_destroy(&wqp->q_lock); |
|
769 |
cv_destroy(&wqp->q_wait); |
|
770 |
||
771 |
mutex_destroy(&sq->sq_lock); |
|
772 |
cv_destroy(&sq->sq_wait); |
|
773 |
cv_destroy(&sq->sq_exitwait); |
|
774 |
} |
|
775 |
||
776 |
/* |
|
777 |
* constructor/destructor routines for the syncq cache |
|
778 |
*/ |
|
779 |
/* ARGSUSED */ |
|
780 |
static int |
|
781 |
syncq_constructor(void *buf, void *cdrarg, int kmflags) |
|
782 |
{ |
|
783 |
syncq_t *sq = buf; |
|
784 |
||
785 |
bzero(buf, sizeof (syncq_t)); |
|
786 |
||
787 |
mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); |
|
788 |
cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); |
|
789 |
cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); |
|
790 |
||
791 |
return (0); |
|
792 |
} |
|
793 |
||
794 |
/* ARGSUSED */ |
|
795 |
static void |
|
796 |
syncq_destructor(void *buf, void *cdrarg) |
|
797 |
{ |
|
798 |
syncq_t *sq = buf; |
|
799 |
||
800 |
ASSERT(sq->sq_head == NULL); |
|
801 |
ASSERT(sq->sq_tail == NULL); |
|
802 |
ASSERT(sq->sq_evhead == NULL); |
|
803 |
ASSERT(sq->sq_evtail == NULL); |
|
804 |
ASSERT(sq->sq_callbpend == NULL); |
|
805 |
ASSERT(sq->sq_callbflags == 0); |
|
806 |
ASSERT(sq->sq_outer == NULL); |
|
807 |
ASSERT(sq->sq_onext == NULL); |
|
808 |
ASSERT(sq->sq_oprev == NULL); |
|
809 |
ASSERT(sq->sq_next == NULL); |
|
810 |
ASSERT(sq->sq_needexcl == 0); |
|
811 |
ASSERT(sq->sq_svcflags == 0); |
|
812 |
ASSERT(sq->sq_servcount == 0); |
|
813 |
ASSERT(sq->sq_nqueues == 0); |
|
814 |
ASSERT(sq->sq_pri == 0); |
|
815 |
ASSERT(sq->sq_count == 0); |
|
816 |
ASSERT(sq->sq_rmqcount == 0); |
|
817 |
ASSERT(sq->sq_cancelid == 0); |
|
818 |
ASSERT(sq->sq_ciputctrl == NULL); |
|
819 |
ASSERT(sq->sq_nciputctrl == 0); |
|
820 |
ASSERT(sq->sq_type == 0); |
|
821 |
ASSERT(sq->sq_flags == 0); |
|
822 |
||
823 |
mutex_destroy(&sq->sq_lock); |
|
824 |
cv_destroy(&sq->sq_wait); |
|
825 |
cv_destroy(&sq->sq_exitwait); |
|
826 |
} |
|
827 |
||
828 |
/* ARGSUSED */ |
|
829 |
static int |
|
830 |
ciputctrl_constructor(void *buf, void *cdrarg, int kmflags) |
|
831 |
{ |
|
832 |
ciputctrl_t *cip = buf; |
|
833 |
int i; |
|
834 |
||
835 |
for (i = 0; i < n_ciputctrl; i++) { |
|
836 |
cip[i].ciputctrl_count = SQ_FASTPUT; |
|
837 |
mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL); |
|
838 |
} |
|
839 |
||
840 |
return (0); |
|
841 |
} |
|
842 |
||
843 |
/* ARGSUSED */ |
|
844 |
static void |
|
845 |
ciputctrl_destructor(void *buf, void *cdrarg) |
|
846 |
{ |
|
847 |
ciputctrl_t *cip = buf; |
|
848 |
int i; |
|
849 |
||
850 |
for (i = 0; i < n_ciputctrl; i++) { |
|
851 |
ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT); |
|
852 |
mutex_destroy(&cip[i].ciputctrl_lock); |
|
853 |
} |
|
854 |
} |
|
855 |
||
856 |
/* |
|
857 |
* Init routine run from main at boot time. |
|
858 |
*/ |
|
859 |
void |
|
860 |
strinit(void) |
|
861 |
{ |
|
862 |
int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus); |
|
863 |
||
864 |
stream_head_cache = kmem_cache_create("stream_head_cache", |
|
865 |
sizeof (stdata_t), 0, |
|
866 |
stream_head_constructor, stream_head_destructor, NULL, |
|
867 |
NULL, NULL, 0); |
|
868 |
||
869 |
queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0, |
|
870 |
queue_constructor, queue_destructor, NULL, NULL, NULL, 0); |
|
871 |
||
872 |
syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0, |
|
873 |
syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0); |
|
874 |
||
875 |
qband_cache = kmem_cache_create("qband_cache", |
|
876 |
sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0); |
|
877 |
||
878 |
linkinfo_cache = kmem_cache_create("linkinfo_cache", |
|
879 |
sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0); |
|
880 |
||
881 |
n_ciputctrl = ncpus; |
|
882 |
n_ciputctrl = 1 << highbit(n_ciputctrl - 1); |
|
883 |
ASSERT(n_ciputctrl >= 1); |
|
884 |
n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl); |
|
885 |
if (n_ciputctrl >= min_n_ciputctrl) { |
|
886 |
ciputctrl_cache = kmem_cache_create("ciputctrl_cache", |
|
887 |
sizeof (ciputctrl_t) * n_ciputctrl, |
|
888 |
sizeof (ciputctrl_t), ciputctrl_constructor, |
|
889 |
ciputctrl_destructor, NULL, NULL, NULL, 0); |
|
890 |
} |
|
891 |
||
892 |
streams_taskq = system_taskq; |
|
893 |
||
894 |
if (streams_taskq == NULL) |
|
895 |
panic("strinit: no memory for streams taskq!"); |
|
896 |
||
897 |
bc_bkgrnd_thread = thread_create(NULL, 0, |
|
898 |
streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri); |
|
899 |
||
900 |
streams_qbkgrnd_thread = thread_create(NULL, 0, |
|
901 |
streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); |
|
902 |
||
903 |
streams_sqbkgrnd_thread = thread_create(NULL, 0, |
|
904 |
streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); |
|
905 |
||
906 |
/* |
|
907 |
* Create STREAMS kstats. |
|
908 |
*/ |
|
909 |
str_kstat = kstat_create("streams", 0, "strstat", |
|
910 |
"net", KSTAT_TYPE_NAMED, |
|
911 |
sizeof (str_statistics) / sizeof (kstat_named_t), |
|
912 |
KSTAT_FLAG_VIRTUAL); |
|
913 |
||
914 |
if (str_kstat != NULL) { |
|
915 |
str_kstat->ks_data = &str_statistics; |
|
916 |
kstat_install(str_kstat); |
|
917 |
} |
|
918 |
||
919 |
/* |
|
920 |
* TPI support routine initialisation. |
|
921 |
*/ |
|
922 |
tpi_init(); |
|
3448 | 923 |
|
924 |
/* |
|
925 |
* Handle to have autopush and persistent link information per |
|
926 |
* zone. |
|
927 |
* Note: uses shutdown hook instead of destroy hook so that the |
|
928 |
* persistent links can be torn down before the destroy hooks |
|
929 |
* in the TCP/IP stack are called. |
|
930 |
*/ |
|
931 |
netstack_register(NS_STR, str_stack_init, str_stack_shutdown, |
|
932 |
str_stack_fini); |
|
0 | 933 |
} |
934 |
||
935 |
void |
|
936 |
str_sendsig(vnode_t *vp, int event, uchar_t band, int error) |
|
937 |
{ |
|
938 |
struct stdata *stp; |
|
939 |
||
940 |
ASSERT(vp->v_stream); |
|
941 |
stp = vp->v_stream; |
|
942 |
/* Have to hold sd_lock to prevent siglist from changing */ |
|
943 |
mutex_enter(&stp->sd_lock); |
|
944 |
if (stp->sd_sigflags & event) |
|
945 |
strsendsig(stp->sd_siglist, event, band, error); |
|
946 |
mutex_exit(&stp->sd_lock); |
|
947 |
} |
|
948 |
||
949 |
/* |
|
950 |
* Send the "sevent" set of signals to a process. |
|
951 |
* This might send more than one signal if the process is registered |
|
952 |
* for multiple events. The caller should pass in an sevent that only |
|
953 |
* includes the events for which the process has registered. |
|
954 |
*/ |
|
955 |
static void |
|
956 |
dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info, |
|
957 |
uchar_t band, int error) |
|
958 |
{ |
|
959 |
ASSERT(MUTEX_HELD(&proc->p_lock)); |
|
960 |
||
961 |
info->si_band = 0; |
|
962 |
info->si_errno = 0; |
|
963 |
||
964 |
if (sevent & S_ERROR) { |
|
965 |
sevent &= ~S_ERROR; |
|
966 |
info->si_code = POLL_ERR; |
|
967 |
info->si_errno = error; |
|
968 |
TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, |
|
969 |
"strsendsig:proc %p info %p", proc, info); |
|
970 |
sigaddq(proc, NULL, info, KM_NOSLEEP); |
|
971 |
info->si_errno = 0; |
|
972 |
} |
|
973 |
if (sevent & S_HANGUP) { |
|
974 |
sevent &= ~S_HANGUP; |
|
975 |
info->si_code = POLL_HUP; |
|
976 |
TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, |
|
977 |
"strsendsig:proc %p info %p", proc, info); |
|
978 |
sigaddq(proc, NULL, info, KM_NOSLEEP); |
|
979 |
} |
|
980 |
if (sevent & S_HIPRI) { |
|
981 |
sevent &= ~S_HIPRI; |
|
982 |
info->si_code = POLL_PRI; |
|
983 |
TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, |
|
984 |
"strsendsig:proc %p info %p", proc, info); |
|
985 |
sigaddq(proc, NULL, info, KM_NOSLEEP); |
|
986 |
} |
|
987 |
if (sevent & S_RDBAND) { |
|
988 |
sevent &= ~S_RDBAND; |
|
989 |
if (events & S_BANDURG) |
|
990 |
sigtoproc(proc, NULL, SIGURG); |
|
991 |
else |
|
992 |
sigtoproc(proc, NULL, SIGPOLL); |
|
993 |
} |
|
994 |
if (sevent & S_WRBAND) { |
|
995 |
sevent &= ~S_WRBAND; |
|
996 |
sigtoproc(proc, NULL, SIGPOLL); |
|
997 |
} |
|
998 |
if (sevent & S_INPUT) { |
|
999 |
sevent &= ~S_INPUT; |
|
1000 |
info->si_code = POLL_IN; |
|
1001 |
info->si_band = band; |
|
1002 |
TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, |
|
1003 |
"strsendsig:proc %p info %p", proc, info); |
|
1004 |
sigaddq(proc, NULL, info, KM_NOSLEEP); |
|
1005 |
info->si_band = 0; |
|
1006 |
} |
|
1007 |
if (sevent & S_OUTPUT) { |
|
1008 |
sevent &= ~S_OUTPUT; |
|
1009 |
info->si_code = POLL_OUT; |
|
1010 |
info->si_band = band; |
|
1011 |
TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, |
|
1012 |
"strsendsig:proc %p info %p", proc, info); |
|
1013 |
sigaddq(proc, NULL, info, KM_NOSLEEP); |
|
1014 |
info->si_band = 0; |
|
1015 |
} |
|
1016 |
if (sevent & S_MSG) { |
|
1017 |
sevent &= ~S_MSG; |
|
1018 |
info->si_code = POLL_MSG; |
|
1019 |
info->si_band = band; |
|
1020 |
TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, |
|
1021 |
"strsendsig:proc %p info %p", proc, info); |
|
1022 |
sigaddq(proc, NULL, info, KM_NOSLEEP); |
|
1023 |
info->si_band = 0; |
|
1024 |
} |
|
1025 |
if (sevent & S_RDNORM) { |
|
1026 |
sevent &= ~S_RDNORM; |
|
1027 |
sigtoproc(proc, NULL, SIGPOLL); |
|
1028 |
} |
|
1029 |
if (sevent != 0) { |
|
1030 |
panic("strsendsig: unknown event(s) %x", sevent); |
|
1031 |
} |
|
1032 |
} |
|
1033 |
||
1034 |
/* |
|
1035 |
* Send SIGPOLL/SIGURG signal to all processes and process groups |
|
1036 |
* registered on the given signal list that want a signal for at |
|
1037 |
* least one of the specified events. |
|
1038 |
* |
|
1039 |
* Must be called with exclusive access to siglist (caller holding sd_lock). |
|
1040 |
* |
|
1041 |
* strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding |
|
1042 |
* sd_lock and the ioctl code maintains a PID_HOLD on the pid structure |
|
1043 |
* while it is in the siglist. |
|
1044 |
* |
|
1045 |
* For performance reasons (MP scalability) the code drops pidlock |
|
1046 |
* when sending signals to a single process. |
|
1047 |
* When sending to a process group the code holds |
|
1048 |
* pidlock to prevent the membership in the process group from changing |
|
1049 |
* while walking the p_pglink list. |
|
1050 |
*/ |
|
1051 |
void |
|
1052 |
strsendsig(strsig_t *siglist, int event, uchar_t band, int error) |
|
1053 |
{ |
|
1054 |
strsig_t *ssp; |
|
1055 |
k_siginfo_t info; |
|
1056 |
struct pid *pidp; |
|
1057 |
proc_t *proc; |
|
1058 |
||
1059 |
info.si_signo = SIGPOLL; |
|
1060 |
info.si_errno = 0; |
|
1061 |
for (ssp = siglist; ssp; ssp = ssp->ss_next) { |
|
1062 |
int sevent; |
|
1063 |
||
1064 |
sevent = ssp->ss_events & event; |
|
1065 |
if (sevent == 0) |
|
1066 |
continue; |
|
1067 |
||
1068 |
if ((pidp = ssp->ss_pidp) == NULL) { |
|
1069 |
/* pid was released but still on event list */ |
|
1070 |
continue; |
|
1071 |
} |
|
1072 |
||
1073 |
||
1074 |
if (ssp->ss_pid > 0) { |
|
1075 |
/* |
|
1076 |
* XXX This unfortunately still generates |
|
1077 |
* a signal when a fd is closed but |
|
1078 |
* the proc is active. |
|
1079 |
*/ |
|
1080 |
ASSERT(ssp->ss_pid == pidp->pid_id); |
|
1081 |
||
1082 |
mutex_enter(&pidlock); |
|
1083 |
proc = prfind_zone(pidp->pid_id, ALL_ZONES); |
|
1084 |
if (proc == NULL) { |
|
1085 |
mutex_exit(&pidlock); |
|
1086 |
continue; |
|
1087 |
} |
|
1088 |
mutex_enter(&proc->p_lock); |
|
1089 |
mutex_exit(&pidlock); |
|
1090 |
dosendsig(proc, ssp->ss_events, sevent, &info, |
|
1091 |
band, error); |
|
1092 |
mutex_exit(&proc->p_lock); |
|
1093 |
} else { |
|
1094 |
/* |
|
1095 |
* Send to process group. Hold pidlock across |
|
1096 |
* calls to dosendsig(). |
|
1097 |
*/ |
|
1098 |
pid_t pgrp = -ssp->ss_pid; |
|
1099 |
||
1100 |
mutex_enter(&pidlock); |
|
1101 |
proc = pgfind_zone(pgrp, ALL_ZONES); |
|
1102 |
while (proc != NULL) { |
|
1103 |
mutex_enter(&proc->p_lock); |
|
1104 |
dosendsig(proc, ssp->ss_events, sevent, |
|
1105 |
&info, band, error); |
|
1106 |
mutex_exit(&proc->p_lock); |
|
1107 |
proc = proc->p_pglink; |
|
1108 |
} |
|
1109 |
mutex_exit(&pidlock); |
|
1110 |
} |
|
1111 |
} |
|
1112 |
} |
|
1113 |
||
1114 |
/* |
|
1115 |
* Attach a stream device or module. |
|
1116 |
* qp is a read queue; the new queue goes in so its next |
|
1117 |
* read ptr is the argument, and the write queue corresponding |
|
1118 |
* to the argument points to this queue. Return 0 on success, |
|
1119 |
* or a non-zero errno on failure. |
|
1120 |
*/ |
|
1121 |
int |
|
1122 |
qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp, |
|
1123 |
boolean_t is_insert) |
|
1124 |
{ |
|
1125 |
major_t major; |
|
1126 |
cdevsw_impl_t *dp; |
|
1127 |
struct streamtab *str; |
|
1128 |
queue_t *rq; |
|
1129 |
queue_t *wrq; |
|
1130 |
uint32_t qflag; |
|
1131 |
uint32_t sqtype; |
|
1132 |
perdm_t *dmp; |
|
1133 |
int error; |
|
1134 |
int sflag; |
|
1135 |
||
1136 |
rq = allocq(); |
|
1137 |
wrq = _WR(rq); |
|
1138 |
STREAM(rq) = STREAM(wrq) = STREAM(qp); |
|
1139 |
||
1140 |
if (fp != NULL) { |
|
1141 |
str = fp->f_str; |
|
1142 |
qflag = fp->f_qflag; |
|
1143 |
sqtype = fp->f_sqtype; |
|
1144 |
dmp = fp->f_dmp; |
|
1145 |
IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); |
|
1146 |
sflag = MODOPEN; |
|
1147 |
||
1148 |
/* |
|
1149 |
* stash away a pointer to the module structure so we can |
|
1150 |
* unref it in qdetach. |
|
1151 |
*/ |
|
1152 |
rq->q_fp = fp; |
|
1153 |
} else { |
|
1154 |
ASSERT(!is_insert); |
|
1155 |
||
1156 |
major = getmajor(*devp); |
|
1157 |
dp = &devimpl[major]; |
|
1158 |
||
1159 |
str = dp->d_str; |
|
1160 |
ASSERT(str == STREAMSTAB(major)); |
|
1161 |
||
1162 |
qflag = dp->d_qflag; |
|
1163 |
ASSERT(qflag & QISDRV); |
|
1164 |
sqtype = dp->d_sqtype; |
|
1165 |
||
1166 |
/* create perdm_t if needed */ |
|
1167 |
if (NEED_DM(dp->d_dmp, qflag)) |
|
1168 |
dp->d_dmp = hold_dm(str, qflag, sqtype); |
|
1169 |
||
1170 |
dmp = dp->d_dmp; |
|
1171 |
sflag = 0; |
|
1172 |
} |
|
1173 |
||
1174 |
TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS, |
|
1175 |
"qattach:qflag == %X(%X)", qflag, *devp); |
|
1176 |
||
1177 |
/* setq might sleep in allocator - avoid holding locks. */ |
|
1178 |
setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE); |
|
1179 |
||
1180 |
/* |
|
1181 |
* Before calling the module's open routine, set up the q_next |
|
1182 |
* pointer for inserting a module in the middle of a stream. |
|
1183 |
* |
|
1184 |
* Note that we can always set _QINSERTING and set up q_next |
|
1185 |
* pointer for both inserting and pushing a module. Then there |
|
1186 |
* is no need for the is_insert parameter. In insertq(), called |
|
1187 |
* by qprocson(), assume that q_next of the new module always points |
|
1188 |
* to the correct queue and use it for insertion. Everything should |
|
1189 |
* work out fine. But in the first release of _I_INSERT, we |
|
1190 |
* distinguish between inserting and pushing to make sure that |
|
1191 |
* pushing a module follows the same code path as before. |
|
1192 |
*/ |
|
1193 |
if (is_insert) { |
|
1194 |
rq->q_flag |= _QINSERTING; |
|
1195 |
rq->q_next = qp; |
|
1196 |
} |
|
1197 |
||
1198 |
/* |
|
1199 |
* If there is an outer perimeter get exclusive access during |
|
1200 |
* the open procedure. Bump up the reference count on the queue. |
|
1201 |
*/ |
|
1202 |
entersq(rq->q_syncq, SQ_OPENCLOSE); |
|
1203 |
error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp); |
|
1204 |
if (error != 0) |
|
1205 |
goto failed; |
|
1206 |
leavesq(rq->q_syncq, SQ_OPENCLOSE); |
|
1207 |
ASSERT(qprocsareon(rq)); |
|
1208 |
return (0); |
|
1209 |
||
1210 |
failed: |
|
1211 |
rq->q_flag &= ~_QINSERTING; |
|
1212 |
if (backq(wrq) != NULL && backq(wrq)->q_next == wrq) |
|
1213 |
qprocsoff(rq); |
|
1214 |
leavesq(rq->q_syncq, SQ_OPENCLOSE); |
|
1215 |
rq->q_next = wrq->q_next = NULL; |
|
1216 |
qdetach(rq, 0, 0, crp, B_FALSE); |
|
1217 |
return (error); |
|
1218 |
} |
|
1219 |
||
1220 |
/* |
|
1221 |
* Handle second open of stream. For modules, set the |
|
1222 |
* last argument to MODOPEN and do not pass any open flags. |
|
1223 |
* Ignore dummydev since this is not the first open. |
|
1224 |
*/ |
|
1225 |
int |
|
1226 |
qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp) |
|
1227 |
{ |
|
1228 |
int error; |
|
1229 |
dev_t dummydev; |
|
1230 |
queue_t *wqp = _WR(qp); |
|
1231 |
||
1232 |
ASSERT(qp->q_flag & QREADR); |
|
1233 |
entersq(qp->q_syncq, SQ_OPENCLOSE); |
|
1234 |
||
1235 |
dummydev = *devp; |
|
1236 |
if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev, |
|
1237 |
(wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) { |
|
1238 |
leavesq(qp->q_syncq, SQ_OPENCLOSE); |
|
1239 |
mutex_enter(&STREAM(qp)->sd_lock); |
|
1240 |
qp->q_stream->sd_flag |= STREOPENFAIL; |
|
1241 |
mutex_exit(&STREAM(qp)->sd_lock); |
|
1242 |
return (error); |
|
1243 |
} |
|
1244 |
leavesq(qp->q_syncq, SQ_OPENCLOSE); |
|
1245 |
||
1246 |
/* |
|
1247 |
* successful open should have done qprocson() |
|
1248 |
*/ |
|
1249 |
ASSERT(qprocsareon(_RD(qp))); |
|
1250 |
return (0); |
|
1251 |
} |
|
1252 |
||
1253 |
/* |
|
1254 |
* Detach a stream module or device. |
|
1255 |
* If clmode == 1 then the module or driver was opened and its |
|
1256 |
* close routine must be called. If clmode == 0, the module |
|
1257 |
* or driver was never opened or the open failed, and so its close |
|
1258 |
* should not be called. |
|
1259 |
*/ |
|
1260 |
void |
|
1261 |
qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove) |
|
1262 |
{ |
|
1263 |
queue_t *wqp = _WR(qp); |
|
1264 |
ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB)); |
|
1265 |
||
1266 |
if (STREAM_NEEDSERVICE(STREAM(qp))) |
|
1267 |
stream_runservice(STREAM(qp)); |
|
1268 |
||
1269 |
if (clmode) { |
|
1270 |
/* |
|
1271 |
* Make sure that all the messages on the write side syncq are |
|
1272 |
* processed and nothing is left. Since we are closing, no new |
|
1273 |
* messages may appear there. |
|
1274 |
*/ |
|
1275 |
wait_q_syncq(wqp); |
|
1276 |
||
1277 |
entersq(qp->q_syncq, SQ_OPENCLOSE); |
|
1278 |
if (is_remove) { |
|
1279 |
mutex_enter(QLOCK(qp)); |
|
1280 |
qp->q_flag |= _QREMOVING; |
|
1281 |
mutex_exit(QLOCK(qp)); |
|
1282 |
} |
|
1283 |
(*qp->q_qinfo->qi_qclose)(qp, flag, crp); |
|
1284 |
/* |
|
1285 |
* Check that qprocsoff() was actually called. |
|
1286 |
*/ |
|
1287 |
ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE)); |
|
1288 |
||
1289 |
leavesq(qp->q_syncq, SQ_OPENCLOSE); |
|
1290 |
} else { |
|
1291 |
disable_svc(qp); |
|
1292 |
} |
|
1293 |
||
1294 |
/* |
|
1295 |
* Allow any threads blocked in entersq to proceed and discover |
|
1296 |
* the QWCLOSE is set. |
|
1297 |
* Note: This assumes that all users of entersq check QWCLOSE. |
|
1298 |
* Currently runservice is the only entersq that can happen |
|
1299 |
* after removeq has finished. |
|
1300 |
* Removeq will have discarded all messages destined to the closing |
|
1301 |
* pair of queues from the syncq. |
|
1302 |
* NOTE: Calling a function inside an assert is unconventional. |
|
1303 |
* However, it does not cause any problem since flush_syncq() does |
|
1304 |
* not change any state except when it returns non-zero i.e. |
|
1305 |
* when the assert will trigger. |
|
1306 |
*/ |
|
1307 |
ASSERT(flush_syncq(qp->q_syncq, qp) == 0); |
|
1308 |
ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0); |
|
1309 |
ASSERT((qp->q_flag & QPERMOD) || |
|
1310 |
((qp->q_syncq->sq_head == NULL) && |
|
1311 |
(wqp->q_syncq->sq_head == NULL))); |
|
1312 |
||
1313 |
/* release any fmodsw_impl_t structure held on behalf of the queue */ |
|
1314 |
ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV); |
|
1315 |
if (qp->q_fp != NULL) |
|
1316 |
fmodsw_rele(qp->q_fp); |
|
1317 |
||
1318 |
/* freeq removes us from the outer perimeter if any */ |
|
1319 |
freeq(qp); |
|
1320 |
} |
|
1321 |
||
1322 |
/* Prevent service procedures from being called */ |
|
1323 |
void |
|
1324 |
disable_svc(queue_t *qp) |
|
1325 |
{ |
|
1326 |
queue_t *wqp = _WR(qp); |
|
1327 |
||
1328 |
ASSERT(qp->q_flag & QREADR); |
|
1329 |
mutex_enter(QLOCK(qp)); |
|
1330 |
qp->q_flag |= QWCLOSE; |
|
1331 |
mutex_exit(QLOCK(qp)); |
|
1332 |
mutex_enter(QLOCK(wqp)); |
|
1333 |
wqp->q_flag |= QWCLOSE; |
|
1334 |
mutex_exit(QLOCK(wqp)); |
|
1335 |
} |
|
1336 |
||
1337 |
/* allow service procedures to be called again */ |
|
1338 |
void |
|
1339 |
enable_svc(queue_t *qp) |
|
1340 |
{ |
|
1341 |
queue_t *wqp = _WR(qp); |
|
1342 |
||
1343 |
ASSERT(qp->q_flag & QREADR); |
|
1344 |
mutex_enter(QLOCK(qp)); |
|
1345 |
qp->q_flag &= ~QWCLOSE; |
|
1346 |
mutex_exit(QLOCK(qp)); |
|
1347 |
mutex_enter(QLOCK(wqp)); |
|
1348 |
wqp->q_flag &= ~QWCLOSE; |
|
1349 |
mutex_exit(QLOCK(wqp)); |
|
1350 |
} |
|
1351 |
||
1352 |
/* |
|
1353 |
* Remove queue from qhead/qtail if it is enabled. |
|
1354 |
* Only reset QENAB if the queue was removed from the runlist. |
|
1355 |
* A queue goes through 3 stages: |
|
1356 |
* It is on the service list and QENAB is set. |
|
1357 |
* It is removed from the service list but QENAB is still set. |
|
1358 |
* QENAB gets changed to QINSERVICE. |
|
1359 |
* QINSERVICE is reset (when the service procedure is done) |
|
1360 |
* Thus we can not reset QENAB unless we actually removed it from the service |
|
1361 |
* queue. |
|
1362 |
*/ |
|
1363 |
void |
|
1364 |
remove_runlist(queue_t *qp) |
|
1365 |
{ |
|
1366 |
if (qp->q_flag & QENAB && qhead != NULL) { |
|
1367 |
queue_t *q_chase; |
|
1368 |
queue_t *q_curr; |
|
1369 |
int removed; |
|
1370 |
||
1371 |
mutex_enter(&service_queue); |
|
1372 |
RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed); |
|
1373 |
mutex_exit(&service_queue); |
|
1374 |
if (removed) { |
|
1375 |
STRSTAT(qremoved); |
|
1376 |
qp->q_flag &= ~QENAB; |
|
1377 |
} |
|
1378 |
} |
|
1379 |
} |
|
1380 |
||
1381 |
||
1382 |
/* |
|
1383 |
* wait for any pending service processing to complete. |
|
1384 |
* The removal of queues from the runlist is not atomic with the |
|
1385 |
* clearing of the QENABLED flag and setting the INSERVICE flag. |
|
1386 |
* consequently it is possible for remove_runlist in strclose |
|
1387 |
* to not find the queue on the runlist but for it to be QENABLED |
|
1388 |
* and not yet INSERVICE -> hence wait_svc needs to check QENABLED |
|
1389 |
* as well as INSERVICE. |
|
1390 |
*/ |
|
1391 |
void |
|
1392 |
wait_svc(queue_t *qp) |
|
1393 |
{ |
|
1394 |
queue_t *wqp = _WR(qp); |
|
1395 |
||
1396 |
ASSERT(qp->q_flag & QREADR); |
|
1397 |
||
1398 |
/* |
|
1399 |
* Try to remove queues from qhead/qtail list. |
|
1400 |
*/ |
|
1401 |
if (qhead != NULL) { |
|
1402 |
remove_runlist(qp); |
|
1403 |
remove_runlist(wqp); |
|
1404 |
} |
|
1405 |
/* |
|
1406 |
* Wait till the syncqs associated with the queue |
|
1407 |
* will dissapear from background processing list. |
|
1408 |
* This only needs to be done for non-PERMOD perimeters since |
|
1409 |
* for PERMOD perimeters the syncq may be shared and will only be freed |
|
1410 |
* when the last module/driver is unloaded. |
|
1411 |
* If for PERMOD perimeters queue was on the syncq list, removeq() |
|
1412 |
* should call propagate_syncq() or drain_syncq() for it. Both of these |
|
1413 |
* function remove the queue from its syncq list, so sqthread will not |
|
1414 |
* try to access the queue. |
|
1415 |
*/ |
|
1416 |
if (!(qp->q_flag & QPERMOD)) { |
|
1417 |
syncq_t *rsq = qp->q_syncq; |
|
1418 |
syncq_t *wsq = wqp->q_syncq; |
|
1419 |
||
1420 |
/* |
|
1421 |
* Disable rsq and wsq and wait for any background processing of |
|
1422 |
* syncq to complete. |
|
1423 |
*/ |
|
1424 |
wait_sq_svc(rsq); |
|
1425 |
if (wsq != rsq) |
|
1426 |
wait_sq_svc(wsq); |
|
1427 |
} |
|
1428 |
||
1429 |
mutex_enter(QLOCK(qp)); |
|
1430 |
while (qp->q_flag & (QINSERVICE|QENAB)) |
|
1431 |
cv_wait(&qp->q_wait, QLOCK(qp)); |
|
1432 |
mutex_exit(QLOCK(qp)); |
|
1433 |
mutex_enter(QLOCK(wqp)); |
|
1434 |
while (wqp->q_flag & (QINSERVICE|QENAB)) |
|
1435 |
cv_wait(&wqp->q_wait, QLOCK(wqp)); |
|
1436 |
mutex_exit(QLOCK(wqp)); |
|
1437 |
} |
|
1438 |
||
1439 |
/* |
|
1440 |
* Put ioctl data from userland buffer `arg' into the mblk chain `bp'. |
|
1441 |
* `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may |
|
1442 |
* also be set, and is passed through to allocb_cred_wait(). |
|
1443 |
* |
|
1444 |
* Returns errno on failure, zero on success. |
|
1445 |
*/ |
|
1446 |
int |
|
1447 |
putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr) |
|
1448 |
{ |
|
1449 |
mblk_t *tmp; |
|
1450 |
ssize_t count; |
|
1451 |
size_t n; |
|
1452 |
int error = 0; |
|
1453 |
||
1454 |
ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K || |
|
1455 |
(flag & (U_TO_K | K_TO_K)) == K_TO_K); |
|
1456 |
||
1457 |
if (bp->b_datap->db_type == M_IOCTL) { |
|
1458 |
count = ((struct iocblk *)bp->b_rptr)->ioc_count; |
|
1459 |
} else { |
|
1460 |
ASSERT(bp->b_datap->db_type == M_COPYIN); |
|
1461 |
count = ((struct copyreq *)bp->b_rptr)->cq_size; |
|
1462 |
} |
|
1463 |
/* |
|
1464 |
* strdoioctl validates ioc_count, so if this assert fails it |
|
1465 |
* cannot be due to user error. |
|
1466 |
*/ |
|
1467 |
ASSERT(count >= 0); |
|
1468 |
||
1469 |
while (count > 0) { |
|
1470 |
n = MIN(MAXIOCBSZ, count); |
|
1471 |
if ((tmp = allocb_cred_wait(n, (flag & STR_NOSIG), &error, |
|
1472 |
cr)) == NULL) { |
|
1473 |
return (error); |
|
1474 |
} |
|
1475 |
error = strcopyin(arg, tmp->b_wptr, n, flag & (U_TO_K|K_TO_K)); |
|
1476 |
if (error != 0) { |
|
1477 |
freeb(tmp); |
|
1478 |
return (error); |
|
1479 |
} |
|
1480 |
arg += n; |
|
1481 |
DB_CPID(tmp) = curproc->p_pid; |
|
1482 |
tmp->b_wptr += n; |
|
1483 |
count -= n; |
|
1484 |
bp = (bp->b_cont = tmp); |
|
1485 |
} |
|
1486 |
||
1487 |
return (0); |
|
1488 |
} |
|
1489 |
||
1490 |
/* |
|
1491 |
* Copy ioctl data to user-land. Return non-zero errno on failure, |
|
1492 |
* 0 for success. |
|
1493 |
*/ |
|
1494 |
int |
|
1495 |
getiocd(mblk_t *bp, char *arg, int copymode) |
|
1496 |
{ |
|
1497 |
ssize_t count; |
|
1498 |
size_t n; |
|
1499 |
int error; |
|
1500 |
||
1501 |
if (bp->b_datap->db_type == M_IOCACK) |
|
1502 |
count = ((struct iocblk *)bp->b_rptr)->ioc_count; |
|
1503 |
else { |
|
1504 |
ASSERT(bp->b_datap->db_type == M_COPYOUT); |
|
1505 |
count = ((struct copyreq *)bp->b_rptr)->cq_size; |
|
1506 |
} |
|
1507 |
ASSERT(count >= 0); |
|
1508 |
||
1509 |
for (bp = bp->b_cont; bp && count; |
|
1510 |
count -= n, bp = bp->b_cont, arg += n) { |
|
1511 |
n = MIN(count, bp->b_wptr - bp->b_rptr); |
|
1512 |
error = strcopyout(bp->b_rptr, arg, n, copymode); |
|
1513 |
if (error) |
|
1514 |
return (error); |
|
1515 |
} |
|
1516 |
ASSERT(count == 0); |
|
1517 |
return (0); |
|
1518 |
} |
|
1519 |
||
1520 |
/* |
|
1521 |
* Allocate a linkinfo entry given the write queue of the |
|
1522 |
* bottom module of the top stream and the write queue of the |
|
1523 |
* stream head of the bottom stream. |
|
1524 |
*/ |
|
1525 |
linkinfo_t * |
|
1526 |
alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown) |
|
1527 |
{ |
|
1528 |
linkinfo_t *linkp; |
|
1529 |
||
1530 |
linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP); |
|
1531 |
||
1532 |
linkp->li_lblk.l_qtop = qup; |
|
1533 |
linkp->li_lblk.l_qbot = qdown; |
|
1534 |
linkp->li_fpdown = fpdown; |
|
1535 |
||
1536 |
mutex_enter(&strresources); |
|
1537 |
linkp->li_next = linkinfo_list; |
|
1538 |
linkp->li_prev = NULL; |
|
1539 |
if (linkp->li_next) |
|
1540 |
linkp->li_next->li_prev = linkp; |
|
1541 |
linkinfo_list = linkp; |
|
1542 |
linkp->li_lblk.l_index = ++lnk_id; |
|
1543 |
ASSERT(lnk_id != 0); /* this should never wrap in practice */ |
|
1544 |
mutex_exit(&strresources); |
|
1545 |
||
1546 |
return (linkp); |
|
1547 |
} |
|
1548 |
||
1549 |
/* |
|
1550 |
* Free a linkinfo entry. |
|
1551 |
*/ |
|
1552 |
void |
|
1553 |
lbfree(linkinfo_t *linkp) |
|
1554 |
{ |
|
1555 |
mutex_enter(&strresources); |
|
1556 |
if (linkp->li_next) |
|
1557 |
linkp->li_next->li_prev = linkp->li_prev; |
|
1558 |
if (linkp->li_prev) |
|
1559 |
linkp->li_prev->li_next = linkp->li_next; |
|
1560 |
else |
|
1561 |
linkinfo_list = linkp->li_next; |
|
1562 |
mutex_exit(&strresources); |
|
1563 |
||
1564 |
kmem_cache_free(linkinfo_cache, linkp); |
|
1565 |
} |
|
1566 |
||
1567 |
/* |
|
1568 |
* Check for a potential linking cycle. |
|
1569 |
* Return 1 if a link will result in a cycle, |
|
1570 |
* and 0 otherwise. |
|
1571 |
*/ |
|
1572 |
int |
|
3448 | 1573 |
linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss) |
0 | 1574 |
{ |
1575 |
struct mux_node *np; |
|
1576 |
struct mux_edge *ep; |
|
1577 |
int i; |
|
1578 |
major_t lomaj; |
|
1579 |
major_t upmaj; |
|
1580 |
/* |
|
1581 |
* if the lower stream is a pipe/FIFO, return, since link |
|
1582 |
* cycles can not happen on pipes/FIFOs |
|
1583 |
*/ |
|
1584 |
if (lostp->sd_vnode->v_type == VFIFO) |
|
1585 |
return (0); |
|
1586 |
||
3448 | 1587 |
for (i = 0; i < ss->ss_devcnt; i++) { |
1588 |
np = &ss->ss_mux_nodes[i]; |
|
0 | 1589 |
MUX_CLEAR(np); |
1590 |
} |
|
1591 |
lomaj = getmajor(lostp->sd_vnode->v_rdev); |
|
1592 |
upmaj = getmajor(upstp->sd_vnode->v_rdev); |
|
3448 | 1593 |
np = &ss->ss_mux_nodes[lomaj]; |
0 | 1594 |
for (;;) { |
1595 |
if (!MUX_DIDVISIT(np)) { |
|
1596 |
if (np->mn_imaj == upmaj) |
|
1597 |
return (1); |
|
1598 |
if (np->mn_outp == NULL) { |
|
1599 |
MUX_VISIT(np); |
|
1600 |
if (np->mn_originp == NULL) |
|
1601 |
return (0); |
|
1602 |
np = np->mn_originp; |
|
1603 |
continue; |
|
1604 |
} |
|
1605 |
MUX_VISIT(np); |
|
1606 |
np->mn_startp = np->mn_outp; |
|
1607 |
} else { |
|
1608 |
if (np->mn_startp == NULL) { |
|
1609 |
if (np->mn_originp == NULL) |
|
1610 |
return (0); |
|
1611 |
else { |
|
1612 |
np = np->mn_originp; |
|
1613 |
continue; |
|
1614 |
} |
|
1615 |
} |
|
1616 |
/* |
|
1617 |
* If ep->me_nodep is a FIFO (me_nodep == NULL), |
|
1618 |
* ignore the edge and move on. ep->me_nodep gets |
|
1619 |
* set to NULL in mux_addedge() if it is a FIFO. |
|
1620 |
* |
|
1621 |
*/ |
|
1622 |
ep = np->mn_startp; |
|
1623 |
np->mn_startp = ep->me_nextp; |
|
1624 |
if (ep->me_nodep == NULL) |
|
1625 |
continue; |
|
1626 |
ep->me_nodep->mn_originp = np; |
|
1627 |
np = ep->me_nodep; |
|
1628 |
} |
|
1629 |
} |
|
1630 |
} |
|
1631 |
||
1632 |
/* |
|
1633 |
* Find linkinfo entry corresponding to the parameters. |
|
1634 |
*/ |
|
1635 |
linkinfo_t * |
|
3448 | 1636 |
findlinks(stdata_t *stp, int index, int type, str_stack_t *ss) |
0 | 1637 |
{ |
1638 |
linkinfo_t *linkp; |
|
1639 |
struct mux_edge *mep; |
|
1640 |
struct mux_node *mnp; |
|
1641 |
queue_t *qup; |
|
1642 |
||
1643 |
mutex_enter(&strresources); |
|
1644 |
if ((type & LINKTYPEMASK) == LINKNORMAL) { |
|
1645 |
qup = getendq(stp->sd_wrq); |
|
1646 |
for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { |
|
1647 |
if ((qup == linkp->li_lblk.l_qtop) && |
|
1648 |
(!index || (index == linkp->li_lblk.l_index))) { |
|
1649 |
mutex_exit(&strresources); |
|
1650 |
return (linkp); |
|
1651 |
} |
|
1652 |
} |
|
1653 |
} else { |
|
1654 |
ASSERT((type & LINKTYPEMASK) == LINKPERSIST); |
|
3448 | 1655 |
mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)]; |
0 | 1656 |
mep = mnp->mn_outp; |
1657 |
while (mep) { |
|
1658 |
if ((index == 0) || (index == mep->me_muxid)) |
|
1659 |
break; |
|
1660 |
mep = mep->me_nextp; |
|
1661 |
} |
|
1662 |
if (!mep) { |
|
1663 |
mutex_exit(&strresources); |
|
1664 |
return (NULL); |
|
1665 |
} |
|
1666 |
for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { |
|
1667 |
if ((!linkp->li_lblk.l_qtop) && |
|
1668 |
(mep->me_muxid == linkp->li_lblk.l_index)) { |
|
1669 |
mutex_exit(&strresources); |
|
1670 |
return (linkp); |
|
1671 |
} |
|
1672 |
} |
|
1673 |
} |
|
1674 |
mutex_exit(&strresources); |
|
1675 |
return (NULL); |
|
1676 |
} |
|
1677 |
||
1678 |
/* |
|
1679 |
* Given a queue ptr, follow the chain of q_next pointers until you reach the |
|
1680 |
* last queue on the chain and return it. |
|
1681 |
*/ |
|
1682 |
queue_t * |
|
1683 |
getendq(queue_t *q) |
|
1684 |
{ |
|
1685 |
ASSERT(q != NULL); |
|
1686 |
while (_SAMESTR(q)) |
|
1687 |
q = q->q_next; |
|
1688 |
return (q); |
|
1689 |
} |
|
1690 |
||
1691 |
/* |
|
1692 |
* wait for the syncq count to drop to zero. |
|
1693 |
* sq could be either outer or inner. |
|
1694 |
*/ |
|
1695 |
||
1696 |
static void |
|
1697 |
wait_syncq(syncq_t *sq) |
|
1698 |
{ |
|
1699 |
uint16_t count; |
|
1700 |
||
1701 |
mutex_enter(SQLOCK(sq)); |
|
1702 |
count = sq->sq_count; |
|
1703 |
SQ_PUTLOCKS_ENTER(sq); |
|
1704 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
1705 |
while (count != 0) { |
|
1706 |
sq->sq_flags |= SQ_WANTWAKEUP; |
|
1707 |
SQ_PUTLOCKS_EXIT(sq); |
|
1708 |
cv_wait(&sq->sq_wait, SQLOCK(sq)); |
|
1709 |
count = sq->sq_count; |
|
1710 |
SQ_PUTLOCKS_ENTER(sq); |
|
1711 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
1712 |
} |
|
1713 |
SQ_PUTLOCKS_EXIT(sq); |
|
1714 |
mutex_exit(SQLOCK(sq)); |
|
1715 |
} |
|
1716 |
||
1717 |
/* |
|
1718 |
* Wait while there are any messages for the queue in its syncq. |
|
1719 |
*/ |
|
1720 |
static void |
|
1721 |
wait_q_syncq(queue_t *q) |
|
1722 |
{ |
|
1723 |
if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { |
|
1724 |
syncq_t *sq = q->q_syncq; |
|
1725 |
||
1726 |
mutex_enter(SQLOCK(sq)); |
|
1727 |
while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { |
|
1728 |
sq->sq_flags |= SQ_WANTWAKEUP; |
|
1729 |
cv_wait(&sq->sq_wait, SQLOCK(sq)); |
|
1730 |
} |
|
1731 |
mutex_exit(SQLOCK(sq)); |
|
1732 |
} |
|
1733 |
} |
|
1734 |
||
1735 |
||
1736 |
int |
|
1737 |
mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp, |
|
1738 |
int lhlink) |
|
1739 |
{ |
|
1740 |
struct stdata *stp; |
|
1741 |
struct strioctl strioc; |
|
1742 |
struct linkinfo *linkp; |
|
1743 |
struct stdata *stpdown; |
|
1744 |
struct streamtab *str; |
|
1745 |
queue_t *passq; |
|
1746 |
syncq_t *passyncq; |
|
1747 |
queue_t *rq; |
|
1748 |
cdevsw_impl_t *dp; |
|
1749 |
uint32_t qflag; |
|
1750 |
uint32_t sqtype; |
|
1751 |
perdm_t *dmp; |
|
1752 |
int error = 0; |
|
3448 | 1753 |
netstack_t *ns; |
1754 |
str_stack_t *ss; |
|
0 | 1755 |
|
1756 |
stp = vp->v_stream; |
|
1757 |
TRACE_1(TR_FAC_STREAMS_FR, |
|
1758 |
TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp); |
|
1759 |
/* |
|
1760 |
* Test for invalid upper stream |
|
1761 |
*/ |
|
1762 |
if (stp->sd_flag & STRHUP) { |
|
1763 |
return (ENXIO); |
|
1764 |
} |
|
1765 |
if (vp->v_type == VFIFO) { |
|
1766 |
return (EINVAL); |
|
1767 |
} |
|
1768 |
if (stp->sd_strtab == NULL) { |
|
1769 |
return (EINVAL); |
|
1770 |
} |
|
1771 |
if (!stp->sd_strtab->st_muxwinit) { |
|
1772 |
return (EINVAL); |
|
1773 |
} |
|
1774 |
if (fpdown == NULL) { |
|
1775 |
return (EBADF); |
|
1776 |
} |
|
3448 | 1777 |
ns = netstack_find_by_cred(crp); |
1778 |
ASSERT(ns != NULL); |
|
1779 |
ss = ns->netstack_str; |
|
1780 |
ASSERT(ss != NULL); |
|
1781 |
||
1782 |
if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) { |
|
1783 |
netstack_rele(ss->ss_netstack); |
|
0 | 1784 |
return (EINVAL); |
1785 |
} |
|
1786 |
mutex_enter(&muxifier); |
|
1787 |
if (stp->sd_flag & STPLEX) { |
|
1788 |
mutex_exit(&muxifier); |
|
3448 | 1789 |
netstack_rele(ss->ss_netstack); |
0 | 1790 |
return (ENXIO); |
1791 |
} |
|
1792 |
||
1793 |
/* |
|
1794 |
* Test for invalid lower stream. |
|
1795 |
* The check for the v_type != VFIFO and having a major |
|
1796 |
* number not >= devcnt is done to avoid problems with |
|
1797 |
* adding mux_node entry past the end of mux_nodes[]. |
|
1798 |
* For FIFO's we don't add an entry so this isn't a |
|
1799 |
* problem. |
|
1800 |
*/ |
|
1801 |
if (((stpdown = fpdown->f_vnode->v_stream) == NULL) || |
|
1802 |
(stpdown == stp) || (stpdown->sd_flag & |
|
1803 |
(STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) || |
|
1804 |
((stpdown->sd_vnode->v_type != VFIFO) && |
|
3448 | 1805 |
(getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) || |
1806 |
linkcycle(stp, stpdown, ss)) { |
|
0 | 1807 |
mutex_exit(&muxifier); |
3448 | 1808 |
netstack_rele(ss->ss_netstack); |
0 | 1809 |
return (EINVAL); |
1810 |
} |
|
1811 |
TRACE_1(TR_FAC_STREAMS_FR, |
|
1812 |
TR_STPDOWN, "stpdown:%p", stpdown); |
|
1813 |
rq = getendq(stp->sd_wrq); |
|
1814 |
if (cmd == I_PLINK) |
|
1815 |
rq = NULL; |
|
1816 |
||
1817 |
linkp = alloclink(rq, stpdown->sd_wrq, fpdown); |
|
1818 |
||
1819 |
strioc.ic_cmd = cmd; |
|
1820 |
strioc.ic_timout = INFTIM; |
|
1821 |
strioc.ic_len = sizeof (struct linkblk); |
|
1822 |
strioc.ic_dp = (char *)&linkp->li_lblk; |
|
1823 |
||
1824 |
/* |
|
1825 |
* STRPLUMB protects plumbing changes and should be set before |
|
1826 |
* link_addpassthru()/link_rempassthru() are called, so it is set here |
|
1827 |
* and cleared in the end of mlink when passthru queue is removed. |
|
1828 |
* Setting of STRPLUMB prevents reopens of the stream while passthru |
|
1829 |
* queue is in-place (it is not a proper module and doesn't have open |
|
1830 |
* entry point). |
|
1831 |
* |
|
1832 |
* STPLEX prevents any threads from entering the stream from above. It |
|
1833 |
* can't be set before the call to link_addpassthru() because putnext |
|
1834 |
* from below may cause stream head I/O routines to be called and these |
|
1835 |
* routines assert that STPLEX is not set. After link_addpassthru() |
|
1836 |
* nothing may come from below since the pass queue syncq is blocked. |
|
1837 |
* Note also that STPLEX should be cleared before the call to |
|
1838 |
* link_remmpassthru() since when messages start flowing to the stream |
|
1839 |
* head (e.g. because of message propagation from the pass queue) stream |
|
1840 |
* head I/O routines may be called with STPLEX flag set. |
|
1841 |
* |
|
1842 |
* When STPLEX is set, nothing may come into the stream from above and |
|
1843 |
* it is safe to do a setq which will change stream head. So, the |
|
1844 |
* correct sequence of actions is: |
|
1845 |
* |
|
1846 |
* 1) Set STRPLUMB |
|
1847 |
* 2) Call link_addpassthru() |
|
1848 |
* 3) Set STPLEX |
|
1849 |
* 4) Call setq and update the stream state |
|
1850 |
* 5) Clear STPLEX |
|
1851 |
* 6) Call link_rempassthru() |
|
1852 |
* 7) Clear STRPLUMB |
|
1853 |
* |
|
1854 |
* The same sequence applies to munlink() code. |
|
1855 |
*/ |
|
1856 |
mutex_enter(&stpdown->sd_lock); |
|
1857 |
stpdown->sd_flag |= STRPLUMB; |
|
1858 |
mutex_exit(&stpdown->sd_lock); |
|
1859 |
/* |
|
1860 |
* Add passthru queue below lower mux. This will block |
|
1861 |
* syncqs of lower muxs read queue during I_LINK/I_UNLINK. |
|
1862 |
*/ |
|
1863 |
passq = link_addpassthru(stpdown); |
|
1864 |
||
1865 |
mutex_enter(&stpdown->sd_lock); |
|
1866 |
stpdown->sd_flag |= STPLEX; |
|
1867 |
mutex_exit(&stpdown->sd_lock); |
|
1868 |
||
1869 |
rq = _RD(stpdown->sd_wrq); |
|
1870 |
/* |
|
1871 |
* There may be messages in the streamhead's syncq due to messages |
|
1872 |
* that arrived before link_addpassthru() was done. To avoid |
|
1873 |
* background processing of the syncq happening simultaneous with |
|
1874 |
* setq processing, we disable the streamhead syncq and wait until |
|
1875 |
* existing background thread finishes working on it. |
|
1876 |
*/ |
|
1877 |
wait_sq_svc(rq->q_syncq); |
|
1878 |
passyncq = passq->q_syncq; |
|
1879 |
if (!(passyncq->sq_flags & SQ_BLOCKED)) |
|
1880 |
blocksq(passyncq, SQ_BLOCKED, 0); |
|
1881 |
||
1882 |
ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); |
|
1883 |
ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); |
|
1884 |
rq->q_ptr = _WR(rq)->q_ptr = NULL; |
|
1885 |
||
1886 |
/* setq might sleep in allocator - avoid holding locks. */ |
|
1887 |
/* Note: we are holding muxifier here. */ |
|
1888 |
||
1889 |
str = stp->sd_strtab; |
|
1890 |
dp = &devimpl[getmajor(vp->v_rdev)]; |
|
1891 |
ASSERT(dp->d_str == str); |
|
1892 |
||
1893 |
qflag = dp->d_qflag; |
|
1894 |
sqtype = dp->d_sqtype; |
|
1895 |
||
1896 |
/* create perdm_t if needed */ |
|
1897 |
if (NEED_DM(dp->d_dmp, qflag)) |
|
1898 |
dp->d_dmp = hold_dm(str, qflag, sqtype); |
|
1899 |
||
1900 |
dmp = dp->d_dmp; |
|
1901 |
||
1902 |
setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype, |
|
1903 |
B_TRUE); |
|
1904 |
||
1905 |
/* |
|
1906 |
* XXX Remove any "odd" messages from the queue. |
|
1907 |
* Keep only M_DATA, M_PROTO, M_PCPROTO. |
|
1908 |
*/ |
|
1909 |
error = strdoioctl(stp, &strioc, FNATIVE, |
|
1910 |
K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); |
|
1911 |
if (error != 0) { |
|
1912 |
lbfree(linkp); |
|
1913 |
||
1914 |
if (!(passyncq->sq_flags & SQ_BLOCKED)) |
|
1915 |
blocksq(passyncq, SQ_BLOCKED, 0); |
|
1916 |
/* |
|
1917 |
* Restore the stream head queue and then remove |
|
1918 |
* the passq. Turn off STPLEX before we turn on |
|
1919 |
* the stream by removing the passq. |
|
1920 |
*/ |
|
1921 |
rq->q_ptr = _WR(rq)->q_ptr = stpdown; |
|
1922 |
setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, |
|
1923 |
B_TRUE); |
|
1924 |
||
1925 |
mutex_enter(&stpdown->sd_lock); |
|
1926 |
stpdown->sd_flag &= ~STPLEX; |
|
1927 |
mutex_exit(&stpdown->sd_lock); |
|
1928 |
||
1929 |
link_rempassthru(passq); |
|
1930 |
||
1931 |
mutex_enter(&stpdown->sd_lock); |
|
1932 |
stpdown->sd_flag &= ~STRPLUMB; |
|
1933 |
/* Wakeup anyone waiting for STRPLUMB to clear. */ |
|
1934 |
cv_broadcast(&stpdown->sd_monitor); |
|
1935 |
mutex_exit(&stpdown->sd_lock); |
|
1936 |
||
1937 |
mutex_exit(&muxifier); |
|
3448 | 1938 |
netstack_rele(ss->ss_netstack); |
0 | 1939 |
return (error); |
1940 |
} |
|
1941 |
mutex_enter(&fpdown->f_tlock); |
|
1942 |
fpdown->f_count++; |
|
1943 |
mutex_exit(&fpdown->f_tlock); |
|
1944 |
||
1945 |
/* |
|
1946 |
* if we've made it here the linkage is all set up so we should also |
|
1947 |
* set up the layered driver linkages |
|
1948 |
*/ |
|
1949 |
||
1950 |
ASSERT((cmd == I_LINK) || (cmd == I_PLINK)); |
|
1951 |
if (cmd == I_LINK) { |
|
1952 |
ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL); |
|
1953 |
} else { |
|
1954 |
ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST); |
|
1955 |
} |
|
1956 |
||
1957 |
link_rempassthru(passq); |
|
1958 |
||
3448 | 1959 |
mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss); |
0 | 1960 |
|
1961 |
/* |
|
1962 |
* Mark the upper stream as having dependent links |
|
1963 |
* so that strclose can clean it up. |
|
1964 |
*/ |
|
1965 |
if (cmd == I_LINK) { |
|
1966 |
mutex_enter(&stp->sd_lock); |
|
1967 |
stp->sd_flag |= STRHASLINKS; |
|
1968 |
mutex_exit(&stp->sd_lock); |
|
1969 |
} |
|
1970 |
/* |
|
1971 |
* Wake up any other processes that may have been |
|
1972 |
* waiting on the lower stream. These will all |
|
1973 |
* error out. |
|
1974 |
*/ |
|
1975 |
mutex_enter(&stpdown->sd_lock); |
|
1976 |
/* The passthru module is removed so we may release STRPLUMB */ |
|
1977 |
stpdown->sd_flag &= ~STRPLUMB; |
|
1978 |
cv_broadcast(&rq->q_wait); |
|
1979 |
cv_broadcast(&_WR(rq)->q_wait); |
|
1980 |
cv_broadcast(&stpdown->sd_monitor); |
|
1981 |
mutex_exit(&stpdown->sd_lock); |
|
1982 |
mutex_exit(&muxifier); |
|
1983 |
*rvalp = linkp->li_lblk.l_index; |
|
3448 | 1984 |
netstack_rele(ss->ss_netstack); |
0 | 1985 |
return (0); |
1986 |
} |
|
1987 |
||
1988 |
int |
|
1989 |
mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink) |
|
1990 |
{ |
|
1991 |
int ret; |
|
1992 |
struct file *fpdown; |
|
1993 |
||
1994 |
fpdown = getf(arg); |
|
1995 |
ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink); |
|
1996 |
if (fpdown != NULL) |
|
1997 |
releasef(arg); |
|
1998 |
return (ret); |
|
1999 |
} |
|
2000 |
||
2001 |
/* |
|
2002 |
* Unlink a multiplexor link. Stp is the controlling stream for the |
|
2003 |
* link, and linkp points to the link's entry in the linkinfo list. |
|
2004 |
* The muxifier lock must be held on entry and is dropped on exit. |
|
2005 |
* |
|
2006 |
* NOTE : Currently it is assumed that mux would process all the messages |
|
2007 |
* sitting on it's queue before ACKing the UNLINK. It is the responsibility |
|
2008 |
* of the mux to handle all the messages that arrive before UNLINK. |
|
2009 |
* If the mux has to send down messages on its lower stream before |
|
2010 |
* ACKing I_UNLINK, then it *should* know to handle messages even |
|
2011 |
* after the UNLINK is acked (actually it should be able to handle till we |
|
2012 |
* re-block the read side of the pass queue here). If the mux does not |
|
2013 |
* open up the lower stream, any messages that arrive during UNLINK |
|
2014 |
* will be put in the stream head. In the case of lower stream opening |
|
2015 |
* up, some messages might land in the stream head depending on when |
|
2016 |
* the message arrived and when the read side of the pass queue was |
|
2017 |
* re-blocked. |
|
2018 |
*/ |
|
2019 |
int |
|
3448 | 2020 |
munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp, |
2021 |
str_stack_t *ss) |
|
0 | 2022 |
{ |
2023 |
struct strioctl strioc; |
|
2024 |
struct stdata *stpdown; |
|
2025 |
queue_t *rq, *wrq; |
|
2026 |
queue_t *passq; |
|
2027 |
syncq_t *passyncq; |
|
2028 |
int error = 0; |
|
2029 |
file_t *fpdown; |
|
2030 |
||
2031 |
ASSERT(MUTEX_HELD(&muxifier)); |
|
2032 |
||
2033 |
stpdown = linkp->li_fpdown->f_vnode->v_stream; |
|
2034 |
||
2035 |
/* |
|
2036 |
* See the comment in mlink() concerning STRPLUMB/STPLEX flags. |
|
2037 |
*/ |
|
2038 |
mutex_enter(&stpdown->sd_lock); |
|
2039 |
stpdown->sd_flag |= STRPLUMB; |
|
2040 |
mutex_exit(&stpdown->sd_lock); |
|
2041 |
||
2042 |
/* |
|
2043 |
* Add passthru queue below lower mux. This will block |
|
2044 |
* syncqs of lower muxs read queue during I_LINK/I_UNLINK. |
|
2045 |
*/ |
|
2046 |
passq = link_addpassthru(stpdown); |
|
2047 |
||
2048 |
if ((flag & LINKTYPEMASK) == LINKNORMAL) |
|
2049 |
strioc.ic_cmd = I_UNLINK; |
|
2050 |
else |
|
2051 |
strioc.ic_cmd = I_PUNLINK; |
|
2052 |
strioc.ic_timout = INFTIM; |
|
2053 |
strioc.ic_len = sizeof (struct linkblk); |
|
2054 |
strioc.ic_dp = (char *)&linkp->li_lblk; |
|
2055 |
||
2056 |
error = strdoioctl(stp, &strioc, FNATIVE, |
|
2057 |
K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); |
|
2058 |
||
2059 |
/* |
|
2060 |
* If there was an error and this is not called via strclose, |
|
2061 |
* return to the user. Otherwise, pretend there was no error |
|
2062 |
* and close the link. |
|
2063 |
*/ |
|
2064 |
if (error) { |
|
2065 |
if (flag & LINKCLOSE) { |
|
2066 |
cmn_err(CE_WARN, "KERNEL: munlink: could not perform " |
|
2067 |
"unlink ioctl, closing anyway (%d)\n", error); |
|
2068 |
} else { |
|
2069 |
link_rempassthru(passq); |
|
2070 |
mutex_enter(&stpdown->sd_lock); |
|
2071 |
stpdown->sd_flag &= ~STRPLUMB; |
|
2072 |
cv_broadcast(&stpdown->sd_monitor); |
|
2073 |
mutex_exit(&stpdown->sd_lock); |
|
2074 |
mutex_exit(&muxifier); |
|
2075 |
return (error); |
|
2076 |
} |
|
2077 |
} |
|
2078 |
||
3448 | 2079 |
mux_rmvedge(stp, linkp->li_lblk.l_index, ss); |
0 | 2080 |
fpdown = linkp->li_fpdown; |
2081 |
lbfree(linkp); |
|
2082 |
||
2083 |
/* |
|
2084 |
* We go ahead and drop muxifier here--it's a nasty global lock that |
|
2085 |
* can slow others down. It's okay to since attempts to mlink() this |
|
2086 |
* stream will be stopped because STPLEX is still set in the stdata |
|
2087 |
* structure, and munlink() is stopped because mux_rmvedge() and |
|
2088 |
* lbfree() have removed it from mux_nodes[] and linkinfo_list, |
|
2089 |
* respectively. Note that we defer the closef() of fpdown until |
|
2090 |
* after we drop muxifier since strclose() can call munlinkall(). |
|
2091 |
*/ |
|
2092 |
mutex_exit(&muxifier); |
|
2093 |
||
2094 |
wrq = stpdown->sd_wrq; |
|
2095 |
rq = _RD(wrq); |
|
2096 |
||
2097 |
/* |
|
2098 |
* Get rid of outstanding service procedure runs, before we make |
|
2099 |
* it a stream head, since a stream head doesn't have any service |
|
2100 |
* procedure. |
|
2101 |
*/ |
|
2102 |
disable_svc(rq); |
|
2103 |
wait_svc(rq); |
|
2104 |
||
2105 |
/* |
|
2106 |
* Since we don't disable the syncq for QPERMOD, we wait for whatever |
|
2107 |
* is queued up to be finished. mux should take care that nothing is |
|
2108 |
* send down to this queue. We should do it now as we're going to block |
|
2109 |
* passyncq if it was unblocked. |
|
2110 |
*/ |
|
2111 |
if (wrq->q_flag & QPERMOD) { |
|
2112 |
syncq_t *sq = wrq->q_syncq; |
|
2113 |
||
2114 |
mutex_enter(SQLOCK(sq)); |
|
2115 |
while (wrq->q_sqflags & Q_SQQUEUED) { |
|
2116 |
sq->sq_flags |= SQ_WANTWAKEUP; |
|
2117 |
cv_wait(&sq->sq_wait, SQLOCK(sq)); |
|
2118 |
} |
|
2119 |
mutex_exit(SQLOCK(sq)); |
|
2120 |
} |
|
2121 |
passyncq = passq->q_syncq; |
|
2122 |
if (!(passyncq->sq_flags & SQ_BLOCKED)) { |
|
2123 |
||
2124 |
syncq_t *sq, *outer; |
|
2125 |
||
2126 |
/* |
|
2127 |
* Messages could be flowing from underneath. We will |
|
2128 |
* block the read side of the passq. This would be |
|
2129 |
* sufficient for QPAIR and QPERQ muxes to ensure |
|
2130 |
* that no data is flowing up into this queue |
|
2131 |
* and hence no thread active in this instance of |
|
2132 |
* lower mux. But for QPERMOD and QMTOUTPERIM there |
|
2133 |
* could be messages on the inner and outer/inner |
|
2134 |
* syncqs respectively. We will wait for them to drain. |
|
2135 |
* Because passq is blocked messages end up in the syncq |
|
2136 |
* And qfill_syncq could possibly end up setting QFULL |
|
2137 |
* which will access the rq->q_flag. Hence, we have to |
|
2138 |
* acquire the QLOCK in setq. |
|
2139 |
* |
|
2140 |
* XXX Messages can also flow from top into this |
|
2141 |
* queue though the unlink is over (Ex. some instance |
|
2142 |
* in putnext() called from top that has still not |
|
2143 |
* accessed this queue. And also putq(lowerq) ?). |
|
2144 |
* Solution : How about blocking the l_qtop queue ? |
|
2145 |
* Do we really care about such pure D_MP muxes ? |
|
2146 |
*/ |
|
2147 |
||
2148 |
blocksq(passyncq, SQ_BLOCKED, 0); |
|
2149 |
||
2150 |
sq = rq->q_syncq; |
|
2151 |
if ((outer = sq->sq_outer) != NULL) { |
|
2152 |
||
2153 |
/* |
|
2154 |
* We have to just wait for the outer sq_count |
|
2155 |
* drop to zero. As this does not prevent new |
|
2156 |
* messages to enter the outer perimeter, this |
|
2157 |
* is subject to starvation. |
|
2158 |
* |
|
2159 |
* NOTE :Because of blocksq above, messages could |
|
2160 |
* be in the inner syncq only because of some |
|
2161 |
* thread holding the outer perimeter exclusively. |
|
2162 |
* Hence it would be sufficient to wait for the |
|
2163 |
* exclusive holder of the outer perimeter to drain |
|
2164 |
* the inner and outer syncqs. But we will not depend |
|
2165 |
* on this feature and hence check the inner syncqs |
|
2166 |
* separately. |
|
2167 |
*/ |
|
2168 |
wait_syncq(outer); |
|
2169 |
} |
|
2170 |
||
2171 |
||
2172 |
/* |
|
2173 |
* There could be messages destined for |
|
2174 |
* this queue. Let the exclusive holder |
|
2175 |
* drain it. |
|
2176 |
*/ |
|
2177 |
||
2178 |
wait_syncq(sq); |
|
2179 |
ASSERT((rq->q_flag & QPERMOD) || |
|
2180 |
((rq->q_syncq->sq_head == NULL) && |
|
2181 |
(_WR(rq)->q_syncq->sq_head == NULL))); |
|
2182 |
} |
|
2183 |
||
2184 |
/* |
|
2185 |
* We haven't taken care of QPERMOD case yet. QPERMOD is a special |
|
2186 |
* case as we don't disable its syncq or remove it off the syncq |
|
2187 |
* service list. |
|
2188 |
*/ |
|
2189 |
if (rq->q_flag & QPERMOD) { |
|
2190 |
syncq_t *sq = rq->q_syncq; |
|
2191 |
||
2192 |
mutex_enter(SQLOCK(sq)); |
|
2193 |
while (rq->q_sqflags & Q_SQQUEUED) { |
|
2194 |
sq->sq_flags |= SQ_WANTWAKEUP; |
|
2195 |
cv_wait(&sq->sq_wait, SQLOCK(sq)); |
|
2196 |
} |
|
2197 |
mutex_exit(SQLOCK(sq)); |
|
2198 |
} |
|
2199 |
||
2200 |
/* |
|
2201 |
* flush_syncq changes states only when there is some messages to |
|
2202 |
* free. ie when it returns non-zero value to return. |
|
2203 |
*/ |
|
2204 |
ASSERT(flush_syncq(rq->q_syncq, rq) == 0); |
|
2205 |
ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0); |
|
2206 |
||
2207 |
/* |
|
2208 |
* No body else should know about this queue now. |
|
2209 |
* If the mux did not process the messages before |
|
2210 |
* acking the I_UNLINK, free them now. |
|
2211 |
*/ |
|
2212 |
||
2213 |
flushq(rq, FLUSHALL); |
|
2214 |
flushq(_WR(rq), FLUSHALL); |
|
2215 |
||
2216 |
/* |
|
2217 |
* Convert the mux lower queue into a stream head queue. |
|
2218 |
* Turn off STPLEX before we turn on the stream by removing the passq. |
|
2219 |
*/ |
|
2220 |
rq->q_ptr = wrq->q_ptr = stpdown; |
|
2221 |
setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE); |
|
2222 |
||
2223 |
ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); |
|
2224 |
ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); |
|
2225 |
||
2226 |
enable_svc(rq); |
|
2227 |
||
2228 |
/* |
|
2229 |
* Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still |
|
2230 |
* needs to be set to prevent reopen() of the stream - such reopen may |
|
2231 |
* try to call non-existent pass queue open routine and panic. |
|
2232 |
*/ |
|
2233 |
mutex_enter(&stpdown->sd_lock); |
|
2234 |
stpdown->sd_flag &= ~STPLEX; |
|
2235 |
mutex_exit(&stpdown->sd_lock); |
|
2236 |
||
2237 |
ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) || |
|
2238 |
((flag & LINKTYPEMASK) == LINKPERSIST)); |
|
2239 |
||
2240 |
/* clean up the layered driver linkages */ |
|
2241 |
if ((flag & LINKTYPEMASK) == LINKNORMAL) { |
|
2242 |
ldi_munlink_fp(stp, fpdown, LINKNORMAL); |
|
2243 |
} else { |
|
2244 |
ldi_munlink_fp(stp, fpdown, LINKPERSIST); |
|
2245 |
} |
|
2246 |
||
2247 |
link_rempassthru(passq); |
|
2248 |
||
2249 |
/* |
|
2250 |
* Now all plumbing changes are finished and STRPLUMB is no |
|
2251 |
* longer needed. |
|
2252 |
*/ |
|
2253 |
mutex_enter(&stpdown->sd_lock); |
|
2254 |
stpdown->sd_flag &= ~STRPLUMB; |
|
2255 |
cv_broadcast(&stpdown->sd_monitor); |
|
2256 |
mutex_exit(&stpdown->sd_lock); |
|
2257 |
||
2258 |
(void) closef(fpdown); |
|
2259 |
return (0); |
|
2260 |
} |
|
2261 |
||
2262 |
/* |
|
2263 |
* Unlink all multiplexor links for which stp is the controlling stream. |
|
2264 |
* Return 0, or a non-zero errno on failure. |
|
2265 |
*/ |
|
2266 |
int |
|
3448 | 2267 |
munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss) |
0 | 2268 |
{ |
2269 |
linkinfo_t *linkp; |
|
2270 |
int error = 0; |
|
2271 |
||
2272 |
mutex_enter(&muxifier); |
|
3448 | 2273 |
while (linkp = findlinks(stp, 0, flag, ss)) { |
0 | 2274 |
/* |
2275 |
* munlink() releases the muxifier lock. |
|
2276 |
*/ |
|
3448 | 2277 |
if (error = munlink(stp, linkp, flag, crp, rvalp, ss)) |
0 | 2278 |
return (error); |
2279 |
mutex_enter(&muxifier); |
|
2280 |
} |
|
2281 |
mutex_exit(&muxifier); |
|
2282 |
return (0); |
|
2283 |
} |
|
2284 |
||
2285 |
/* |
|
2286 |
* A multiplexor link has been made. Add an |
|
2287 |
* edge to the directed graph. |
|
2288 |
*/ |
|
2289 |
void |
|
3448 | 2290 |
mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss) |
0 | 2291 |
{ |
2292 |
struct mux_node *np; |
|
2293 |
struct mux_edge *ep; |
|
2294 |
major_t upmaj; |
|
2295 |
major_t lomaj; |
|
2296 |
||
2297 |
upmaj = getmajor(upstp->sd_vnode->v_rdev); |
|
2298 |
lomaj = getmajor(lostp->sd_vnode->v_rdev); |
|
3448 | 2299 |
np = &ss->ss_mux_nodes[upmaj]; |
0 | 2300 |
if (np->mn_outp) { |
2301 |
ep = np->mn_outp; |
|
2302 |
while (ep->me_nextp) |
|
2303 |
ep = ep->me_nextp; |
|
2304 |
ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); |
|
2305 |
ep = ep->me_nextp; |
|
2306 |
} else { |
|
2307 |
np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); |
|
2308 |
ep = np->mn_outp; |
|
2309 |
} |
|
2310 |
ep->me_nextp = NULL; |
|
2311 |
ep->me_muxid = muxid; |
|
3448 | 2312 |
/* |
2313 |
* Save the dev_t for the purposes of str_stack_shutdown. |
|
2314 |
* str_stack_shutdown assumes that the device allows reopen, since |
|
2315 |
* this dev_t is the one after any cloning by xx_open(). |
|
2316 |
* Would prefer finding the dev_t from before any cloning, |
|
2317 |
* but specfs doesn't retain that. |
|
2318 |
*/ |
|
2319 |
ep->me_dev = upstp->sd_vnode->v_rdev; |
|
0 | 2320 |
if (lostp->sd_vnode->v_type == VFIFO) |
2321 |
ep->me_nodep = NULL; |
|
2322 |
else |
|
3448 | 2323 |
ep->me_nodep = &ss->ss_mux_nodes[lomaj]; |
0 | 2324 |
} |
2325 |
||
2326 |
/* |
|
2327 |
* A multiplexor link has been removed. Remove the |
|
2328 |
* edge in the directed graph. |
|
2329 |
*/ |
|
2330 |
void |
|
3448 | 2331 |
mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss) |
0 | 2332 |
{ |
2333 |
struct mux_node *np; |
|
2334 |
struct mux_edge *ep; |
|
2335 |
struct mux_edge *pep = NULL; |
|
2336 |
major_t upmaj; |
|
2337 |
||
2338 |
upmaj = getmajor(upstp->sd_vnode->v_rdev); |
|
3448 | 2339 |
np = &ss->ss_mux_nodes[upmaj]; |
0 | 2340 |
ASSERT(np->mn_outp != NULL); |
2341 |
ep = np->mn_outp; |
|
2342 |
while (ep) { |
|
2343 |
if (ep->me_muxid == muxid) { |
|
2344 |
if (pep) |
|
2345 |
pep->me_nextp = ep->me_nextp; |
|
2346 |
else |
|
2347 |
np->mn_outp = ep->me_nextp; |
|
2348 |
kmem_free(ep, sizeof (struct mux_edge)); |
|
2349 |
return; |
|
2350 |
} |
|
2351 |
pep = ep; |
|
2352 |
ep = ep->me_nextp; |
|
2353 |
} |
|
2354 |
ASSERT(0); /* should not reach here */ |
|
2355 |
} |
|
2356 |
||
2357 |
/* |
|
2358 |
* Translate the device flags (from conf.h) to the corresponding |
|
2359 |
* qflag and sq_flag (type) values. |
|
2360 |
*/ |
|
2361 |
int |
|
2362 |
devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp, |
|
2363 |
uint32_t *sqtypep) |
|
2364 |
{ |
|
2365 |
uint32_t qflag = 0; |
|
2366 |
uint32_t sqtype = 0; |
|
2367 |
||
2368 |
if (devflag & _D_OLD) |
|
2369 |
goto bad; |
|
2370 |
||
2371 |
/* Inner perimeter presence and scope */ |
|
2372 |
switch (devflag & D_MTINNER_MASK) { |
|
2373 |
case D_MP: |
|
2374 |
qflag |= QMTSAFE; |
|
2375 |
sqtype |= SQ_CI; |
|
2376 |
break; |
|
2377 |
case D_MTPERQ|D_MP: |
|
2378 |
qflag |= QPERQ; |
|
2379 |
break; |
|
2380 |
case D_MTQPAIR|D_MP: |
|
2381 |
qflag |= QPAIR; |
|
2382 |
break; |
|
2383 |
case D_MTPERMOD|D_MP: |
|
2384 |
qflag |= QPERMOD; |
|
2385 |
break; |
|
2386 |
default: |
|
2387 |
goto bad; |
|
2388 |
} |
|
2389 |
||
2390 |
/* Outer perimeter */ |
|
2391 |
if (devflag & D_MTOUTPERIM) { |
|
2392 |
switch (devflag & D_MTINNER_MASK) { |
|
2393 |
case D_MP: |
|
2394 |
case D_MTPERQ|D_MP: |
|
2395 |
case D_MTQPAIR|D_MP: |
|
2396 |
break; |
|
2397 |
default: |
|
2398 |
goto bad; |
|
2399 |
} |
|
2400 |
qflag |= QMTOUTPERIM; |
|
2401 |
} |
|
2402 |
||
2403 |
/* Inner perimeter modifiers */ |
|
2404 |
if (devflag & D_MTINNER_MOD) { |
|
2405 |
switch (devflag & D_MTINNER_MASK) { |
|
2406 |
case D_MP: |
|
2407 |
goto bad; |
|
2408 |
default: |
|
2409 |
break; |
|
2410 |
} |
|
2411 |
if (devflag & D_MTPUTSHARED) |
|
2412 |
sqtype |= SQ_CIPUT; |
|
2413 |
if (devflag & _D_MTOCSHARED) { |
|
2414 |
/* |
|
2415 |
* The code in putnext assumes that it has the |
|
2416 |
* highest concurrency by not checking sq_count. |
|
2417 |
* Thus _D_MTOCSHARED can only be supported when |
|
2418 |
* D_MTPUTSHARED is set. |
|
2419 |
*/ |
|
2420 |
if (!(devflag & D_MTPUTSHARED)) |
|
2421 |
goto bad; |
|
2422 |
sqtype |= SQ_CIOC; |
|
2423 |
} |
|
2424 |
if (devflag & _D_MTCBSHARED) { |
|
2425 |
/* |
|
2426 |
* The code in putnext assumes that it has the |
|
2427 |
* highest concurrency by not checking sq_count. |
|
2428 |
* Thus _D_MTCBSHARED can only be supported when |
|
2429 |
* D_MTPUTSHARED is set. |
|
2430 |
*/ |
|
2431 |
if (!(devflag & D_MTPUTSHARED)) |
|
2432 |
goto bad; |
|
2433 |
sqtype |= SQ_CICB; |
|
2434 |
} |
|
2435 |
if (devflag & _D_MTSVCSHARED) { |
|
2436 |
/* |
|
2437 |
* The code in putnext assumes that it has the |
|
2438 |
* highest concurrency by not checking sq_count. |
|
2439 |
* Thus _D_MTSVCSHARED can only be supported when |
|
2440 |
* D_MTPUTSHARED is set. Also _D_MTSVCSHARED is |
|
2441 |
* supported only for QPERMOD. |
|
2442 |
*/ |
|
2443 |
if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD)) |
|
2444 |
goto bad; |
|
2445 |
sqtype |= SQ_CISVC; |
|
2446 |
} |
|
2447 |
} |
|
2448 |
||
2449 |
/* Default outer perimeter concurrency */ |
|
2450 |
sqtype |= SQ_CO; |
|
2451 |
||
2452 |
/* Outer perimeter modifiers */ |
|
2453 |
if (devflag & D_MTOCEXCL) { |
|
2454 |
if (!(devflag & D_MTOUTPERIM)) { |
|
2455 |
/* No outer perimeter */ |
|
2456 |
goto bad; |
|
2457 |
} |
|
2458 |
sqtype &= ~SQ_COOC; |
|
2459 |
} |
|
2460 |
||
2461 |
/* Synchronous Streams extended qinit structure */ |
|
2462 |
if (devflag & D_SYNCSTR) |
|
2463 |
qflag |= QSYNCSTR; |
|
2464 |
||
741
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
2465 |
/* |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
2466 |
* Private flag used by a transport module to indicate |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
2467 |
* to sockfs that it supports direct-access mode without |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
2468 |
* having to go through STREAMS. |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
2469 |
*/ |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
2470 |
if (devflag & _D_DIRECT) { |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
2471 |
/* Reject unless the module is fully-MT (no perimeter) */ |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
2472 |
if ((qflag & QMT_TYPEMASK) != QMTSAFE) |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
2473 |
goto bad; |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
2474 |
qflag |= _QDIRECT; |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
2475 |
} |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
2476 |
|
0 | 2477 |
*qflagp = qflag; |
2478 |
*sqtypep = sqtype; |
|
2479 |
return (0); |
|
2480 |
||
2481 |
bad: |
|
2482 |
cmn_err(CE_WARN, |
|
2483 |
"stropen: bad MT flags (0x%x) in driver '%s'", |
|
2484 |
(int)(qflag & D_MTSAFETY_MASK), |
|
2485 |
stp->st_rdinit->qi_minfo->mi_idname); |
|
2486 |
||
2487 |
return (EINVAL); |
|
2488 |
} |
|
2489 |
||
2490 |
/* |
|
2491 |
* Set the interface values for a pair of queues (qinit structure, |
|
2492 |
* packet sizes, water marks). |
|
2493 |
* setq assumes that the caller does not have a claim (entersq or claimq) |
|
2494 |
* on the queue. |
|
2495 |
*/ |
|
2496 |
void |
|
2497 |
setq(queue_t *rq, struct qinit *rinit, struct qinit *winit, |
|
2498 |
perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed) |
|
2499 |
{ |
|
2500 |
queue_t *wq; |
|
2501 |
syncq_t *sq, *outer; |
|
2502 |
||
2503 |
ASSERT(rq->q_flag & QREADR); |
|
2504 |
ASSERT((qflag & QMT_TYPEMASK) != 0); |
|
2505 |
IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); |
|
2506 |
||
2507 |
wq = _WR(rq); |
|
2508 |
rq->q_qinfo = rinit; |
|
2509 |
rq->q_hiwat = rinit->qi_minfo->mi_hiwat; |
|
2510 |
rq->q_lowat = rinit->qi_minfo->mi_lowat; |
|
2511 |
rq->q_minpsz = rinit->qi_minfo->mi_minpsz; |
|
2512 |
rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz; |
|
2513 |
wq->q_qinfo = winit; |
|
2514 |
wq->q_hiwat = winit->qi_minfo->mi_hiwat; |
|
2515 |
wq->q_lowat = winit->qi_minfo->mi_lowat; |
|
2516 |
wq->q_minpsz = winit->qi_minfo->mi_minpsz; |
|
2517 |
wq->q_maxpsz = winit->qi_minfo->mi_maxpsz; |
|
2518 |
||
2519 |
/* Remove old syncqs */ |
|
2520 |
sq = rq->q_syncq; |
|
2521 |
outer = sq->sq_outer; |
|
2522 |
if (outer != NULL) { |
|
2523 |
ASSERT(wq->q_syncq->sq_outer == outer); |
|
2524 |
outer_remove(outer, rq->q_syncq); |
|
2525 |
if (wq->q_syncq != rq->q_syncq) |
|
2526 |
outer_remove(outer, wq->q_syncq); |
|
2527 |
} |
|
2528 |
ASSERT(sq->sq_outer == NULL); |
|
2529 |
ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); |
|
2530 |
||
2531 |
if (sq != SQ(rq)) { |
|
2532 |
if (!(rq->q_flag & QPERMOD)) |
|
2533 |
free_syncq(sq); |
|
2534 |
if (wq->q_syncq == rq->q_syncq) |
|
2535 |
wq->q_syncq = NULL; |
|
2536 |
rq->q_syncq = NULL; |
|
2537 |
} |
|
2538 |
if (wq->q_syncq != NULL && wq->q_syncq != sq && |
|
2539 |
wq->q_syncq != SQ(rq)) { |
|
2540 |
free_syncq(wq->q_syncq); |
|
2541 |
wq->q_syncq = NULL; |
|
2542 |
} |
|
2543 |
ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL && |
|
2544 |
rq->q_syncq->sq_tail == NULL)); |
|
2545 |
ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL && |
|
2546 |
wq->q_syncq->sq_tail == NULL)); |
|
2547 |
||
2548 |
if (!(rq->q_flag & QPERMOD) && |
|
2549 |
rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) { |
|
2550 |
ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); |
|
2551 |
SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl, |
|
2552 |
rq->q_syncq->sq_nciputctrl, 0); |
|
2553 |
ASSERT(ciputctrl_cache != NULL); |
|
2554 |
kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl); |
|
2555 |
rq->q_syncq->sq_ciputctrl = NULL; |
|
2556 |
rq->q_syncq->sq_nciputctrl = 0; |
|
2557 |
} |
|
2558 |
||
2559 |
if (!(wq->q_flag & QPERMOD) && |
|
2560 |
wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) { |
|
2561 |
ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); |
|
2562 |
SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl, |
|
2563 |
wq->q_syncq->sq_nciputctrl, 0); |
|
2564 |
ASSERT(ciputctrl_cache != NULL); |
|
2565 |
kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl); |
|
2566 |
wq->q_syncq->sq_ciputctrl = NULL; |
|
2567 |
wq->q_syncq->sq_nciputctrl = 0; |
|
2568 |
} |
|
2569 |
||
2570 |
sq = SQ(rq); |
|
2571 |
ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); |
|
2572 |
ASSERT(sq->sq_outer == NULL); |
|
2573 |
ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); |
|
2574 |
||
2575 |
/* |
|
2576 |
* Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS |
|
2577 |
* bits in sq_flag based on the sqtype. |
|
2578 |
*/ |
|
2579 |
ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0); |
|
2580 |
||
2581 |
rq->q_syncq = wq->q_syncq = sq; |
|
2582 |
sq->sq_type = sqtype; |
|
2583 |
sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS); |
|
2584 |
||
2585 |
/* |
|
2586 |
* We are making sq_svcflags zero, |
|
2587 |
* resetting SQ_DISABLED in case it was set by |
|
2588 |
* wait_svc() in the munlink path. |
|
2589 |
* |
|
2590 |
*/ |
|
2591 |
ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0); |
|
2592 |
sq->sq_svcflags = 0; |
|
2593 |
||
2594 |
/* |
|
2595 |
* We need to acquire the lock here for the mlink and munlink case, |
|
2596 |
* where canputnext, backenable, etc can access the q_flag. |
|
2597 |
*/ |
|
2598 |
if (lock_needed) { |
|
2599 |
mutex_enter(QLOCK(rq)); |
|
2600 |
rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; |
|
2601 |
mutex_exit(QLOCK(rq)); |
|
2602 |
mutex_enter(QLOCK(wq)); |
|
2603 |
wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; |
|
2604 |
mutex_exit(QLOCK(wq)); |
|
2605 |
} else { |
|
2606 |
rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; |
|
2607 |
wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; |
|
2608 |
} |
|
2609 |
||
2610 |
if (qflag & QPERQ) { |
|
2611 |
/* Allocate a separate syncq for the write side */ |
|
2612 |
sq = new_syncq(); |
|
2613 |
sq->sq_type = rq->q_syncq->sq_type; |
|
2614 |
sq->sq_flags = rq->q_syncq->sq_flags; |
|
2615 |
ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && |
|
2616 |
sq->sq_oprev == NULL); |
|
2617 |
wq->q_syncq = sq; |
|
2618 |
} |
|
2619 |
if (qflag & QPERMOD) { |
|
2620 |
sq = dmp->dm_sq; |
|
2621 |
||
2622 |
/* |
|
2623 |
* Assert that we do have an inner perimeter syncq and that it |
|
2624 |
* does not have an outer perimeter associated with it. |
|
2625 |
*/ |
|
2626 |
ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && |
|
2627 |
sq->sq_oprev == NULL); |
|
2628 |
rq->q_syncq = wq->q_syncq = sq; |
|
2629 |
} |
|
2630 |
if (qflag & QMTOUTPERIM) { |
|
2631 |
outer = dmp->dm_sq; |
|
2632 |
||
2633 |
ASSERT(outer->sq_outer == NULL); |
|
2634 |
outer_insert(outer, rq->q_syncq); |
|
2635 |
if (wq->q_syncq != rq->q_syncq) |
|
2636 |
outer_insert(outer, wq->q_syncq); |
|
2637 |
} |
|
2638 |
ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == |
|
2639 |
(rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); |
|
2640 |
ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == |
|
2641 |
(wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); |
|
2642 |
ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK)); |
|
2643 |
||
2644 |
/* |
|
2645 |
* Initialize struio() types. |
|
2646 |
*/ |
|
2647 |
rq->q_struiot = |
|
2648 |
(rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE; |
|
2649 |
wq->q_struiot = |
|
2650 |
(wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE; |
|
2651 |
} |
|
2652 |
||
2653 |
perdm_t * |
|
2654 |
hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype) |
|
2655 |
{ |
|
2656 |
syncq_t *sq; |
|
2657 |
perdm_t **pp; |
|
2658 |
perdm_t *p; |
|
2659 |
perdm_t *dmp; |
|
2660 |
||
2661 |
ASSERT(str != NULL); |
|
2662 |
ASSERT(qflag & (QPERMOD | QMTOUTPERIM)); |
|
2663 |
||
2664 |
rw_enter(&perdm_rwlock, RW_READER); |
|
2665 |
for (p = perdm_list; p != NULL; p = p->dm_next) { |
|
2666 |
if (p->dm_str == str) { /* found one */ |
|
2667 |
atomic_add_32(&(p->dm_ref), 1); |
|
2668 |
rw_exit(&perdm_rwlock); |
|
2669 |
return (p); |
|
2670 |
} |
|
2671 |
} |
|
2672 |
rw_exit(&perdm_rwlock); |
|
2673 |
||
2674 |
sq = new_syncq(); |
|
2675 |
if (qflag & QPERMOD) { |
|
2676 |
sq->sq_type = sqtype | SQ_PERMOD; |
|
2677 |
sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS; |
|
2678 |
} else { |
|
2679 |
ASSERT(qflag & QMTOUTPERIM); |
|
2680 |
sq->sq_onext = sq->sq_oprev = sq; |
|
2681 |
} |
|
2682 |
||
2683 |
dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP); |
|
2684 |
dmp->dm_sq = sq; |
|
2685 |
dmp->dm_str = str; |
|
2686 |
dmp->dm_ref = 1; |
|
2687 |
dmp->dm_next = NULL; |
|
2688 |
||
2689 |
rw_enter(&perdm_rwlock, RW_WRITER); |
|
2690 |
for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) { |
|
2691 |
if (p->dm_str == str) { /* already present */ |
|
2692 |
p->dm_ref++; |
|
2693 |
rw_exit(&perdm_rwlock); |
|
2694 |
free_syncq(sq); |
|
2695 |
kmem_free(dmp, sizeof (perdm_t)); |
|
2696 |
return (p); |
|
2697 |
} |
|
2698 |
} |
|
2699 |
||
2700 |
*pp = dmp; |
|
2701 |
rw_exit(&perdm_rwlock); |
|
2702 |
return (dmp); |
|
2703 |
} |
|
2704 |
||
2705 |
void |
|
2706 |
rele_dm(perdm_t *dmp) |
|
2707 |
{ |
|
2708 |
perdm_t **pp; |
|
2709 |
perdm_t *p; |
|
2710 |
||
2711 |
rw_enter(&perdm_rwlock, RW_WRITER); |
|
2712 |
ASSERT(dmp->dm_ref > 0); |
|
2713 |
||
2714 |
if (--dmp->dm_ref > 0) { |
|
2715 |
rw_exit(&perdm_rwlock); |
|
2716 |
return; |
|
2717 |
} |
|
2718 |
||
2719 |
for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) |
|
2720 |
if (p == dmp) |
|
2721 |
break; |
|
2722 |
ASSERT(p == dmp); |
|
2723 |
*pp = p->dm_next; |
|
2724 |
rw_exit(&perdm_rwlock); |
|
2725 |
||
2726 |
/* |
|
2727 |
* Wait for any background processing that relies on the |
|
2728 |
* syncq to complete before it is freed. |
|
2729 |
*/ |
|
2730 |
wait_sq_svc(p->dm_sq); |
|
2731 |
free_syncq(p->dm_sq); |
|
2732 |
kmem_free(p, sizeof (perdm_t)); |
|
2733 |
} |
|
2734 |
||
2735 |
/* |
|
2736 |
* Make a protocol message given control and data buffers. |
|
2737 |
* n.b., this can block; be careful of what locks you hold when calling it. |
|
2738 |
* |
|
2739 |
* If sd_maxblk is less than *iosize this routine can fail part way through |
|
2740 |
* (due to an allocation failure). In this case on return *iosize will contain |
|
2741 |
* the amount that was consumed. Otherwise *iosize will not be modified |
|
2742 |
* i.e. it will contain the amount that was consumed. |
|
2743 |
*/ |
|
2744 |
int |
|
2745 |
strmakemsg( |
|
2746 |
struct strbuf *mctl, |
|
2747 |
ssize_t *iosize, |
|
2748 |
struct uio *uiop, |
|
2749 |
stdata_t *stp, |
|
2750 |
int32_t flag, |
|
2751 |
mblk_t **mpp) |
|
2752 |
{ |
|
2753 |
mblk_t *mpctl = NULL; |
|
2754 |
mblk_t *mpdata = NULL; |
|
2755 |
int error; |
|
2756 |
||
2757 |
ASSERT(uiop != NULL); |
|
2758 |
||
2759 |
*mpp = NULL; |
|
2760 |
/* Create control part, if any */ |
|
2761 |
if ((mctl != NULL) && (mctl->len >= 0)) { |
|
2762 |
error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl); |
|
2763 |
if (error) |
|
2764 |
return (error); |
|
2765 |
} |
|
2766 |
/* Create data part, if any */ |
|
2767 |
if (*iosize >= 0) { |
|
2768 |
error = strmakedata(iosize, uiop, stp, flag, &mpdata); |
|
2769 |
if (error) { |
|
2770 |
freemsg(mpctl); |
|
2771 |
return (error); |
|
2772 |
} |
|
2773 |
} |
|
2774 |
if (mpctl != NULL) { |
|
2775 |
if (mpdata != NULL) |
|
2776 |
linkb(mpctl, mpdata); |
|
2777 |
*mpp = mpctl; |
|
2778 |
} else { |
|
2779 |
*mpp = mpdata; |
|
2780 |
} |
|
2781 |
return (0); |
|
2782 |
} |
|
2783 |
||
2784 |
/* |
|
2785 |
* Make the control part of a protocol message given a control buffer. |
|
2786 |
* n.b., this can block; be careful of what locks you hold when calling it. |
|
2787 |
*/ |
|
2788 |
int |
|
2789 |
strmakectl( |
|
2790 |
struct strbuf *mctl, |
|
2791 |
int32_t flag, |
|
2792 |
int32_t fflag, |
|
2793 |
mblk_t **mpp) |
|
2794 |
{ |
|
2795 |
mblk_t *bp = NULL; |
|
2796 |
unsigned char msgtype; |
|
2797 |
int error = 0; |
|
2798 |
||
2799 |
*mpp = NULL; |
|
2800 |
/* |
|
2801 |
* Create control part of message, if any. |
|
2802 |
*/ |
|
2803 |
if ((mctl != NULL) && (mctl->len >= 0)) { |
|
2804 |
caddr_t base; |
|
2805 |
int ctlcount; |
|
2806 |
int allocsz; |
|
2807 |
||
2808 |
if (flag & RS_HIPRI) |
|
2809 |
msgtype = M_PCPROTO; |
|
2810 |
else |
|
2811 |
msgtype = M_PROTO; |
|
2812 |
||
2813 |
ctlcount = mctl->len; |
|
2814 |
base = mctl->buf; |
|
2815 |
||
2816 |
/* |
|
2817 |
* Give modules a better chance to reuse M_PROTO/M_PCPROTO |
|
2818 |
* blocks by increasing the size to something more usable. |
|
2819 |
*/ |
|
2820 |
allocsz = MAX(ctlcount, 64); |
|
2821 |
||
2822 |
/* |
|
2823 |
* Range checking has already been done; simply try |
|
2824 |
* to allocate a message block for the ctl part. |
|
2825 |
*/ |
|
2826 |
while (!(bp = allocb(allocsz, BPRI_MED))) { |
|
2827 |
if (fflag & (FNDELAY|FNONBLOCK)) |
|
2828 |
return (EAGAIN); |
|
2829 |
if (error = strwaitbuf(allocsz, BPRI_MED)) |
|
2830 |
return (error); |
|
2831 |
} |
|
2832 |
||
2833 |
bp->b_datap->db_type = msgtype; |
|
2834 |
if (copyin(base, bp->b_wptr, ctlcount)) { |
|
2835 |
freeb(bp); |
|
2836 |
return (EFAULT); |
|
2837 |
} |
|
2838 |
bp->b_wptr += ctlcount; |
|
2839 |
} |
|
2840 |
*mpp = bp; |
|
2841 |
return (0); |
|
2842 |
} |
|
2843 |
||
2844 |
/* |
|
2845 |
* Make a protocol message given data buffers. |
|
2846 |
* n.b., this can block; be careful of what locks you hold when calling it. |
|
2847 |
* |
|
2848 |
* If sd_maxblk is less than *iosize this routine can fail part way through |
|
2849 |
* (due to an allocation failure). In this case on return *iosize will contain |
|
2850 |
* the amount that was consumed. Otherwise *iosize will not be modified |
|
2851 |
* i.e. it will contain the amount that was consumed. |
|
2852 |
*/ |
|
2853 |
int |
|
2854 |
strmakedata( |
|
2855 |
ssize_t *iosize, |
|
2856 |
struct uio *uiop, |
|
2857 |
stdata_t *stp, |
|
2858 |
int32_t flag, |
|
2859 |
mblk_t **mpp) |
|
2860 |
{ |
|
2861 |
mblk_t *mp = NULL; |
|
2862 |
mblk_t *bp; |
|
2863 |
int wroff = (int)stp->sd_wroff; |
|
898 | 2864 |
int tail_len = (int)stp->sd_tail; |
2865 |
int extra = wroff + tail_len; |
|
0 | 2866 |
int error = 0; |
2867 |
ssize_t maxblk; |
|
2868 |
ssize_t count = *iosize; |
|
2869 |
cred_t *cr = CRED(); |
|
2870 |
||
2871 |
*mpp = NULL; |
|
2872 |
if (count < 0) |
|
2873 |
return (0); |
|
2874 |
||
2875 |
maxblk = stp->sd_maxblk; |
|
2876 |
if (maxblk == INFPSZ) |
|
2877 |
maxblk = count; |
|
2878 |
||
2879 |
/* |
|
2880 |
* Create data part of message, if any. |
|
2881 |
*/ |
|
2882 |
do { |
|
2883 |
ssize_t size; |
|
2884 |
dblk_t *dp; |
|
2885 |
||
2886 |
ASSERT(uiop); |
|
2887 |
||
2888 |
size = MIN(count, maxblk); |
|
2889 |
||
898 | 2890 |
while ((bp = allocb_cred(size + extra, cr)) == NULL) { |
0 | 2891 |
error = EAGAIN; |
2892 |
if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) || |
|
898 | 2893 |
(error = strwaitbuf(size + extra, BPRI_MED)) != 0) { |
0 | 2894 |
if (count == *iosize) { |
2895 |
freemsg(mp); |
|
2896 |
return (error); |
|
2897 |
} else { |
|
2898 |
*iosize -= count; |
|
2899 |
*mpp = mp; |
|
2900 |
return (0); |
|
2901 |
} |
|
2902 |
} |
|
2903 |
} |
|
2904 |
dp = bp->b_datap; |
|
2905 |
dp->db_cpid = curproc->p_pid; |
|
2906 |
ASSERT(wroff <= dp->db_lim - bp->b_wptr); |
|
2907 |
bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff; |
|
2908 |
||
2909 |
if (flag & STRUIO_POSTPONE) { |
|
2910 |
/* |
|
2911 |
* Setup the stream uio portion of the |
|
2912 |
* dblk for subsequent use by struioget(). |
|
2913 |
*/ |
|
2914 |
dp->db_struioflag = STRUIO_SPEC; |
|
2915 |
dp->db_cksumstart = 0; |
|
2916 |
dp->db_cksumstuff = 0; |
|
2917 |
dp->db_cksumend = size; |
|
2918 |
*(long long *)dp->db_struioun.data = 0ll; |
|
898 | 2919 |
bp->b_wptr += size; |
0 | 2920 |
} else { |
2921 |
if (stp->sd_copyflag & STRCOPYCACHED) |
|
2922 |
uiop->uio_extflg |= UIO_COPY_CACHED; |
|
2923 |
||
2924 |
if (size != 0) { |
|
2925 |
error = uiomove(bp->b_wptr, size, UIO_WRITE, |
|
2926 |
uiop); |
|
2927 |
if (error != 0) { |
|
2928 |
freeb(bp); |
|
2929 |
freemsg(mp); |
|
2930 |
return (error); |
|
2931 |
} |
|
2932 |
} |
|
898 | 2933 |
bp->b_wptr += size; |
2934 |
||
2935 |
if (stp->sd_wputdatafunc != NULL) { |
|
2936 |
mblk_t *newbp; |
|
2937 |
||
2938 |
newbp = (stp->sd_wputdatafunc)(stp->sd_vnode, |
|
2939 |
bp, NULL, NULL, NULL, NULL); |
|
2940 |
if (newbp == NULL) { |
|
2941 |
freeb(bp); |
|
2942 |
freemsg(mp); |
|
2943 |
return (ECOMM); |
|
2944 |
} |
|
2945 |
bp = newbp; |
|
2946 |
} |
|
2947 |
} |
|
2948 |
||
0 | 2949 |
count -= size; |
2950 |
||
2951 |
if (mp == NULL) |
|
2952 |
mp = bp; |
|
2953 |
else |
|
2954 |
linkb(mp, bp); |
|
2955 |
} while (count > 0); |
|
2956 |
||
2957 |
*mpp = mp; |
|
2958 |
return (0); |
|
2959 |
} |
|
2960 |
||
2961 |
/* |
|
2962 |
* Wait for a buffer to become available. Return non-zero errno |
|
2963 |
* if not able to wait, 0 if buffer is probably there. |
|
2964 |
*/ |
|
2965 |
int |
|
2966 |
strwaitbuf(size_t size, int pri) |
|
2967 |
{ |
|
2968 |
bufcall_id_t id; |
|
2969 |
||
2970 |
mutex_enter(&bcall_monitor); |
|
2971 |
if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast, |
|
2972 |
&ttoproc(curthread)->p_flag_cv)) == 0) { |
|
2973 |
mutex_exit(&bcall_monitor); |
|
2974 |
return (ENOSR); |
|
2975 |
} |
|
2976 |
if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) { |
|
2977 |
unbufcall(id); |
|
2978 |
mutex_exit(&bcall_monitor); |
|
2979 |
return (EINTR); |
|
2980 |
} |
|
2981 |
unbufcall(id); |
|
2982 |
mutex_exit(&bcall_monitor); |
|
2983 |
return (0); |
|
2984 |
} |
|
2985 |
||
2986 |
/* |
|
2987 |
* This function waits for a read or write event to happen on a stream. |
|
2988 |
* fmode can specify FNDELAY and/or FNONBLOCK. |
|
2989 |
* The timeout is in ms with -1 meaning infinite. |
|
2990 |
* The flag values work as follows: |
|
2991 |
* READWAIT Check for read side errors, send M_READ |
|
2992 |
* GETWAIT Check for read side errors, no M_READ |
|
2993 |
* WRITEWAIT Check for write side errors. |
|
2994 |
* NOINTR Do not return error if nonblocking or timeout. |
|
2995 |
* STR_NOERROR Ignore all errors except STPLEX. |
|
2996 |
* STR_NOSIG Ignore/hold signals during the duration of the call. |
|
2997 |
* STR_PEEK Pass through the strgeterr(). |
|
2998 |
*/ |
|
2999 |
int |
|
3000 |
strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout, |
|
3001 |
int *done) |
|
3002 |
{ |
|
3003 |
int slpflg, errs; |
|
3004 |
int error; |
|
3005 |
kcondvar_t *sleepon; |
|
3006 |
mblk_t *mp; |
|
3007 |
ssize_t *rd_count; |
|
3008 |
clock_t rval; |
|
3009 |
||
3010 |
ASSERT(MUTEX_HELD(&stp->sd_lock)); |
|
3011 |
if ((flag & READWAIT) || (flag & GETWAIT)) { |
|
3012 |
slpflg = RSLEEP; |
|
3013 |
sleepon = &_RD(stp->sd_wrq)->q_wait; |
|
3014 |
errs = STRDERR|STPLEX; |
|
3015 |
} else { |
|
3016 |
slpflg = WSLEEP; |
|
3017 |
sleepon = &stp->sd_wrq->q_wait; |
|
3018 |
errs = STWRERR|STRHUP|STPLEX; |
|
3019 |
} |
|
3020 |
if (flag & STR_NOERROR) |
|
3021 |
errs = STPLEX; |
|
3022 |
||
3023 |
if (stp->sd_wakeq & slpflg) { |
|
3024 |
/* |
|
3025 |
* A strwakeq() is pending, no need to sleep. |
|
3026 |
*/ |
|
3027 |
stp->sd_wakeq &= ~slpflg; |
|
3028 |
*done = 0; |
|
3029 |
return (0); |
|
3030 |
} |
|
3031 |
||
3032 |
if (fmode & (FNDELAY|FNONBLOCK)) { |
|
3033 |
if (!(flag & NOINTR)) |
|
3034 |
error = EAGAIN; |
|
3035 |
else |
|
3036 |
error = 0; |
|
3037 |
*done = 1; |
|
3038 |
return (error); |
|
3039 |
} |
|
3040 |
||
3041 |
if (stp->sd_flag & errs) { |
|
3042 |
/* |
|
3043 |
* Check for errors before going to sleep since the |
|
3044 |
* caller might not have checked this while holding |
|
3045 |
* sd_lock. |
|
3046 |
*/ |
|
3047 |
error = strgeterr(stp, errs, (flag & STR_PEEK)); |
|
3048 |
if (error != 0) { |
|
3049 |
*done = 1; |
|
3050 |
return (error); |
|
3051 |
} |
|
3052 |
} |
|
3053 |
||
3054 |
/* |
|
3055 |
* If any module downstream has requested read notification |
|
3056 |
* by setting SNDMREAD flag using M_SETOPTS, send a message |
|
3057 |
* down stream. |
|
3058 |
*/ |
|
3059 |
if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) { |
|
3060 |
mutex_exit(&stp->sd_lock); |
|
3061 |
if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED, |
|
3062 |
(flag & STR_NOSIG), &error))) { |
|
3063 |
mutex_enter(&stp->sd_lock); |
|
3064 |
*done = 1; |
|
3065 |
return (error); |
|
3066 |
} |
|
3067 |
mp->b_datap->db_type = M_READ; |
|
3068 |
rd_count = (ssize_t *)mp->b_wptr; |
|
3069 |
*rd_count = count; |
|
3070 |
mp->b_wptr += sizeof (ssize_t); |
|
3071 |
/* |
|
3072 |
* Send the number of bytes requested by the |
|
3073 |
* read as the argument to M_READ. |
|
3074 |
*/ |
|
3075 |
stream_willservice(stp); |
|
3076 |
putnext(stp->sd_wrq, mp); |
|
3077 |
stream_runservice(stp); |
|
3078 |
mutex_enter(&stp->sd_lock); |
|
3079 |
||
3080 |
/* |
|
3081 |
* If any data arrived due to inline processing |
|
3082 |
* of putnext(), don't sleep. |
|
3083 |
*/ |
|
3084 |
if (_RD(stp->sd_wrq)->q_first != NULL) { |
|
3085 |
*done = 0; |
|
3086 |
return (0); |
|
3087 |
} |
|
3088 |
} |
|
3089 |
||
3090 |
stp->sd_flag |= slpflg; |
|
3091 |
TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2, |
|
3092 |
"strwaitq sleeps (2):%p, %X, %lX, %X, %p", |
|
3093 |
stp, flag, count, fmode, done); |
|
3094 |
||
3095 |
rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG); |
|
3096 |
if (rval > 0) { |
|
3097 |
/* EMPTY */ |
|
3098 |
TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2, |
|
3099 |
"strwaitq awakes(2):%X, %X, %X, %X, %X", |
|
3100 |
stp, flag, count, fmode, done); |
|
3101 |
} else if (rval == 0) { |
|
3102 |
TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2, |
|
3103 |
"strwaitq interrupt #2:%p, %X, %lX, %X, %p", |
|
3104 |
stp, flag, count, fmode, done); |
|
3105 |
stp->sd_flag &= ~slpflg; |
|
3106 |
cv_broadcast(sleepon); |
|
3107 |
if (!(flag & NOINTR)) |
|
3108 |
error = EINTR; |
|
3109 |
else |
|
3110 |
error = 0; |
|
3111 |
*done = 1; |
|
3112 |
return (error); |
|
3113 |
} else { |
|
3114 |
/* timeout */ |
|
3115 |
TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME, |
|
3116 |
"strwaitq timeout:%p, %X, %lX, %X, %p", |
|
3117 |
stp, flag, count, fmode, done); |
|
3118 |
*done = 1; |
|
3119 |
if (!(flag & NOINTR)) |
|
3120 |
return (ETIME); |
|
3121 |
else |
|
3122 |
return (0); |
|
3123 |
} |
|
3124 |
/* |
|
3125 |
* If the caller implements delayed errors (i.e. queued after data) |
|
3126 |
* we can not check for errors here since data as well as an |
|
3127 |
* error might have arrived at the stream head. We return to |
|
3128 |
* have the caller check the read queue before checking for errors. |
|
3129 |
*/ |
|
3130 |
if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) { |
|
3131 |
error = strgeterr(stp, errs, (flag & STR_PEEK)); |
|
3132 |
if (error != 0) { |
|
3133 |
*done = 1; |
|
3134 |
return (error); |
|
3135 |
} |
|
3136 |
} |
|
3137 |
*done = 0; |
|
3138 |
return (0); |
|
3139 |
} |
|
3140 |
||
3141 |
/* |
|
3142 |
* Perform job control discipline access checks. |
|
3143 |
* Return 0 for success and the errno for failure. |
|
3144 |
*/ |
|
3145 |
||
3146 |
#define cantsend(p, t, sig) \ |
|
3147 |
(sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig)) |
|
3148 |
||
3149 |
int |
|
3150 |
straccess(struct stdata *stp, enum jcaccess mode) |
|
3151 |
{ |
|
3152 |
extern kcondvar_t lbolt_cv; /* XXX: should be in a header file */ |
|
3153 |
kthread_t *t = curthread; |
|
3154 |
proc_t *p = ttoproc(t); |
|
3155 |
sess_t *sp; |
|
3156 |
||
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3157 |
ASSERT(mutex_owned(&stp->sd_lock)); |
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3158 |
|
0 | 3159 |
if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO) |
3160 |
return (0); |
|
3161 |
||
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3162 |
mutex_enter(&p->p_lock); /* protects p_pgidp */ |
0 | 3163 |
|
3164 |
for (;;) { |
|
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3165 |
mutex_enter(&p->p_splock); /* protects p->p_sessp */ |
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3166 |
sp = p->p_sessp; |
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3167 |
mutex_enter(&sp->s_lock); /* protects sp->* */ |
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3168 |
|
0 | 3169 |
/* |
3170 |
* If this is not the calling process's controlling terminal |
|
3171 |
* or if the calling process is already in the foreground |
|
3172 |
* then allow access. |
|
3173 |
*/ |
|
3174 |
if (sp->s_dev != stp->sd_vnode->v_rdev || |
|
3175 |
p->p_pgidp == stp->sd_pgidp) { |
|
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3176 |
mutex_exit(&sp->s_lock); |
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3177 |
mutex_exit(&p->p_splock); |
0 | 3178 |
mutex_exit(&p->p_lock); |
3179 |
return (0); |
|
3180 |
} |
|
3181 |
||
3182 |
/* |
|
3183 |
* Check to see if controlling terminal has been deallocated. |
|
3184 |
*/ |
|
3185 |
if (sp->s_vp == NULL) { |
|
3186 |
if (!cantsend(p, t, SIGHUP)) |
|
3187 |
sigtoproc(p, t, SIGHUP); |
|
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3188 |
mutex_exit(&sp->s_lock); |
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3189 |
mutex_exit(&p->p_splock); |
0 | 3190 |
mutex_exit(&p->p_lock); |
3191 |
return (EIO); |
|
3192 |
} |
|
3193 |
||
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3194 |
mutex_exit(&sp->s_lock); |
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3195 |
mutex_exit(&p->p_splock); |
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3196 |
|
0 | 3197 |
if (mode == JCGETP) { |
3198 |
mutex_exit(&p->p_lock); |
|
3199 |
return (0); |
|
3200 |
} |
|
3201 |
||
3202 |
if (mode == JCREAD) { |
|
3203 |
if (p->p_detached || cantsend(p, t, SIGTTIN)) { |
|
3204 |
mutex_exit(&p->p_lock); |
|
3205 |
return (EIO); |
|
3206 |
} |
|
3207 |
mutex_exit(&p->p_lock); |
|
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3208 |
mutex_exit(&stp->sd_lock); |
0 | 3209 |
pgsignal(p->p_pgidp, SIGTTIN); |
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3210 |
mutex_enter(&stp->sd_lock); |
0 | 3211 |
mutex_enter(&p->p_lock); |
3212 |
} else { /* mode == JCWRITE or JCSETP */ |
|
3213 |
if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) || |
|
3214 |
cantsend(p, t, SIGTTOU)) { |
|
3215 |
mutex_exit(&p->p_lock); |
|
3216 |
return (0); |
|
3217 |
} |
|
3218 |
if (p->p_detached) { |
|
3219 |
mutex_exit(&p->p_lock); |
|
3220 |
return (EIO); |
|
3221 |
} |
|
3222 |
mutex_exit(&p->p_lock); |
|
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3223 |
mutex_exit(&stp->sd_lock); |
0 | 3224 |
pgsignal(p->p_pgidp, SIGTTOU); |
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3225 |
mutex_enter(&stp->sd_lock); |
0 | 3226 |
mutex_enter(&p->p_lock); |
3227 |
} |
|
3228 |
||
3229 |
/* |
|
3230 |
* We call cv_wait_sig_swap() to cause the appropriate |
|
3231 |
* action for the jobcontrol signal to take place. |
|
3232 |
* If the signal is being caught, we will take the |
|
3233 |
* EINTR error return. Otherwise, the default action |
|
3234 |
* of causing the process to stop will take place. |
|
3235 |
* In this case, we rely on the periodic cv_broadcast() on |
|
3236 |
* &lbolt_cv to wake us up to loop around and test again. |
|
3237 |
* We can't get here if the signal is ignored or |
|
3238 |
* if the current thread is blocking the signal. |
|
3239 |
*/ |
|
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3240 |
mutex_exit(&stp->sd_lock); |
0 | 3241 |
if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) { |
3242 |
mutex_exit(&p->p_lock); |
|
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3243 |
mutex_enter(&stp->sd_lock); |
0 | 3244 |
return (EINTR); |
3245 |
} |
|
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3246 |
mutex_exit(&p->p_lock); |
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3247 |
mutex_enter(&stp->sd_lock); |
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
3248 |
mutex_enter(&p->p_lock); |
0 | 3249 |
} |
3250 |
} |
|
3251 |
||
3252 |
/* |
|
3253 |
* Return size of message of block type (bp->b_datap->db_type) |
|
3254 |
*/ |
|
3255 |
size_t |
|
3256 |
xmsgsize(mblk_t *bp) |
|
3257 |
{ |
|
3258 |
unsigned char type; |
|
3259 |
size_t count = 0; |
|
3260 |
||
3261 |
type = bp->b_datap->db_type; |
|
3262 |
||
3263 |
for (; bp; bp = bp->b_cont) { |
|
3264 |
if (type != bp->b_datap->db_type) |
|
3265 |
break; |
|
3266 |
ASSERT(bp->b_wptr >= bp->b_rptr); |
|
3267 |
count += bp->b_wptr - bp->b_rptr; |
|
3268 |
} |
|
3269 |
return (count); |
|
3270 |
} |
|
3271 |
||
3272 |
/* |
|
3273 |
* Allocate a stream head. |
|
3274 |
*/ |
|
3275 |
struct stdata * |
|
3276 |
shalloc(queue_t *qp) |
|
3277 |
{ |
|
3278 |
stdata_t *stp; |
|
3279 |
||
3280 |
stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP); |
|
3281 |
||
3282 |
stp->sd_wrq = _WR(qp); |
|
3283 |
stp->sd_strtab = NULL; |
|
3284 |
stp->sd_iocid = 0; |
|
3285 |
stp->sd_mate = NULL; |
|
3286 |
stp->sd_freezer = NULL; |
|
3287 |
stp->sd_refcnt = 0; |
|
3288 |
stp->sd_wakeq = 0; |
|
3289 |
stp->sd_anchor = 0; |
|
3290 |
stp->sd_struiowrq = NULL; |
|
3291 |
stp->sd_struiordq = NULL; |
|
3292 |
stp->sd_struiodnak = 0; |
|
3293 |
stp->sd_struionak = NULL; |
|
3294 |
#ifdef C2_AUDIT |
|
3295 |
stp->sd_t_audit_data = NULL; |
|
3296 |
#endif |
|
3297 |
stp->sd_rput_opt = 0; |
|
3298 |
stp->sd_wput_opt = 0; |
|
3299 |
stp->sd_read_opt = 0; |
|
3300 |
stp->sd_rprotofunc = strrput_proto; |
|
3301 |
stp->sd_rmiscfunc = strrput_misc; |
|
3302 |
stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL; |
|
898 | 3303 |
stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL; |
0 | 3304 |
stp->sd_ciputctrl = NULL; |
3305 |
stp->sd_nciputctrl = 0; |
|
3306 |
stp->sd_qhead = NULL; |
|
3307 |
stp->sd_qtail = NULL; |
|
3308 |
stp->sd_servid = NULL; |
|
3309 |
stp->sd_nqueues = 0; |
|
3310 |
stp->sd_svcflags = 0; |
|
3311 |
stp->sd_copyflag = 0; |
|
898 | 3312 |
|
0 | 3313 |
return (stp); |
3314 |
} |
|
3315 |
||
3316 |
/* |
|
3317 |
* Free a stream head. |
|
3318 |
*/ |
|
3319 |
void |
|
3320 |
shfree(stdata_t *stp) |
|
3321 |
{ |
|
3322 |
ASSERT(MUTEX_NOT_HELD(&stp->sd_lock)); |
|
3323 |
||
3324 |
stp->sd_wrq = NULL; |
|
3325 |
||
3326 |
mutex_enter(&stp->sd_qlock); |
|
3327 |
while (stp->sd_svcflags & STRS_SCHEDULED) { |
|
3328 |
STRSTAT(strwaits); |
|
3329 |
cv_wait(&stp->sd_qcv, &stp->sd_qlock); |
|
3330 |
} |
|
3331 |
mutex_exit(&stp->sd_qlock); |
|
3332 |
||
3333 |
if (stp->sd_ciputctrl != NULL) { |
|
3334 |
ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1); |
|
3335 |
SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl, |
|
3336 |
stp->sd_nciputctrl, 0); |
|
3337 |
ASSERT(ciputctrl_cache != NULL); |
|
3338 |
kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl); |
|
3339 |
stp->sd_ciputctrl = NULL; |
|
3340 |
stp->sd_nciputctrl = 0; |
|
3341 |
} |
|
3342 |
ASSERT(stp->sd_qhead == NULL); |
|
3343 |
ASSERT(stp->sd_qtail == NULL); |
|
3344 |
ASSERT(stp->sd_nqueues == 0); |
|
3345 |
kmem_cache_free(stream_head_cache, stp); |
|
3346 |
} |
|
3347 |
||
3348 |
/* |
|
3349 |
* Allocate a pair of queues and a syncq for the pair |
|
3350 |
*/ |
|
3351 |
queue_t * |
|
3352 |
allocq(void) |
|
3353 |
{ |
|
3354 |
queinfo_t *qip; |
|
3355 |
queue_t *qp, *wqp; |
|
3356 |
syncq_t *sq; |
|
3357 |
||
3358 |
qip = kmem_cache_alloc(queue_cache, KM_SLEEP); |
|
3359 |
||
3360 |
qp = &qip->qu_rqueue; |
|
3361 |
wqp = &qip->qu_wqueue; |
|
3362 |
sq = &qip->qu_syncq; |
|
3363 |
||
3364 |
qp->q_last = NULL; |
|
3365 |
qp->q_next = NULL; |
|
3366 |
qp->q_ptr = NULL; |
|
3367 |
qp->q_flag = QUSE | QREADR; |
|
3368 |
qp->q_bandp = NULL; |
|
3369 |
qp->q_stream = NULL; |
|
3370 |
qp->q_syncq = sq; |
|
3371 |
qp->q_nband = 0; |
|
3372 |
qp->q_nfsrv = NULL; |
|
3373 |
qp->q_draining = 0; |
|
3374 |
qp->q_syncqmsgs = 0; |
|
3375 |
qp->q_spri = 0; |
|
3376 |
qp->q_qtstamp = 0; |
|
3377 |
qp->q_sqtstamp = 0; |
|
3378 |
qp->q_fp = NULL; |
|
3379 |
||
3380 |
wqp->q_last = NULL; |
|
3381 |
wqp->q_next = NULL; |
|
3382 |
wqp->q_ptr = NULL; |
|
3383 |
wqp->q_flag = QUSE; |
|
3384 |
wqp->q_bandp = NULL; |
|
3385 |
wqp->q_stream = NULL; |
|
3386 |
wqp->q_syncq = sq; |
|
3387 |
wqp->q_nband = 0; |
|
3388 |
wqp->q_nfsrv = NULL; |
|
3389 |
wqp->q_draining = 0; |
|
3390 |
wqp->q_syncqmsgs = 0; |
|
3391 |
wqp->q_qtstamp = 0; |
|
3392 |
wqp->q_sqtstamp = 0; |
|
3393 |
wqp->q_spri = 0; |
|
3394 |
||
3395 |
sq->sq_count = 0; |
|
3396 |
sq->sq_rmqcount = 0; |
|
3397 |
sq->sq_flags = 0; |
|
3398 |
sq->sq_type = 0; |
|
3399 |
sq->sq_callbflags = 0; |
|
3400 |
sq->sq_cancelid = 0; |
|
3401 |
sq->sq_ciputctrl = NULL; |
|
3402 |
sq->sq_nciputctrl = 0; |
|
3403 |
sq->sq_needexcl = 0; |
|
3404 |
sq->sq_svcflags = 0; |
|
3405 |
||
3406 |
return (qp); |
|
3407 |
} |
|
3408 |
||
3409 |
/* |
|
3410 |
* Free a pair of queues and the "attached" syncq. |
|
3411 |
* Discard any messages left on the syncq(s), remove the syncq(s) from the |
|
3412 |
* outer perimeter, and free the syncq(s) if they are not the "attached" syncq. |
|
3413 |
*/ |
|
3414 |
void |
|
3415 |
freeq(queue_t *qp) |
|
3416 |
{ |
|
3417 |
qband_t *qbp, *nqbp; |
|
3418 |
syncq_t *sq, *outer; |
|
3419 |
queue_t *wqp = _WR(qp); |
|
3420 |
||
3421 |
ASSERT(qp->q_flag & QREADR); |
|
3422 |
||
2497
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3423 |
/* |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3424 |
* If a previously dispatched taskq job is scheduled to run |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3425 |
* sync_service() or a service routine is scheduled for the |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3426 |
* queues about to be freed, wait here until all service is |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3427 |
* done on the queue and all associated queues and syncqs. |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3428 |
*/ |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3429 |
wait_svc(qp); |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3430 |
|
0 | 3431 |
(void) flush_syncq(qp->q_syncq, qp); |
3432 |
(void) flush_syncq(wqp->q_syncq, wqp); |
|
3433 |
ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0); |
|
3434 |
||
2497
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3435 |
/* |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3436 |
* Flush the queues before q_next is set to NULL This is needed |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3437 |
* in order to backenable any downstream queue before we go away. |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3438 |
* Note: we are already removed from the stream so that the |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3439 |
* backenabling will not cause any messages to be delivered to our |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3440 |
* put procedures. |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3441 |
*/ |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3442 |
flushq(qp, FLUSHALL); |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3443 |
flushq(wqp, FLUSHALL); |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3444 |
|
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3445 |
/* Tidy up - removeq only does a half-remove from stream */ |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3446 |
qp->q_next = wqp->q_next = NULL; |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3447 |
ASSERT(!(qp->q_flag & QENAB)); |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3448 |
ASSERT(!(wqp->q_flag & QENAB)); |
a05c2ff105e2
5089813 write side queue/syncq handling is not clean in link_rempassthru()
georges
parents:
898
diff
changeset
|
3449 |
|
0 | 3450 |
outer = qp->q_syncq->sq_outer; |
3451 |
if (outer != NULL) { |
|
3452 |
outer_remove(outer, qp->q_syncq); |
|
3453 |
if (wqp->q_syncq != qp->q_syncq) |
|
3454 |
outer_remove(outer, wqp->q_syncq); |
|
3455 |
} |
|
3456 |
/* |
|
3457 |
* Free any syncqs that are outside what allocq returned. |
|
3458 |
*/ |
|
3459 |
if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD)) |
|
3460 |
free_syncq(qp->q_syncq); |
|
3461 |
if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp)) |
|
3462 |
free_syncq(wqp->q_syncq); |
|
3463 |
||
3464 |
ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); |
|
3465 |
ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); |
|
3466 |
ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); |
|
3467 |
ASSERT(MUTEX_NOT_HELD(QLOCK(wqp))); |
|
3468 |
sq = SQ(qp); |
|
3469 |
ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); |
|
3470 |
ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); |
|
3471 |
ASSERT(sq->sq_outer == NULL); |
|
3472 |
ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); |
|
3473 |
ASSERT(sq->sq_callbpend == NULL); |
|
3474 |
ASSERT(sq->sq_needexcl == 0); |
|
3475 |
||
3476 |
if (sq->sq_ciputctrl != NULL) { |
|
3477 |
ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); |
|
3478 |
SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, |
|
3479 |
sq->sq_nciputctrl, 0); |
|
3480 |
ASSERT(ciputctrl_cache != NULL); |
|
3481 |
kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); |
|
3482 |
sq->sq_ciputctrl = NULL; |
|
3483 |
sq->sq_nciputctrl = 0; |
|
3484 |
} |
|
3485 |
||
3486 |
ASSERT(qp->q_first == NULL && wqp->q_first == NULL); |
|
3487 |
ASSERT(qp->q_count == 0 && wqp->q_count == 0); |
|
3488 |
ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0); |
|
3489 |
||
3490 |
qp->q_flag &= ~QUSE; |
|
3491 |
wqp->q_flag &= ~QUSE; |
|
3492 |
||
3493 |
/* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */ |
|
3494 |
/* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */ |
|
3495 |
||
3496 |
qbp = qp->q_bandp; |
|
3497 |
while (qbp) { |
|
3498 |
nqbp = qbp->qb_next; |
|
3499 |
freeband(qbp); |
|
3500 |
qbp = nqbp; |
|
3501 |
} |
|
3502 |
qbp = wqp->q_bandp; |
|
3503 |
while (qbp) { |
|
3504 |
nqbp = qbp->qb_next; |
|
3505 |
freeband(qbp); |
|
3506 |
qbp = nqbp; |
|
3507 |
} |
|
3508 |
kmem_cache_free(queue_cache, qp); |
|
3509 |
} |
|
3510 |
||
3511 |
/* |
|
3512 |
* Allocate a qband structure. |
|
3513 |
*/ |
|
3514 |
qband_t * |
|
3515 |
allocband(void) |
|
3516 |
{ |
|
3517 |
qband_t *qbp; |
|
3518 |
||
3519 |
qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP); |
|
3520 |
if (qbp == NULL) |
|
3521 |
return (NULL); |
|
3522 |
||
3523 |
qbp->qb_next = NULL; |
|
3524 |
qbp->qb_count = 0; |
|
3525 |
qbp->qb_mblkcnt = 0; |
|
3526 |
qbp->qb_first = NULL; |
|
3527 |
qbp->qb_last = NULL; |
|
3528 |
qbp->qb_flag = 0; |
|
3529 |
||
3530 |
return (qbp); |
|
3531 |
} |
|
3532 |
||
3533 |
/* |
|
3534 |
* Free a qband structure. |
|
3535 |
*/ |
|
3536 |
void |
|
3537 |
freeband(qband_t *qbp) |
|
3538 |
{ |
|
3539 |
kmem_cache_free(qband_cache, qbp); |
|
3540 |
} |
|
3541 |
||
3542 |
/* |
|
3543 |
* Just like putnextctl(9F), except that allocb_wait() is used. |
|
3544 |
* |
|
3545 |
* Consolidation Private, and of course only callable from the stream head or |
|
3546 |
* routines that may block. |
|
3547 |
*/ |
|
3548 |
int |
|
3549 |
putnextctl_wait(queue_t *q, int type) |
|
3550 |
{ |
|
3551 |
mblk_t *bp; |
|
3552 |
int error; |
|
3553 |
||
3554 |
if ((datamsg(type) && (type != M_DELAY)) || |
|
3555 |
(bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL) |
|
3556 |
return (0); |
|
3557 |
||
3558 |
bp->b_datap->db_type = (unsigned char)type; |
|
3559 |
putnext(q, bp); |
|
3560 |
return (1); |
|
3561 |
} |
|
3562 |
||
3563 |
/* |
|
3564 |
* run any possible bufcalls. |
|
3565 |
*/ |
|
3566 |
void |
|
3567 |
runbufcalls(void) |
|
3568 |
{ |
|
3569 |
strbufcall_t *bcp; |
|
3570 |
||
3571 |
mutex_enter(&bcall_monitor); |
|
3572 |
mutex_enter(&strbcall_lock); |
|
3573 |
||
3574 |
if (strbcalls.bc_head) { |
|
3575 |
size_t count; |
|
3576 |
int nevent; |
|
3577 |
||
3578 |
/* |
|
3579 |
* count how many events are on the list |
|
3580 |
* now so we can check to avoid looping |
|
3581 |
* in low memory situations |
|
3582 |
*/ |
|
3583 |
nevent = 0; |
|
3584 |
for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next) |
|
3585 |
nevent++; |
|
3586 |
||
3587 |
/* |
|
3588 |
* get estimate of available memory from kmem_avail(). |
|
3589 |
* awake all bufcall functions waiting for |
|
3590 |
* memory whose request could be satisfied |
|
3591 |
* by 'count' memory and let 'em fight for it. |
|
3592 |
*/ |
|
3593 |
count = kmem_avail(); |
|
3594 |
while ((bcp = strbcalls.bc_head) != NULL && nevent) { |
|
3595 |
STRSTAT(bufcalls); |
|
3596 |
--nevent; |
|
3597 |
if (bcp->bc_size <= count) { |
|
3598 |
bcp->bc_executor = curthread; |
|
3599 |
mutex_exit(&strbcall_lock); |
|
3600 |
(*bcp->bc_func)(bcp->bc_arg); |
|
3601 |
mutex_enter(&strbcall_lock); |
|
3602 |
bcp->bc_executor = NULL; |
|
3603 |
cv_broadcast(&bcall_cv); |
|
3604 |
strbcalls.bc_head = bcp->bc_next; |
|
3605 |
kmem_free(bcp, sizeof (strbufcall_t)); |
|
3606 |
} else { |
|
3607 |
/* |
|
3608 |
* too big, try again later - note |
|
3609 |
* that nevent was decremented above |
|
3610 |
* so we won't retry this one on this |
|
3611 |
* iteration of the loop |
|
3612 |
*/ |
|
3613 |
if (bcp->bc_next != NULL) { |
|
3614 |
strbcalls.bc_head = bcp->bc_next; |
|
3615 |
bcp->bc_next = NULL; |
|
3616 |
strbcalls.bc_tail->bc_next = bcp; |
|
3617 |
strbcalls.bc_tail = bcp; |
|
3618 |
} |
|
3619 |
} |
|
3620 |
} |
|
3621 |
if (strbcalls.bc_head == NULL) |
|
3622 |
strbcalls.bc_tail = NULL; |
|
3623 |
} |
|
3624 |
||
3625 |
mutex_exit(&strbcall_lock); |
|
3626 |
mutex_exit(&bcall_monitor); |
|
3627 |
} |
|
3628 |
||
3629 |
||
3630 |
/* |
|
3631 |
* actually run queue's service routine. |
|
3632 |
*/ |
|
3633 |
static void |
|
3634 |
runservice(queue_t *q) |
|
3635 |
{ |
|
3636 |
qband_t *qbp; |
|
3637 |
||
3638 |
ASSERT(q->q_qinfo->qi_srvp); |
|
3639 |
again: |
|
3640 |
entersq(q->q_syncq, SQ_SVC); |
|
3641 |
TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START, |
|
3642 |
"runservice starts:%p", q); |
|
3643 |
||
3644 |
if (!(q->q_flag & QWCLOSE)) |
|
3645 |
(*q->q_qinfo->qi_srvp)(q); |
|
3646 |
||
3647 |
TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END, |
|
3648 |
"runservice ends:(%p)", q); |
|
3649 |
||
3650 |
leavesq(q->q_syncq, SQ_SVC); |
|
3651 |
||
3652 |
mutex_enter(QLOCK(q)); |
|
3653 |
if (q->q_flag & QENAB) { |
|
3654 |
q->q_flag &= ~QENAB; |
|
3655 |
mutex_exit(QLOCK(q)); |
|
3656 |
goto again; |
|
3657 |
} |
|
3658 |
q->q_flag &= ~QINSERVICE; |
|
3659 |
q->q_flag &= ~QBACK; |
|
3660 |
for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next) |
|
3661 |
qbp->qb_flag &= ~QB_BACK; |
|
3662 |
/* |
|
3663 |
* Wakeup thread waiting for the service procedure |
|
3664 |
* to be run (strclose and qdetach). |
|
3665 |
*/ |
|
3666 |
cv_broadcast(&q->q_wait); |
|
3667 |
||
3668 |
mutex_exit(QLOCK(q)); |
|
3669 |
} |
|
3670 |
||
3671 |
/* |
|
3672 |
* Background processing of bufcalls. |
|
3673 |
*/ |
|
3674 |
void |
|
3675 |
streams_bufcall_service(void) |
|
3676 |
{ |
|
3677 |
callb_cpr_t cprinfo; |
|
3678 |
||
3679 |
CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr, |
|
3680 |
"streams_bufcall_service"); |
|
3681 |
||
3682 |
mutex_enter(&strbcall_lock); |
|
3683 |
||
3684 |
for (;;) { |
|
3685 |
if (strbcalls.bc_head != NULL && kmem_avail() > 0) { |
|
3686 |
mutex_exit(&strbcall_lock); |
|
3687 |
runbufcalls(); |
|
3688 |
mutex_enter(&strbcall_lock); |
|
3689 |
} |
|
3690 |
if (strbcalls.bc_head != NULL) { |
|
3691 |
clock_t wt, tick; |
|
3692 |
||
3693 |
STRSTAT(bcwaits); |
|
3694 |
/* Wait for memory to become available */ |
|
3695 |
CALLB_CPR_SAFE_BEGIN(&cprinfo); |
|
3696 |
tick = SEC_TO_TICK(60); |
|
3697 |
time_to_wait(&wt, tick); |
|
3698 |
(void) cv_timedwait(&memavail_cv, &strbcall_lock, wt); |
|
3699 |
CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); |
|
3700 |
} |
|
3701 |
||
3702 |
/* Wait for new work to arrive */ |
|
3703 |
if (strbcalls.bc_head == NULL) { |
|
3704 |
CALLB_CPR_SAFE_BEGIN(&cprinfo); |
|
3705 |
cv_wait(&strbcall_cv, &strbcall_lock); |
|
3706 |
CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); |
|
3707 |
} |
|
3708 |
} |
|
3709 |
} |
|
3710 |
||
3711 |
/* |
|
3712 |
* Background processing of streams background tasks which failed |
|
3713 |
* taskq_dispatch. |
|
3714 |
*/ |
|
3715 |
static void |
|
3716 |
streams_qbkgrnd_service(void) |
|
3717 |
{ |
|
3718 |
callb_cpr_t cprinfo; |
|
3719 |
queue_t *q; |
|
3720 |
||
3721 |
CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, |
|
3722 |
"streams_bkgrnd_service"); |
|
3723 |
||
3724 |
mutex_enter(&service_queue); |
|
3725 |
||
3726 |
for (;;) { |
|
3727 |
/* |
|
3728 |
* Wait for work to arrive. |
|
3729 |
*/ |
|
3730 |
while ((freebs_list == NULL) && (qhead == NULL)) { |
|
3731 |
CALLB_CPR_SAFE_BEGIN(&cprinfo); |
|
3732 |
cv_wait(&services_to_run, &service_queue); |
|
3733 |
CALLB_CPR_SAFE_END(&cprinfo, &service_queue); |
|
3734 |
} |
|
3735 |
/* |
|
3736 |
* Handle all pending freebs requests to free memory. |
|
3737 |
*/ |
|
3738 |
while (freebs_list != NULL) { |
|
3739 |
mblk_t *mp = freebs_list; |
|
3740 |
freebs_list = mp->b_next; |
|
3741 |
mutex_exit(&service_queue); |
|
3742 |
mblk_free(mp); |
|
3743 |
mutex_enter(&service_queue); |
|
3744 |
} |
|
3745 |
/* |
|
3746 |
* Run pending queues. |
|
3747 |
*/ |
|
3748 |
while (qhead != NULL) { |
|
3749 |
DQ(q, qhead, qtail, q_link); |
|
3750 |
ASSERT(q != NULL); |
|
3751 |
mutex_exit(&service_queue); |
|
3752 |
queue_service(q); |
|
3753 |
mutex_enter(&service_queue); |
|
3754 |
} |
|
3755 |
ASSERT(qhead == NULL && qtail == NULL); |
|
3756 |
} |
|
3757 |
} |
|
3758 |
||
3759 |
/* |
|
3760 |
* Background processing of streams background tasks which failed |
|
3761 |
* taskq_dispatch. |
|
3762 |
*/ |
|
3763 |
static void |
|
3764 |
streams_sqbkgrnd_service(void) |
|
3765 |
{ |
|
3766 |
callb_cpr_t cprinfo; |
|
3767 |
syncq_t *sq; |
|
3768 |
||
3769 |
CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, |
|
3770 |
"streams_sqbkgrnd_service"); |
|
3771 |
||
3772 |
mutex_enter(&service_queue); |
|
3773 |
||
3774 |
for (;;) { |
|
3775 |
/* |
|
3776 |
* Wait for work to arrive. |
|
3777 |
*/ |
|
3778 |
while (sqhead == NULL) { |
|
3779 |
CALLB_CPR_SAFE_BEGIN(&cprinfo); |
|
3780 |
cv_wait(&syncqs_to_run, &service_queue); |
|
3781 |
CALLB_CPR_SAFE_END(&cprinfo, &service_queue); |
|
3782 |
} |
|
3783 |
||
3784 |
/* |
|
3785 |
* Run pending syncqs. |
|
3786 |
*/ |
|
3787 |
while (sqhead != NULL) { |
|
3788 |
DQ(sq, sqhead, sqtail, sq_next); |
|
3789 |
ASSERT(sq != NULL); |
|
3790 |
ASSERT(sq->sq_svcflags & SQ_BGTHREAD); |
|
3791 |
mutex_exit(&service_queue); |
|
3792 |
syncq_service(sq); |
|
3793 |
mutex_enter(&service_queue); |
|
3794 |
} |
|
3795 |
} |
|
3796 |
} |
|
3797 |
||
3798 |
/* |
|
3799 |
* Disable the syncq and wait for background syncq processing to complete. |
|
3800 |
* If the syncq is placed on the sqhead/sqtail queue, try to remove it from the |
|
3801 |
* list. |
|
3802 |
*/ |
|
3803 |
void |
|
3804 |
wait_sq_svc(syncq_t *sq) |
|
3805 |
{ |
|
3806 |
mutex_enter(SQLOCK(sq)); |
|
3807 |
sq->sq_svcflags |= SQ_DISABLED; |
|
3808 |
if (sq->sq_svcflags & SQ_BGTHREAD) { |
|
3809 |
syncq_t *sq_chase; |
|
3810 |
syncq_t *sq_curr; |
|
3811 |
int removed; |
|
3812 |
||
3813 |
ASSERT(sq->sq_servcount == 1); |
|
3814 |
mutex_enter(&service_queue); |
|
3815 |
RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed); |
|
3816 |
mutex_exit(&service_queue); |
|
3817 |
if (removed) { |
|
3818 |
sq->sq_svcflags &= ~SQ_BGTHREAD; |
|
3819 |
sq->sq_servcount = 0; |
|
3820 |
STRSTAT(sqremoved); |
|
3821 |
goto done; |
|
3822 |
} |
|
3823 |
} |
|
3824 |
while (sq->sq_servcount != 0) { |
|
3825 |
sq->sq_flags |= SQ_WANTWAKEUP; |
|
3826 |
cv_wait(&sq->sq_wait, SQLOCK(sq)); |
|
3827 |
} |
|
3828 |
done: |
|
3829 |
mutex_exit(SQLOCK(sq)); |
|
3830 |
} |
|
3831 |
||
3832 |
/* |
|
3833 |
* Put a syncq on the list of syncq's to be serviced by the sqthread. |
|
3834 |
* Add the argument to the end of the sqhead list and set the flag |
|
3835 |
* indicating this syncq has been enabled. If it has already been |
|
3836 |
* enabled, don't do anything. |
|
3837 |
* This routine assumes that SQLOCK is held. |
|
3838 |
* NOTE that the lock order is to have the SQLOCK first, |
|
3839 |
* so if the service_syncq lock is held, we need to release it |
|
3840 |
* before aquiring the SQLOCK (mostly relevant for the background |
|
3841 |
* thread, and this seems to be common among the STREAMS global locks). |
|
3842 |
* Note the the sq_svcflags are protected by the SQLOCK. |
|
3843 |
*/ |
|
3844 |
void |
|
3845 |
sqenable(syncq_t *sq) |
|
3846 |
{ |
|
3847 |
/* |
|
3848 |
* This is probably not important except for where I believe it |
|
3849 |
* is being called. At that point, it should be held (and it |
|
3850 |
* is a pain to release it just for this routine, so don't do |
|
3851 |
* it). |
|
3852 |
*/ |
|
3853 |
ASSERT(MUTEX_HELD(SQLOCK(sq))); |
|
3854 |
||
3855 |
IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL); |
|
3856 |
IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD); |
|
3857 |
||
3858 |
/* |
|
3859 |
* Do not put on list if background thread is scheduled or |
|
3860 |
* syncq is disabled. |
|
3861 |
*/ |
|
3862 |
if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD)) |
|
3863 |
return; |
|
3864 |
||
3865 |
/* |
|
3866 |
* Check whether we should enable sq at all. |
|
3867 |
* Non PERMOD syncqs may be drained by at most one thread. |
|
3868 |
* PERMOD syncqs may be drained by several threads but we limit the |
|
3869 |
* total amount to the lesser of |
|
3870 |
* Number of queues on the squeue and |
|
3871 |
* Number of CPUs. |
|
3872 |
*/ |
|
3873 |
if (sq->sq_servcount != 0) { |
|
3874 |
if (((sq->sq_type & SQ_PERMOD) == 0) || |
|
3875 |
(sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) { |
|
3876 |
STRSTAT(sqtoomany); |
|
3877 |
return; |
|
3878 |
} |
|
3879 |
} |
|
3880 |
||
3881 |
sq->sq_tstamp = lbolt; |
|
3882 |
STRSTAT(sqenables); |
|
3883 |
||
3884 |
/* Attempt a taskq dispatch */ |
|
3885 |
sq->sq_servid = (void *)taskq_dispatch(streams_taskq, |
|
3886 |
(task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE); |
|
3887 |
if (sq->sq_servid != NULL) { |
|
3888 |
sq->sq_servcount++; |
|
3889 |
return; |
|
3890 |
} |
|
3891 |
||
3892 |
/* |
|
3893 |
* This taskq dispatch failed, but a previous one may have succeeded. |
|
3894 |
* Don't try to schedule on the background thread whilst there is |
|
3895 |
* outstanding taskq processing. |
|
3896 |
*/ |
|
3897 |
if (sq->sq_servcount != 0) |
|
3898 |
return; |
|
3899 |
||
3900 |
/* |
|
3901 |
* System is low on resources and can't perform a non-sleeping |
|
3902 |
* dispatch. Schedule the syncq for a background thread and mark the |
|
3903 |
* syncq to avoid any further taskq dispatch attempts. |
|
3904 |
*/ |
|
3905 |
mutex_enter(&service_queue); |
|
3906 |
STRSTAT(taskqfails); |
|
3907 |
ENQUEUE(sq, sqhead, sqtail, sq_next); |
|
3908 |
sq->sq_svcflags |= SQ_BGTHREAD; |
|
3909 |
sq->sq_servcount = 1; |
|
3910 |
cv_signal(&syncqs_to_run); |
|
3911 |
mutex_exit(&service_queue); |
|
3912 |
} |
|
3913 |
||
3914 |
/* |
|
3915 |
* Note: fifo_close() depends on the mblk_t on the queue being freed |
|
3916 |
* asynchronously. The asynchronous freeing of messages breaks the |
|
3917 |
* recursive call chain of fifo_close() while there are I_SENDFD type of |
|
3918 |
* messages refering other file pointers on the queue. Then when |
|
3919 |
* closing pipes it can avoid stack overflow in case of daisy-chained |
|
3920 |
* pipes, and also avoid deadlock in case of fifonode_t pairs (which |
|
3921 |
* share the same fifolock_t). |
|
3922 |
*/ |
|
3923 |
||
3924 |
void |
|
3925 |
freebs_enqueue(mblk_t *mp, dblk_t *dbp) |
|
3926 |
{ |
|
3932
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3927 |
esb_queue_t *eqp = &system_esbq; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3928 |
|
0 | 3929 |
ASSERT(dbp->db_mblk == mp); |
3930 |
||
3931 |
/* |
|
3932 |
* Check data sanity. The dblock should have non-empty free function. |
|
3933 |
* It is better to panic here then later when the dblock is freed |
|
3934 |
* asynchronously when the context is lost. |
|
3935 |
*/ |
|
3936 |
if (dbp->db_frtnp->free_func == NULL) { |
|
3937 |
panic("freebs_enqueue: dblock %p has a NULL free callback", |
|
3932
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3938 |
(void *)dbp); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3939 |
} |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3940 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3941 |
mutex_enter(&eqp->eq_lock); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3942 |
/* queue the new mblk on the esballoc queue */ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3943 |
if (eqp->eq_head == NULL) { |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3944 |
eqp->eq_head = eqp->eq_tail = mp; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3945 |
} else { |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3946 |
eqp->eq_tail->b_next = mp; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3947 |
eqp->eq_tail = mp; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3948 |
} |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3949 |
eqp->eq_len++; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3950 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3951 |
/* If we're the first thread to reach the threshold, process */ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3952 |
if (eqp->eq_len >= esbq_max_qlen && |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3953 |
!(eqp->eq_flags & ESBQ_PROCESSING)) |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3954 |
esballoc_process_queue(eqp); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3955 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3956 |
esballoc_set_timer(eqp, esbq_timeout); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3957 |
mutex_exit(&eqp->eq_lock); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3958 |
} |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3959 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3960 |
static void |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3961 |
esballoc_process_queue(esb_queue_t *eqp) |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3962 |
{ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3963 |
mblk_t *mp; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3964 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3965 |
ASSERT(MUTEX_HELD(&eqp->eq_lock)); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3966 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3967 |
eqp->eq_flags |= ESBQ_PROCESSING; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3968 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3969 |
do { |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3970 |
/* |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3971 |
* Detach the message chain for processing. |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3972 |
*/ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3973 |
mp = eqp->eq_head; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3974 |
eqp->eq_tail->b_next = NULL; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3975 |
eqp->eq_head = eqp->eq_tail = NULL; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3976 |
eqp->eq_len = 0; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3977 |
mutex_exit(&eqp->eq_lock); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3978 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3979 |
/* |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3980 |
* Process the message chain. |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3981 |
*/ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3982 |
esballoc_enqueue_mblk(mp); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3983 |
mutex_enter(&eqp->eq_lock); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3984 |
} while ((eqp->eq_len >= esbq_max_qlen) && (eqp->eq_len > 0)); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3985 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3986 |
eqp->eq_flags &= ~ESBQ_PROCESSING; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3987 |
} |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3988 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3989 |
/* |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3990 |
* taskq callback routine to free esballoced mblk's |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3991 |
*/ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3992 |
static void |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3993 |
esballoc_mblk_free(mblk_t *mp) |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3994 |
{ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3995 |
mblk_t *nextmp; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3996 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3997 |
for (; mp != NULL; mp = nextmp) { |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3998 |
nextmp = mp->b_next; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
3999 |
mp->b_next = NULL; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4000 |
mblk_free(mp); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4001 |
} |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4002 |
} |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4003 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4004 |
static void |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4005 |
esballoc_enqueue_mblk(mblk_t *mp) |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4006 |
{ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4007 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4008 |
if (taskq_dispatch(system_taskq, (task_func_t *)esballoc_mblk_free, mp, |
0 | 4009 |
TQ_NOSLEEP) == NULL) { |
3932
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4010 |
mblk_t *first_mp = mp; |
0 | 4011 |
/* |
4012 |
* System is low on resources and can't perform a non-sleeping |
|
4013 |
* dispatch. Schedule for a background thread. |
|
4014 |
*/ |
|
4015 |
mutex_enter(&service_queue); |
|
4016 |
STRSTAT(taskqfails); |
|
3932
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4017 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4018 |
while (mp->b_next != NULL) |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4019 |
mp = mp->b_next; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4020 |
|
0 | 4021 |
mp->b_next = freebs_list; |
3932
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4022 |
freebs_list = first_mp; |
0 | 4023 |
cv_signal(&services_to_run); |
4024 |
mutex_exit(&service_queue); |
|
4025 |
} |
|
4026 |
} |
|
4027 |
||
3932
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4028 |
static void |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4029 |
esballoc_timer(void *arg) |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4030 |
{ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4031 |
esb_queue_t *eqp = arg; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4032 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4033 |
mutex_enter(&eqp->eq_lock); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4034 |
eqp->eq_flags &= ~ESBQ_TIMER; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4035 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4036 |
if (!(eqp->eq_flags & ESBQ_PROCESSING) && |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4037 |
eqp->eq_len > 0) |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4038 |
esballoc_process_queue(eqp); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4039 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4040 |
esballoc_set_timer(eqp, esbq_timeout); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4041 |
mutex_exit(&eqp->eq_lock); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4042 |
} |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4043 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4044 |
static void |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4045 |
esballoc_set_timer(esb_queue_t *eqp, clock_t eq_timeout) |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4046 |
{ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4047 |
ASSERT(MUTEX_HELD(&eqp->eq_lock)); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4048 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4049 |
if (eqp->eq_len > 0 && !(eqp->eq_flags & ESBQ_TIMER)) { |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4050 |
(void) timeout(esballoc_timer, eqp, eq_timeout); |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4051 |
eqp->eq_flags |= ESBQ_TIMER; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4052 |
} |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4053 |
} |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4054 |
|
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4055 |
void |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4056 |
esballoc_queue_init(void) |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4057 |
{ |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4058 |
system_esbq.eq_len = 0; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4059 |
system_esbq.eq_head = system_esbq.eq_tail = NULL; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4060 |
system_esbq.eq_flags = 0; |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4061 |
} |
efce29b04ab4
6522339 Performance enhancement with enhanced esballoc design
ss146032
parents:
3448
diff
changeset
|
4062 |
|
0 | 4063 |
/* |
4064 |
* Set the QBACK or QB_BACK flag in the given queue for |
|
4065 |
* the given priority band. |
|
4066 |
*/ |
|
4067 |
void |
|
4068 |
setqback(queue_t *q, unsigned char pri) |
|
4069 |
{ |
|
4070 |
int i; |
|
4071 |
qband_t *qbp; |
|
4072 |
qband_t **qbpp; |
|
4073 |
||
4074 |
ASSERT(MUTEX_HELD(QLOCK(q))); |
|
4075 |
if (pri != 0) { |
|
4076 |
if (pri > q->q_nband) { |
|
4077 |
qbpp = &q->q_bandp; |
|
4078 |
while (*qbpp) |
|
4079 |
qbpp = &(*qbpp)->qb_next; |
|
4080 |
while (pri > q->q_nband) { |
|
4081 |
if ((*qbpp = allocband()) == NULL) { |
|
4082 |
cmn_err(CE_WARN, |
|
4083 |
"setqback: can't allocate qband\n"); |
|
4084 |
return; |
|
4085 |
} |
|
4086 |
(*qbpp)->qb_hiwat = q->q_hiwat; |
|
4087 |
(*qbpp)->qb_lowat = q->q_lowat; |
|
4088 |
q->q_nband++; |
|
4089 |
qbpp = &(*qbpp)->qb_next; |
|
4090 |
} |
|
4091 |
} |
|
4092 |
qbp = q->q_bandp; |
|
4093 |
i = pri; |
|
4094 |
while (--i) |
|
4095 |
qbp = qbp->qb_next; |
|
4096 |
qbp->qb_flag |= QB_BACK; |
|
4097 |
} else { |
|
4098 |
q->q_flag |= QBACK; |
|
4099 |
} |
|
4100 |
} |
|
4101 |
||
4102 |
int |
|
4103 |
strcopyin(void *from, void *to, size_t len, int copyflag) |
|
4104 |
{ |
|
4105 |
if (copyflag & U_TO_K) { |
|
4106 |
ASSERT((copyflag & K_TO_K) == 0); |
|
4107 |
if (copyin(from, to, len)) |
|
4108 |
return (EFAULT); |
|
4109 |
} else { |
|
4110 |
ASSERT(copyflag & K_TO_K); |
|
4111 |
bcopy(from, to, len); |
|
4112 |
} |
|
4113 |
return (0); |
|
4114 |
} |
|
4115 |
||
4116 |
int |
|
4117 |
strcopyout(void *from, void *to, size_t len, int copyflag) |
|
4118 |
{ |
|
4119 |
if (copyflag & U_TO_K) { |
|
4120 |
if (copyout(from, to, len)) |
|
4121 |
return (EFAULT); |
|
4122 |
} else { |
|
4123 |
ASSERT(copyflag & K_TO_K); |
|
4124 |
bcopy(from, to, len); |
|
4125 |
} |
|
4126 |
return (0); |
|
4127 |
} |
|
4128 |
||
4129 |
/* |
|
4130 |
* strsignal_nolock() posts a signal to the process(es) at the stream head. |
|
4131 |
* It assumes that the stream head lock is already held, whereas strsignal() |
|
4132 |
* acquires the lock first. This routine was created because a few callers |
|
4133 |
* release the stream head lock before calling only to re-acquire it after |
|
4134 |
* it returns. |
|
4135 |
*/ |
|
4136 |
void |
|
4137 |
strsignal_nolock(stdata_t *stp, int sig, int32_t band) |
|
4138 |
{ |
|
4139 |
ASSERT(MUTEX_HELD(&stp->sd_lock)); |
|
4140 |
switch (sig) { |
|
4141 |
case SIGPOLL: |
|
4142 |
if (stp->sd_sigflags & S_MSG) |
|
4143 |
strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); |
|
4144 |
break; |
|
4145 |
||
4146 |
default: |
|
4147 |
if (stp->sd_pgidp) { |
|
4148 |
pgsignal(stp->sd_pgidp, sig); |
|
4149 |
} |
|
4150 |
break; |
|
4151 |
} |
|
4152 |
} |
|
4153 |
||
4154 |
void |
|
4155 |
strsignal(stdata_t *stp, int sig, int32_t band) |
|
4156 |
{ |
|
4157 |
TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG, |
|
4158 |
"strsignal:%p, %X, %X", stp, sig, band); |
|
4159 |
||
4160 |
mutex_enter(&stp->sd_lock); |
|
4161 |
switch (sig) { |
|
4162 |
case SIGPOLL: |
|
4163 |
if (stp->sd_sigflags & S_MSG) |
|
4164 |
strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); |
|
4165 |
break; |
|
4166 |
||
4167 |
default: |
|
4168 |
if (stp->sd_pgidp) { |
|
4169 |
pgsignal(stp->sd_pgidp, sig); |
|
4170 |
} |
|
4171 |
break; |
|
4172 |
} |
|
4173 |
mutex_exit(&stp->sd_lock); |
|
4174 |
} |
|
4175 |
||
4176 |
void |
|
4177 |
strhup(stdata_t *stp) |
|
4178 |
{ |
|
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
4179 |
ASSERT(mutex_owned(&stp->sd_lock)); |
0 | 4180 |
pollwakeup(&stp->sd_pollist, POLLHUP); |
4181 |
if (stp->sd_sigflags & S_HANGUP) |
|
4182 |
strsendsig(stp->sd_siglist, S_HANGUP, 0, 0); |
|
2712
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
4183 |
} |
f74a135872bc
PSARC/2005/471 BrandZ: Support for non-native zones
nn35248
parents:
2497
diff
changeset
|
4184 |
|
560
fab7db0411ac
6322179 strfreectty() passed a stream that was not yet a ctty
meem
parents:
235
diff
changeset
|
4185 |
/* |
fab7db0411ac
6322179 strfreectty() passed a stream that was not yet a ctty
meem
parents:
235
diff
changeset
|
4186 |
* Backenable the first queue upstream from `q' with a service procedure. |
0 | 4187 |
*/ |
4188 |
void |
|
235
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4189 |
backenable(queue_t *q, uchar_t pri) |
0 | 4190 |
{ |
4191 |
queue_t *nq; |
|
4192 |
||
4193 |
/* |
|
4194 |
* our presence might not prevent other modules in our own |
|
4195 |
* stream from popping/pushing since the caller of getq might not |
|
4196 |
* have a claim on the queue (some drivers do a getq on somebody |
|
4197 |
* else's queue - they know that the queue itself is not going away |
|
4198 |
* but the framework has to guarantee q_next in that stream.) |
|
4199 |
*/ |
|
4200 |
claimstr(q); |
|
4201 |
||
4202 |
/* find nearest back queue with service proc */ |
|
4203 |
for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) { |
|
4204 |
ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq)); |
|
4205 |
} |
|
4206 |
||
4207 |
if (nq) { |
|
4208 |
kthread_t *freezer; |
|
4209 |
/* |
|
4210 |
* backenable can be called either with no locks held |
|
4211 |
* or with the stream frozen (the latter occurs when a module |
|
4212 |
* calls rmvq with the stream frozen.) If the stream is frozen |
|
4213 |
* by the caller the caller will hold all qlocks in the stream. |
|
3448 | 4214 |
* Note that a frozen stream doesn't freeze a mated stream, |
4215 |
* so we explicitly check for that. |
|
0 | 4216 |
*/ |
4217 |
freezer = STREAM(q)->sd_freezer; |
|
3448 | 4218 |
if (freezer != curthread || STREAM(q) != STREAM(nq)) { |
0 | 4219 |
mutex_enter(QLOCK(nq)); |
4220 |
} |
|
4221 |
#ifdef DEBUG |
|
4222 |
else { |
|
4223 |
ASSERT(frozenstr(q)); |
|
4224 |
ASSERT(MUTEX_HELD(QLOCK(q))); |
|
4225 |
ASSERT(MUTEX_HELD(QLOCK(nq))); |
|
4226 |
} |
|
4227 |
#endif |
|
235
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4228 |
setqback(nq, pri); |
0 | 4229 |
qenable_locked(nq); |
3448 | 4230 |
if (freezer != curthread || STREAM(q) != STREAM(nq)) |
0 | 4231 |
mutex_exit(QLOCK(nq)); |
4232 |
} |
|
4233 |
releasestr(q); |
|
4234 |
} |
|
4235 |
||
4236 |
/* |
|
4237 |
* Return the appropriate errno when one of flags_to_check is set |
|
4238 |
* in sd_flags. Uses the exported error routines if they are set. |
|
4239 |
* Will return 0 if non error is set (or if the exported error routines |
|
4240 |
* do not return an error). |
|
4241 |
* |
|
4242 |
* If there is both a read and write error to check we prefer the read error. |
|
4243 |
* Also, give preference to recorded errno's over the error functions. |
|
4244 |
* The flags that are handled are: |
|
4245 |
* STPLEX return EINVAL |
|
4246 |
* STRDERR return sd_rerror (and clear if STRDERRNONPERSIST) |
|
4247 |
* STWRERR return sd_werror (and clear if STWRERRNONPERSIST) |
|
4248 |
* STRHUP return sd_werror |
|
4249 |
* |
|
4250 |
* If the caller indicates that the operation is a peek a nonpersistent error |
|
4251 |
* is not cleared. |
|
4252 |
*/ |
|
4253 |
int |
|
4254 |
strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek) |
|
4255 |
{ |
|
4256 |
int32_t sd_flag = stp->sd_flag & flags_to_check; |
|
4257 |
int error = 0; |
|
4258 |
||
4259 |
ASSERT(MUTEX_HELD(&stp->sd_lock)); |
|
4260 |
ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0); |
|
4261 |
if (sd_flag & STPLEX) |
|
4262 |
error = EINVAL; |
|
4263 |
else if (sd_flag & STRDERR) { |
|
4264 |
error = stp->sd_rerror; |
|
4265 |
if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) { |
|
4266 |
/* |
|
4267 |
* Read errors are non-persistent i.e. discarded once |
|
4268 |
* returned to a non-peeking caller, |
|
4269 |
*/ |
|
4270 |
stp->sd_rerror = 0; |
|
4271 |
stp->sd_flag &= ~STRDERR; |
|
4272 |
} |
|
4273 |
if (error == 0 && stp->sd_rderrfunc != NULL) { |
|
4274 |
int clearerr = 0; |
|
4275 |
||
4276 |
error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek, |
|
4277 |
&clearerr); |
|
4278 |
if (clearerr) { |
|
4279 |
stp->sd_flag &= ~STRDERR; |
|
4280 |
stp->sd_rderrfunc = NULL; |
|
4281 |
} |
|
4282 |
} |
|
4283 |
} else if (sd_flag & STWRERR) { |
|
4284 |
error = stp->sd_werror; |
|
4285 |
if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) { |
|
4286 |
/* |
|
4287 |
* Write errors are non-persistent i.e. discarded once |
|
4288 |
* returned to a non-peeking caller, |
|
4289 |
*/ |
|
4290 |
stp->sd_werror = 0; |
|
4291 |
stp->sd_flag &= ~STWRERR; |
|
4292 |
} |
|
4293 |
if (error == 0 && stp->sd_wrerrfunc != NULL) { |
|
4294 |
int clearerr = 0; |
|
4295 |
||
4296 |
error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek, |
|
4297 |
&clearerr); |
|
4298 |
if (clearerr) { |
|
4299 |
stp->sd_flag &= ~STWRERR; |
|
4300 |
stp->sd_wrerrfunc = NULL; |
|
4301 |
} |
|
4302 |
} |
|
4303 |
} else if (sd_flag & STRHUP) { |
|
4304 |
/* sd_werror set when STRHUP */ |
|
4305 |
error = stp->sd_werror; |
|
4306 |
} |
|
4307 |
return (error); |
|
4308 |
} |
|
4309 |
||
4310 |
||
4311 |
/* |
|
4312 |
* single-thread open/close/push/pop |
|
4313 |
* for twisted streams also |
|
4314 |
*/ |
|
4315 |
int |
|
4316 |
strstartplumb(stdata_t *stp, int flag, int cmd) |
|
4317 |
{ |
|
4318 |
int waited = 1; |
|
4319 |
int error = 0; |
|
4320 |
||
4321 |
if (STRMATED(stp)) { |
|
4322 |
struct stdata *stmatep = stp->sd_mate; |
|
4323 |
||
4324 |
STRLOCKMATES(stp); |
|
4325 |
while (waited) { |
|
4326 |
waited = 0; |
|
4327 |
while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { |
|
4328 |
if ((cmd == I_POP) && |
|
4329 |
(flag & (FNDELAY|FNONBLOCK))) { |
|
4330 |
STRUNLOCKMATES(stp); |
|
4331 |
return (EAGAIN); |
|
4332 |
} |
|
4333 |
waited = 1; |
|
4334 |
mutex_exit(&stp->sd_lock); |
|
4335 |
if (!cv_wait_sig(&stmatep->sd_monitor, |
|
4336 |
&stmatep->sd_lock)) { |
|
4337 |
mutex_exit(&stmatep->sd_lock); |
|
4338 |
return (EINTR); |
|
4339 |
} |
|
4340 |
mutex_exit(&stmatep->sd_lock); |
|
4341 |
STRLOCKMATES(stp); |
|
4342 |
} |
|
4343 |
while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { |
|
4344 |
if ((cmd == I_POP) && |
|
4345 |
(flag & (FNDELAY|FNONBLOCK))) { |
|
4346 |
STRUNLOCKMATES(stp); |
|
4347 |
return (EAGAIN); |
|
4348 |
} |
|
4349 |
waited = 1; |
|
4350 |
mutex_exit(&stmatep->sd_lock); |
|
4351 |
if (!cv_wait_sig(&stp->sd_monitor, |
|
4352 |
&stp->sd_lock)) { |
|
4353 |
mutex_exit(&stp->sd_lock); |
|
4354 |
return (EINTR); |
|
4355 |
} |
|
4356 |
mutex_exit(&stp->sd_lock); |
|
4357 |
STRLOCKMATES(stp); |
|
4358 |
} |
|
4359 |
if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { |
|
4360 |
error = strgeterr(stp, |
|
4361 |
STRDERR|STWRERR|STRHUP|STPLEX, 0); |
|
4362 |
if (error != 0) { |
|
4363 |
STRUNLOCKMATES(stp); |
|
4364 |
return (error); |
|
4365 |
} |
|
4366 |
} |
|
4367 |
} |
|
4368 |
stp->sd_flag |= STRPLUMB; |
|
4369 |
STRUNLOCKMATES(stp); |
|
4370 |
} else { |
|
4371 |
mutex_enter(&stp->sd_lock); |
|
4372 |
while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { |
|
4373 |
if (((cmd == I_POP) || (cmd == _I_REMOVE)) && |
|
4374 |
(flag & (FNDELAY|FNONBLOCK))) { |
|
4375 |
mutex_exit(&stp->sd_lock); |
|
4376 |
return (EAGAIN); |
|
4377 |
} |
|
4378 |
if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) { |
|
4379 |
mutex_exit(&stp->sd_lock); |
|
4380 |
return (EINTR); |
|
4381 |
} |
|
4382 |
if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { |
|
4383 |
error = strgeterr(stp, |
|
4384 |
STRDERR|STWRERR|STRHUP|STPLEX, 0); |
|
4385 |
if (error != 0) { |
|
4386 |
mutex_exit(&stp->sd_lock); |
|
4387 |
return (error); |
|
4388 |
} |
|
4389 |
} |
|
4390 |
} |
|
4391 |
stp->sd_flag |= STRPLUMB; |
|
4392 |
mutex_exit(&stp->sd_lock); |
|
4393 |
} |
|
4394 |
return (0); |
|
4395 |
} |
|
4396 |
||
4397 |
/* |
|
4398 |
* Complete the plumbing operation associated with stream `stp'. |
|
4399 |
*/ |
|
4400 |
void |
|
4401 |
strendplumb(stdata_t *stp) |
|
4402 |
{ |
|
4403 |
ASSERT(MUTEX_HELD(&stp->sd_lock)); |
|
4404 |
ASSERT(stp->sd_flag & STRPLUMB); |
|
4405 |
stp->sd_flag &= ~STRPLUMB; |
|
4406 |
cv_broadcast(&stp->sd_monitor); |
|
4407 |
} |
|
4408 |
||
4409 |
/* |
|
4410 |
* This describes how the STREAMS framework handles synchronization |
|
4411 |
* during open/push and close/pop. |
|
4412 |
* The key interfaces for open and close are qprocson and qprocsoff, |
|
4413 |
* respectively. While the close case in general is harder both open |
|
4414 |
* have close have significant similarities. |
|
4415 |
* |
|
4416 |
* During close the STREAMS framework has to both ensure that there |
|
4417 |
* are no stale references to the queue pair (and syncq) that |
|
4418 |
* are being closed and also provide the guarantees that are documented |
|
4419 |
* in qprocsoff(9F). |
|
4420 |
* If there are stale references to the queue that is closing it can |
|
4421 |
* result in kernel memory corruption or kernel panics. |
|
4422 |
* |
|
4423 |
* Note that is it up to the module/driver to ensure that it itself |
|
4424 |
* does not have any stale references to the closing queues once its close |
|
4425 |
* routine returns. This includes: |
|
4426 |
* - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines |
|
4427 |
* associated with the queues. For timeout and bufcall callbacks the |
|
4428 |
* module/driver also has to ensure (or wait for) any callbacks that |
|
4429 |
* are in progress. |
|
4430 |
* - If the module/driver is using esballoc it has to ensure that any |
|
4431 |
* esballoc free functions do not refer to a queue that has closed. |
|
4432 |
* (Note that in general the close routine can not wait for the esballoc'ed |
|
4433 |
* messages to be freed since that can cause a deadlock.) |
|
4434 |
* - Cancelling any interrupts that refer to the closing queues and |
|
4435 |
* also ensuring that there are no interrupts in progress that will |
|
4436 |
* refer to the closing queues once the close routine returns. |
|
4437 |
* - For multiplexors removing any driver global state that refers to |
|
4438 |
* the closing queue and also ensuring that there are no threads in |
|
4439 |
* the multiplexor that has picked up a queue pointer but not yet |
|
4440 |
* finished using it. |
|
4441 |
* |
|
4442 |
* In addition, a driver/module can only reference the q_next pointer |
|
4443 |
* in its open, close, put, or service procedures or in a |
|
4444 |
* qtimeout/qbufcall callback procedure executing "on" the correct |
|
4445 |
* stream. Thus it can not reference the q_next pointer in an interrupt |
|
4446 |
* routine or a timeout, bufcall or esballoc callback routine. Likewise |
|
4447 |
* it can not reference q_next of a different queue e.g. in a mux that |
|
4448 |
* passes messages from one queues put/service procedure to another queue. |
|
4449 |
* In all the cases when the driver/module can not access the q_next |
|
4450 |
* field it must use the *next* versions e.g. canputnext instead of |
|
4451 |
* canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...). |
|
4452 |
* |
|
4453 |
* |
|
4454 |
* Assuming that the driver/module conforms to the above constraints |
|
4455 |
* the STREAMS framework has to avoid stale references to q_next for all |
|
4456 |
* the framework internal cases which include (but are not limited to): |
|
4457 |
* - Threads in canput/canputnext/backenable and elsewhere that are |
|
4458 |
* walking q_next. |
|
4459 |
* - Messages on a syncq that have a reference to the queue through b_queue. |
|
4460 |
* - Messages on an outer perimeter (syncq) that have a reference to the |
|
4461 |
* queue through b_queue. |
|
4462 |
* - Threads that use q_nfsrv (e.g. canput) to find a queue. |
|
4463 |
* Note that only canput and bcanput use q_nfsrv without any locking. |
|
4464 |
* |
|
4465 |
* The STREAMS framework providing the qprocsoff(9F) guarantees means that |
|
4466 |
* after qprocsoff returns, the framework has to ensure that no threads can |
|
4467 |
* enter the put or service routines for the closing read or write-side queue. |
|
4468 |
* In addition to preventing "direct" entry into the put procedures |
|
4469 |
* the framework also has to prevent messages being drained from |
|
4470 |
* the syncq or the outer perimeter. |
|
4471 |
* XXX Note that currently qdetach does relies on D_MTOCEXCL as the only |
|
4472 |
* mechanism to prevent qwriter(PERIM_OUTER) from running after |
|
4473 |
* qprocsoff has returned. |
|
4474 |
* Note that if a module/driver uses put(9F) on one of its own queues |
|
4475 |
* it is up to the module/driver to ensure that the put() doesn't |
|
4476 |
* get called when the queue is closing. |
|
4477 |
* |
|
4478 |
* |
|
4479 |
* The framework aspects of the above "contract" is implemented by |
|
4480 |
* qprocsoff, removeq, and strlock: |
|
4481 |
* - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from |
|
4482 |
* entering the service procedures. |
|
4483 |
* - strlock acquires the sd_lock and sd_reflock to prevent putnext, |
|
4484 |
* canputnext, backenable etc from dereferencing the q_next that will |
|
4485 |
* soon change. |
|
4486 |
* - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext |
|
4487 |
* or other q_next walker that uses claimstr/releasestr to finish. |
|
4488 |
* - optionally for every syncq in the stream strlock acquires all the |
|
4489 |
* sq_lock's and waits for all sq_counts to drop to a value that indicates |
|
4490 |
* that no thread executes in the put or service procedures and that no |
|
4491 |
* thread is draining into the module/driver. This ensures that no |
|
4492 |
* open, close, put, service, or qtimeout/qbufcall callback procedure is |
|
4493 |
* currently executing hence no such thread can end up with the old stale |
|
4494 |
* q_next value and no canput/backenable can have the old stale |
|
4495 |
* q_nfsrv/q_next. |
|
4496 |
* - qdetach (wait_svc) makes sure that any scheduled or running threads |
|
4497 |
* have either finished or observed the QWCLOSE flag and gone away. |
|
4498 |
*/ |
|
4499 |
||
4500 |
||
4501 |
/* |
|
4502 |
* Get all the locks necessary to change q_next. |
|
4503 |
* |
|
4504 |
* Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the |
|
4505 |
* sq_count of each syncq in the list to drop to sq_rmqcount, indicating that |
|
4506 |
* the only threads inside the sqncq are threads currently calling removeq(). |
|
4507 |
* Since threads calling removeq() are in the process of removing their queues |
|
4508 |
* from the stream, we do not need to worry about them accessing a stale q_next |
|
4509 |
* pointer and thus we do not need to wait for them to exit (in fact, waiting |
|
4510 |
* for them can cause deadlock). |
|
4511 |
* |
|
4512 |
* This routine is subject to starvation since it does not set any flag to |
|
4513 |
* prevent threads from entering a module in the stream(i.e. sq_count can |
|
4514 |
* increase on some syncq while it is waiting on some other syncq.) |
|
4515 |
* |
|
4516 |
* Assumes that only one thread attempts to call strlock for a given |
|
4517 |
* stream. If this is not the case the two threads would deadlock. |
|
4518 |
* This assumption is guaranteed since strlock is only called by insertq |
|
4519 |
* and removeq and streams plumbing changes are single-threaded for |
|
4520 |
* a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags. |
|
4521 |
* |
|
4522 |
* For pipes, it is not difficult to atomically designate a pair of streams |
|
4523 |
* to be mated. Once mated atomically by the framework the twisted pair remain |
|
4524 |
* configured that way until dismantled atomically by the framework. |
|
4525 |
* When plumbing takes place on a twisted stream it is necessary to ensure that |
|
4526 |
* this operation is done exclusively on the twisted stream since two such |
|
4527 |
* operations, each initiated on different ends of the pipe will deadlock |
|
4528 |
* waiting for each other to complete. |
|
4529 |
* |
|
4530 |
* On entry, no locks should be held. |
|
4531 |
* The locks acquired and held by strlock depends on a few factors. |
|
4532 |
* - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired |
|
4533 |
* and held on exit and all sq_count are at an acceptable level. |
|
4534 |
* - In all cases, sd_lock and sd_reflock are acquired and held on exit with |
|
4535 |
* sd_refcnt being zero. |
|
4536 |
*/ |
|
4537 |
||
4538 |
static void |
|
4539 |
strlock(struct stdata *stp, sqlist_t *sqlist) |
|
4540 |
{ |
|
4541 |
syncql_t *sql, *sql2; |
|
4542 |
retry: |
|
4543 |
/* |
|
4544 |
* Wait for any claimstr to go away. |
|
4545 |
*/ |
|
4546 |
if (STRMATED(stp)) { |
|
4547 |
struct stdata *stp1, *stp2; |
|
4548 |
||
4549 |
STRLOCKMATES(stp); |
|
4550 |
/* |
|
4551 |
* Note that the selection of locking order is not |
|
4552 |
* important, just that they are always aquired in |
|
4553 |
* the same order. To assure this, we choose this |
|
4554 |
* order based on the value of the pointer, and since |
|
4555 |
* the pointer will not change for the life of this |
|
4556 |
* pair, we will always grab the locks in the same |
|
4557 |
* order (and hence, prevent deadlocks). |
|
4558 |
*/ |
|
4559 |
if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) { |
|
4560 |
stp1 = stp; |
|
4561 |
stp2 = stp->sd_mate; |
|
4562 |
} else { |
|
4563 |
stp2 = stp; |
|
4564 |
stp1 = stp->sd_mate; |
|
4565 |
} |
|
4566 |
mutex_enter(&stp1->sd_reflock); |
|
4567 |
if (stp1->sd_refcnt > 0) { |
|
4568 |
STRUNLOCKMATES(stp); |
|
166
519e5268bee7
4657000 releasestr() should only wakeup waiters when sq_refcnt is zero.
xy158873
parents:
0
diff
changeset
|
4569 |
cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock); |
0 | 4570 |
mutex_exit(&stp1->sd_reflock); |
4571 |
goto retry; |
|
4572 |
} |
|
4573 |
mutex_enter(&stp2->sd_reflock); |
|
4574 |
if (stp2->sd_refcnt > 0) { |
|
4575 |
STRUNLOCKMATES(stp); |
|
4576 |
mutex_exit(&stp1->sd_reflock); |
|
166
519e5268bee7
4657000 releasestr() should only wakeup waiters when sq_refcnt is zero.
xy158873
parents:
0
diff
changeset
|
4577 |
cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock); |
0 | 4578 |
mutex_exit(&stp2->sd_reflock); |
4579 |
goto retry; |
|
4580 |
} |
|
4581 |
STREAM_PUTLOCKS_ENTER(stp1); |
|
4582 |
STREAM_PUTLOCKS_ENTER(stp2); |
|
4583 |
} else { |
|
4584 |
mutex_enter(&stp->sd_lock); |
|
4585 |
mutex_enter(&stp->sd_reflock); |
|
4586 |
while (stp->sd_refcnt > 0) { |
|
4587 |
mutex_exit(&stp->sd_lock); |
|
166
519e5268bee7
4657000 releasestr() should only wakeup waiters when sq_refcnt is zero.
xy158873
parents:
0
diff
changeset
|
4588 |
cv_wait(&stp->sd_refmonitor, &stp->sd_reflock); |
0 | 4589 |
if (mutex_tryenter(&stp->sd_lock) == 0) { |
4590 |
mutex_exit(&stp->sd_reflock); |
|
4591 |
mutex_enter(&stp->sd_lock); |
|
4592 |
mutex_enter(&stp->sd_reflock); |
|
4593 |
} |
|
4594 |
} |
|
4595 |
STREAM_PUTLOCKS_ENTER(stp); |
|
4596 |
} |
|
4597 |
||
4598 |
if (sqlist == NULL) |
|
4599 |
return; |
|
4600 |
||
4601 |
for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { |
|
4602 |
syncq_t *sq = sql->sql_sq; |
|
4603 |
uint16_t count; |
|
4604 |
||
4605 |
mutex_enter(SQLOCK(sq)); |
|
4606 |
count = sq->sq_count; |
|
4607 |
ASSERT(sq->sq_rmqcount <= count); |
|
4608 |
SQ_PUTLOCKS_ENTER(sq); |
|
4609 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
4610 |
if (count == sq->sq_rmqcount) |
|
4611 |
continue; |
|
4612 |
||
4613 |
/* Failed - drop all locks that we have acquired so far */ |
|
4614 |
if (STRMATED(stp)) { |
|
4615 |
STREAM_PUTLOCKS_EXIT(stp); |
|
4616 |
STREAM_PUTLOCKS_EXIT(stp->sd_mate); |
|
4617 |
STRUNLOCKMATES(stp); |
|
4618 |
mutex_exit(&stp->sd_reflock); |
|
4619 |
mutex_exit(&stp->sd_mate->sd_reflock); |
|
4620 |
} else { |
|
4621 |
STREAM_PUTLOCKS_EXIT(stp); |
|
4622 |
mutex_exit(&stp->sd_lock); |
|
4623 |
mutex_exit(&stp->sd_reflock); |
|
4624 |
} |
|
4625 |
for (sql2 = sqlist->sqlist_head; sql2 != sql; |
|
4626 |
sql2 = sql2->sql_next) { |
|
4627 |
SQ_PUTLOCKS_EXIT(sql2->sql_sq); |
|
4628 |
mutex_exit(SQLOCK(sql2->sql_sq)); |
|
4629 |
} |
|
4630 |
||
4631 |
/* |
|
4632 |
* The wait loop below may starve when there are many threads |
|
4633 |
* claiming the syncq. This is especially a problem with permod |
|
4634 |
* syncqs (IP). To lessen the impact of the problem we increment |
|
4635 |
* sq_needexcl and clear fastbits so that putnexts will slow |
|
4636 |
* down and call sqenable instead of draining right away. |
|
4637 |
*/ |
|
4638 |
sq->sq_needexcl++; |
|
4639 |
SQ_PUTCOUNT_CLRFAST_LOCKED(sq); |
|
4640 |
while (count > sq->sq_rmqcount) { |
|
4641 |
sq->sq_flags |= SQ_WANTWAKEUP; |
|
4642 |
SQ_PUTLOCKS_EXIT(sq); |
|
4643 |
cv_wait(&sq->sq_wait, SQLOCK(sq)); |
|
4644 |
count = sq->sq_count; |
|
4645 |
SQ_PUTLOCKS_ENTER(sq); |
|
4646 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
4647 |
} |
|
4648 |
sq->sq_needexcl--; |
|
4649 |
if (sq->sq_needexcl == 0) |
|
4650 |
SQ_PUTCOUNT_SETFAST_LOCKED(sq); |
|
4651 |
SQ_PUTLOCKS_EXIT(sq); |
|
4652 |
ASSERT(count == sq->sq_rmqcount); |
|
4653 |
mutex_exit(SQLOCK(sq)); |
|
4654 |
goto retry; |
|
4655 |
} |
|
4656 |
} |
|
4657 |
||
4658 |
/* |
|
4659 |
* Drop all the locks that strlock acquired. |
|
4660 |
*/ |
|
4661 |
static void |
|
4662 |
strunlock(struct stdata *stp, sqlist_t *sqlist) |
|
4663 |
{ |
|
4664 |
syncql_t *sql; |
|
4665 |
||
4666 |
if (STRMATED(stp)) { |
|
4667 |
STREAM_PUTLOCKS_EXIT(stp); |
|
4668 |
STREAM_PUTLOCKS_EXIT(stp->sd_mate); |
|
4669 |
STRUNLOCKMATES(stp); |
|
4670 |
mutex_exit(&stp->sd_reflock); |
|
4671 |
mutex_exit(&stp->sd_mate->sd_reflock); |
|
4672 |
} else { |
|
4673 |
STREAM_PUTLOCKS_EXIT(stp); |
|
4674 |
mutex_exit(&stp->sd_lock); |
|
4675 |
mutex_exit(&stp->sd_reflock); |
|
4676 |
} |
|
4677 |
||
4678 |
if (sqlist == NULL) |
|
4679 |
return; |
|
4680 |
||
4681 |
for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { |
|
4682 |
SQ_PUTLOCKS_EXIT(sql->sql_sq); |
|
4683 |
mutex_exit(SQLOCK(sql->sql_sq)); |
|
4684 |
} |
|
4685 |
} |
|
4686 |
||
235
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4687 |
/* |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4688 |
* When the module has service procedure, we need check if the next |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4689 |
* module which has service procedure is in flow control to trigger |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4690 |
* the backenable. |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4691 |
*/ |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4692 |
static void |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4693 |
backenable_insertedq(queue_t *q) |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4694 |
{ |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4695 |
qband_t *qbp; |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4696 |
|
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4697 |
claimstr(q); |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4698 |
if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) { |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4699 |
if (q->q_next->q_nfsrv->q_flag & QWANTW) |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4700 |
backenable(q, 0); |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4701 |
|
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4702 |
qbp = q->q_next->q_nfsrv->q_bandp; |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4703 |
for (; qbp != NULL; qbp = qbp->qb_next) |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4704 |
if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL) |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4705 |
backenable(q, qbp->qb_first->b_band); |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4706 |
} |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4707 |
releasestr(q); |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4708 |
} |
0 | 4709 |
|
4710 |
/* |
|
4711 |
* Given two read queues, insert a new single one after another. |
|
4712 |
* |
|
4713 |
* This routine acquires all the necessary locks in order to change |
|
4714 |
* q_next and related pointer using strlock(). |
|
4715 |
* It depends on the stream head ensuring that there are no concurrent |
|
4716 |
* insertq or removeq on the same stream. The stream head ensures this |
|
4717 |
* using the flags STWOPEN, STRCLOSE, and STRPLUMB. |
|
4718 |
* |
|
4719 |
* Note that no syncq locks are held during the q_next change. This is |
|
4720 |
* applied to all streams since, unlike removeq, there is no problem of stale |
|
4721 |
* pointers when adding a module to the stream. Thus drivers/modules that do a |
|
4722 |
* canput(rq->q_next) would never get a closed/freed queue pointer even if we |
|
4723 |
* applied this optimization to all streams. |
|
4724 |
*/ |
|
4725 |
void |
|
4726 |
insertq(struct stdata *stp, queue_t *new) |
|
4727 |
{ |
|
4728 |
queue_t *after; |
|
4729 |
queue_t *wafter; |
|
4730 |
queue_t *wnew = _WR(new); |
|
4731 |
boolean_t have_fifo = B_FALSE; |
|
4732 |
||
4733 |
if (new->q_flag & _QINSERTING) { |
|
4734 |
ASSERT(stp->sd_vnode->v_type != VFIFO); |
|
4735 |
after = new->q_next; |
|
4736 |
wafter = _WR(new->q_next); |
|
4737 |
} else { |
|
4738 |
after = _RD(stp->sd_wrq); |
|
4739 |
wafter = stp->sd_wrq; |
|
4740 |
} |
|
4741 |
||
4742 |
TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ, |
|
4743 |
"insertq:%p, %p", after, new); |
|
4744 |
ASSERT(after->q_flag & QREADR); |
|
4745 |
ASSERT(new->q_flag & QREADR); |
|
4746 |
||
4747 |
strlock(stp, NULL); |
|
4748 |
||
4749 |
/* Do we have a FIFO? */ |
|
4750 |
if (wafter->q_next == after) { |
|
4751 |
have_fifo = B_TRUE; |
|
4752 |
wnew->q_next = new; |
|
4753 |
} else { |
|
4754 |
wnew->q_next = wafter->q_next; |
|
4755 |
} |
|
4756 |
new->q_next = after; |
|
4757 |
||
4758 |
set_nfsrv_ptr(new, wnew, after, wafter); |
|
4759 |
/* |
|
4760 |
* set_nfsrv_ptr() needs to know if this is an insertion or not, |
|
4761 |
* so only reset this flag after calling it. |
|
4762 |
*/ |
|
4763 |
new->q_flag &= ~_QINSERTING; |
|
4764 |
||
4765 |
if (have_fifo) { |
|
4766 |
wafter->q_next = wnew; |
|
4767 |
} else { |
|
4768 |
if (wafter->q_next) |
|
4769 |
_OTHERQ(wafter->q_next)->q_next = new; |
|
4770 |
wafter->q_next = wnew; |
|
4771 |
} |
|
4772 |
||
4773 |
set_qend(new); |
|
4774 |
/* The QEND flag might have to be updated for the upstream guy */ |
|
4775 |
set_qend(after); |
|
4776 |
||
4777 |
ASSERT(_SAMESTR(new) == O_SAMESTR(new)); |
|
4778 |
ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew)); |
|
4779 |
ASSERT(_SAMESTR(after) == O_SAMESTR(after)); |
|
4780 |
ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter)); |
|
4781 |
strsetuio(stp); |
|
4782 |
||
4783 |
/* |
|
4784 |
* If this was a module insertion, bump the push count. |
|
4785 |
*/ |
|
4786 |
if (!(new->q_flag & QISDRV)) |
|
4787 |
stp->sd_pushcnt++; |
|
4788 |
||
4789 |
strunlock(stp, NULL); |
|
235
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4790 |
|
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4791 |
/* check if the write Q needs backenable */ |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4792 |
backenable_insertedq(wnew); |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4793 |
|
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4794 |
/* check if the read Q needs backenable */ |
e87a1ba1f4a2
6233064 svc.startd is wedged trying to talk to the system console
micheng
parents:
166
diff
changeset
|
4795 |
backenable_insertedq(new); |
0 | 4796 |
} |
4797 |
||
4798 |
/* |
|
4799 |
* Given a read queue, unlink it from any neighbors. |
|
4800 |
* |
|
4801 |
* This routine acquires all the necessary locks in order to |
|
4802 |
* change q_next and related pointers and also guard against |
|
4803 |
* stale references (e.g. through q_next) to the queue that |
|
4804 |
* is being removed. It also plays part of the role in ensuring |
|
4805 |
* that the module's/driver's put procedure doesn't get called |
|
4806 |
* after qprocsoff returns. |
|
4807 |
* |
|
4808 |
* Removeq depends on the stream head ensuring that there are |
|
4809 |
* no concurrent insertq or removeq on the same stream. The |
|
4810 |
* stream head ensures this using the flags STWOPEN, STRCLOSE and |
|
4811 |
* STRPLUMB. |
|
4812 |
* |
|
4813 |
* The set of locks needed to remove the queue is different in |
|
4814 |
* different cases: |
|
4815 |
* |
|
4816 |
* Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after |
|
4817 |
* waiting for the syncq reference count to drop to 0 indicating that no |
|
4818 |
* non-close threads are present anywhere in the stream. This ensures that any |
|
4819 |
* module/driver can reference q_next in its open, close, put, or service |
|
4820 |
* procedures. |
|
4821 |
* |
|
4822 |
* The sq_rmqcount counter tracks the number of threads inside removeq(). |
|
4823 |
* strlock() ensures that there is either no threads executing inside perimeter |
|
4824 |
* or there is only a thread calling qprocsoff(). |
|
4825 |
* |
|
4826 |
* strlock() compares the value of sq_count with the number of threads inside |
|
4827 |
* removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup |
|
4828 |
* any threads waiting in strlock() when the sq_rmqcount increases. |
|
4829 |
*/ |
|
4830 |
||
4831 |
void |
|
4832 |
removeq(queue_t *qp) |
|
4833 |
{ |
|
4834 |
queue_t *wqp = _WR(qp); |
|
4835 |
struct stdata *stp = STREAM(qp); |
|
4836 |
sqlist_t *sqlist = NULL; |
|
4837 |
boolean_t isdriver; |
|
4838 |
int moved; |
|
4839 |
syncq_t *sq = qp->q_syncq; |
|
4840 |
syncq_t *wsq = wqp->q_syncq; |
|
4841 |
||
4842 |
ASSERT(stp); |
|
4843 |
||
4844 |
TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ, |
|
4845 |
"removeq:%p %p", qp, wqp); |
|
4846 |
ASSERT(qp->q_flag&QREADR); |
|
4847 |
||
4848 |
/* |
|
4849 |
* For queues using Synchronous streams, we must wait for all threads in |
|
4850 |
* rwnext() to drain out before proceeding. |
|
4851 |
*/ |
|
4852 |
if (qp->q_flag & QSYNCSTR) { |
|
4853 |
/* First, we need wakeup any threads blocked in rwnext() */ |
|
4854 |
mutex_enter(SQLOCK(sq)); |
|
4855 |
if (sq->sq_flags & SQ_WANTWAKEUP) { |
|
4856 |
sq->sq_flags &= ~SQ_WANTWAKEUP; |
|
4857 |
cv_broadcast(&sq->sq_wait); |
|
4858 |
} |
|
4859 |
mutex_exit(SQLOCK(sq)); |
|
4860 |
||
4861 |
if (wsq != sq) { |
|
4862 |
mutex_enter(SQLOCK(wsq)); |
|
4863 |
if (wsq->sq_flags & SQ_WANTWAKEUP) { |
|
4864 |
wsq->sq_flags &= ~SQ_WANTWAKEUP; |
|
4865 |
cv_broadcast(&wsq->sq_wait); |
|
4866 |
} |
|
4867 |
mutex_exit(SQLOCK(wsq)); |
|
4868 |
} |
|
4869 |
||
4870 |
mutex_enter(QLOCK(qp)); |
|
4871 |
while (qp->q_rwcnt > 0) { |
|
4872 |
qp->q_flag |= QWANTRMQSYNC; |
|
4873 |
cv_wait(&qp->q_wait, QLOCK(qp)); |
|
4874 |
} |
|
4875 |
mutex_exit(QLOCK(qp)); |
|
4876 |
||
4877 |
mutex_enter(QLOCK(wqp)); |
|
4878 |
while (wqp->q_rwcnt > 0) { |
|
4879 |
wqp->q_flag |= QWANTRMQSYNC; |
|
4880 |
cv_wait(&wqp->q_wait, QLOCK(wqp)); |
|
4881 |
} |
|
4882 |
mutex_exit(QLOCK(wqp)); |
|
4883 |
} |
|
4884 |
||
4885 |
mutex_enter(SQLOCK(sq)); |
|
4886 |
sq->sq_rmqcount++; |
|
4887 |
if (sq->sq_flags & SQ_WANTWAKEUP) { |
|
4888 |
sq->sq_flags &= ~SQ_WANTWAKEUP; |
|
4889 |
cv_broadcast(&sq->sq_wait); |
|
4890 |
} |
|
4891 |
mutex_exit(SQLOCK(sq)); |
|
4892 |
||
4893 |
isdriver = (qp->q_flag & QISDRV); |
|
4894 |
||
4895 |
sqlist = sqlist_build(qp, stp, STRMATED(stp)); |
|
4896 |
strlock(stp, sqlist); |
|
4897 |
||
4898 |
reset_nfsrv_ptr(qp, wqp); |
|
4899 |
||
4900 |
ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp); |
|
4901 |
ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp); |
|
4902 |
/* Do we have a FIFO? */ |
|
4903 |
if (wqp->q_next == qp) { |
|
4904 |
stp->sd_wrq->q_next = _RD(stp->sd_wrq); |
|
4905 |
} else { |
|
4906 |
if (wqp->q_next) |
|
4907 |
backq(qp)->q_next = qp->q_next; |
|
4908 |
if (qp->q_next) |
|
4909 |
backq(wqp)->q_next = wqp->q_next; |
|
4910 |
} |
|
4911 |
||
4912 |
/* The QEND flag might have to be updated for the upstream guy */ |
|
4913 |
if (qp->q_next) |
|
4914 |
set_qend(qp->q_next); |
|
4915 |
||
4916 |
ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq)); |
|
4917 |
ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq))); |
|
4918 |
||
4919 |
/* |
|
4920 |
* Move any messages destined for the put procedures to the next |
|
4921 |
* syncq in line. Otherwise free them. |
|
4922 |
*/ |
|
4923 |
moved = 0; |
|
4924 |
/* |
|
4925 |
* Quick check to see whether there are any messages or events. |
|
4926 |
*/ |
|
4927 |
if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS)) |
|
4928 |
moved += propagate_syncq(qp); |
|
4929 |
if (wqp->q_syncqmsgs != 0 || |
|
4930 |
(wqp->q_syncq->sq_flags & SQ_EVENTS)) |
|
4931 |
moved += propagate_syncq(wqp); |
|
4932 |
||
4933 |
strsetuio(stp); |
|
4934 |
||
4935 |
/* |
|
4936 |
* If this was a module removal, decrement the push count. |
|
4937 |
*/ |
|
4938 |
if (!isdriver) |
|
4939 |
stp->sd_pushcnt--; |
|
4940 |
||
4941 |
strunlock(stp, sqlist); |
|
4942 |
sqlist_free(sqlist); |
|
4943 |
||
4944 |
/* |
|
4945 |
* Make sure any messages that were propagated are drained. |
|
4946 |
* Also clear any QFULL bit caused by messages that were propagated. |
|
4947 |
*/ |
|
4948 |
||
4949 |
if (qp->q_next != NULL) { |
|
4950 |
clr_qfull(qp); |
|
4951 |
/* |
|
4952 |
* For the driver calling qprocsoff, propagate_syncq |
|
4953 |
* frees all the messages instead of putting it in |
|
4954 |
* the stream head |
|
4955 |
*/ |
|
4956 |
if (!isdriver && (moved > 0)) |
|
4957 |
emptysq(qp->q_next->q_syncq); |
|
4958 |
} |
|
4959 |
if (wqp->q_next != NULL) { |
|
4960 |
clr_qfull(wqp); |
|
4961 |
/* |
|
4962 |
* We come here for any pop of a module except for the |
|
4963 |
* case of driver being removed. We don't call emptysq |
|
4964 |
* if we did not move any messages. This will avoid holding |
|
4965 |
* PERMOD syncq locks in emptysq |
|
4966 |
*/ |
|
4967 |
if (moved > 0) |
|
4968 |
emptysq(wqp->q_next->q_syncq); |
|
4969 |
} |
|
4970 |
||
4971 |
mutex_enter(SQLOCK(sq)); |
|
4972 |
sq->sq_rmqcount--; |
|
4973 |
mutex_exit(SQLOCK(sq)); |
|
4974 |
} |
|
4975 |
||
4976 |
/* |
|
4977 |
* Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or |
|
4978 |
* SQ_WRITER) on a syncq. |
|
4979 |
* If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the |
|
4980 |
* sync queue and waits until sq_count reaches maxcnt. |
|
4981 |
* |
|
4982 |
* if maxcnt is -1 there's no need to grab sq_putlocks since the caller |
|
4983 |
* does not care about putnext threads that are in the middle of calling put |
|
4984 |
* entry points. |
|
4985 |
* |
|
4986 |
* This routine is used for both inner and outer syncqs. |
|
4987 |
*/ |
|
4988 |
static void |
|
4989 |
blocksq(syncq_t *sq, ushort_t flag, int maxcnt) |
|
4990 |
{ |
|
4991 |
uint16_t count = 0; |
|
4992 |
||
4993 |
mutex_enter(SQLOCK(sq)); |
|
4994 |
/* |
|
4995 |
* Wait for SQ_FROZEN/SQ_BLOCKED to be reset. |
|
4996 |
* SQ_FROZEN will be set if there is a frozen stream that has a |
|
4997 |
* queue which also refers to this "shared" syncq. |
|
4998 |
* SQ_BLOCKED will be set if there is "off" queue which also |
|
4999 |
* refers to this "shared" syncq. |
|
5000 |
*/ |
|
5001 |
if (maxcnt != -1) { |
|
5002 |
count = sq->sq_count; |
|
5003 |
SQ_PUTLOCKS_ENTER(sq); |
|
5004 |
SQ_PUTCOUNT_CLRFAST_LOCKED(sq); |
|
5005 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
5006 |
} |
|
5007 |
sq->sq_needexcl++; |
|
5008 |
ASSERT(sq->sq_needexcl != 0); /* wraparound */ |
|
5009 |
||
5010 |
while ((sq->sq_flags & flag) || |
|
5011 |
(maxcnt != -1 && count > (unsigned)maxcnt)) { |
|
5012 |
sq->sq_flags |= SQ_WANTWAKEUP; |
|
5013 |
if (maxcnt != -1) { |
|
5014 |
SQ_PUTLOCKS_EXIT(sq); |
|
5015 |
} |
|
5016 |
cv_wait(&sq->sq_wait, SQLOCK(sq)); |
|
5017 |
if (maxcnt != -1) { |
|
5018 |
count = sq->sq_count; |
|
5019 |
SQ_PUTLOCKS_ENTER(sq); |
|
5020 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
5021 |
} |
|
5022 |
} |
|
5023 |
sq->sq_needexcl--; |
|
5024 |
sq->sq_flags |= flag; |
|
5025 |
ASSERT(maxcnt == -1 || count == maxcnt); |
|
5026 |
if (maxcnt != -1) { |
|
5027 |
if (sq->sq_needexcl == 0) { |
|
5028 |
SQ_PUTCOUNT_SETFAST_LOCKED(sq); |
|
5029 |
} |
|
5030 |
SQ_PUTLOCKS_EXIT(sq); |
|
5031 |
} else if (sq->sq_needexcl == 0) { |
|
5032 |
SQ_PUTCOUNT_SETFAST(sq); |
|
5033 |
} |
|
5034 |
||
5035 |
mutex_exit(SQLOCK(sq)); |
|
5036 |
} |
|
5037 |
||
5038 |
/* |
|
5039 |
* Reset a flag that was set with blocksq. |
|
5040 |
* |
|
5041 |
* Can not use this routine to reset SQ_WRITER. |
|
5042 |
* |
|
5043 |
* If "isouter" is set then the syncq is assumed to be an outer perimeter |
|
5044 |
* and drain_syncq is not called. Instead we rely on the qwriter_outer thread |
|
5045 |
* to handle the queued qwriter operations. |
|
5046 |
* |
|
5047 |
* no need to grab sq_putlocks here. See comment in strsubr.h that explains when |
|
5048 |
* sq_putlocks are used. |
|
5049 |
*/ |
|
5050 |
static void |
|
5051 |
unblocksq(syncq_t *sq, uint16_t resetflag, int isouter) |
|
5052 |
{ |
|
5053 |
uint16_t flags; |
|
5054 |
||
5055 |
mutex_enter(SQLOCK(sq)); |
|
5056 |
ASSERT(resetflag != SQ_WRITER); |
|
5057 |
ASSERT(sq->sq_flags & resetflag); |
|
5058 |
flags = sq->sq_flags & ~resetflag; |
|
5059 |
sq->sq_flags = flags; |
|
5060 |
if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) { |
|
5061 |
if (flags & SQ_WANTWAKEUP) { |
|
5062 |
flags &= ~SQ_WANTWAKEUP; |
|
5063 |
cv_broadcast(&sq->sq_wait); |
|
5064 |
} |
|
5065 |
sq->sq_flags = flags; |
|
5066 |
if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { |
|
5067 |
if (!isouter) { |
|
5068 |
/* drain_syncq drops SQLOCK */ |
|
5069 |
drain_syncq(sq); |
|
5070 |
return; |
|
5071 |
} |
|
5072 |
} |
|
5073 |
} |
|
5074 |
mutex_exit(SQLOCK(sq)); |
|
5075 |
} |
|
5076 |
||
5077 |
/* |
|
5078 |
* Reset a flag that was set with blocksq. |
|
5079 |
* Does not drain the syncq. Use emptysq() for that. |
|
5080 |
* Returns 1 if SQ_QUEUED is set. Otherwise 0. |
|
5081 |
* |
|
5082 |
* no need to grab sq_putlocks here. See comment in strsubr.h that explains when |
|
5083 |
* sq_putlocks are used. |
|
5084 |
*/ |
|
5085 |
static int |
|
5086 |
dropsq(syncq_t *sq, uint16_t resetflag) |
|
5087 |
{ |
|
5088 |
uint16_t flags; |
|
5089 |
||
5090 |
mutex_enter(SQLOCK(sq)); |
|
5091 |
ASSERT(sq->sq_flags & resetflag); |
|
5092 |
flags = sq->sq_flags & ~resetflag; |
|
5093 |
if (flags & SQ_WANTWAKEUP) { |
|
5094 |
flags &= ~SQ_WANTWAKEUP; |
|
5095 |
cv_broadcast(&sq->sq_wait); |
|
5096 |
} |
|
5097 |
sq->sq_flags = flags; |
|
5098 |
mutex_exit(SQLOCK(sq)); |
|
5099 |
if (flags & SQ_QUEUED) |
|
5100 |
return (1); |
|
5101 |
return (0); |
|
5102 |
} |
|
5103 |
||
5104 |
/* |
|
5105 |
* Empty all the messages on a syncq. |
|
5106 |
* |
|
5107 |
* no need to grab sq_putlocks here. See comment in strsubr.h that explains when |
|
5108 |
* sq_putlocks are used. |
|
5109 |
*/ |
|
5110 |
static void |
|
5111 |
emptysq(syncq_t *sq) |
|
5112 |
{ |
|
5113 |
uint16_t flags; |
|
5114 |
||
5115 |
mutex_enter(SQLOCK(sq)); |
|
5116 |
flags = sq->sq_flags; |
|
5117 |
if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { |
|
5118 |
/* |
|
5119 |
* To prevent potential recursive invocation of drain_syncq we |
|
5120 |
* do not call drain_syncq if count is non-zero. |
|
5121 |
*/ |
|
5122 |
if (sq->sq_count == 0) { |
|
5123 |
/* drain_syncq() drops SQLOCK */ |
|
5124 |
drain_syncq(sq); |
|
5125 |
return; |
|
5126 |
} else |
|
5127 |
sqenable(sq); |
|
5128 |
} |
|
5129 |
mutex_exit(SQLOCK(sq)); |
|
5130 |
} |
|
5131 |
||
5132 |
/* |
|
5133 |
* Ordered insert while removing duplicates. |
|
5134 |
*/ |
|
5135 |
static void |
|
5136 |
sqlist_insert(sqlist_t *sqlist, syncq_t *sqp) |
|
5137 |
{ |
|
5138 |
syncql_t *sqlp, **prev_sqlpp, *new_sqlp; |
|
5139 |
||
5140 |
prev_sqlpp = &sqlist->sqlist_head; |
|
5141 |
while ((sqlp = *prev_sqlpp) != NULL) { |
|
5142 |
if (sqlp->sql_sq >= sqp) { |
|
5143 |
if (sqlp->sql_sq == sqp) /* duplicate */ |
|
5144 |
return; |
|
5145 |
break; |
|
5146 |
} |
|
5147 |
prev_sqlpp = &sqlp->sql_next; |
|
5148 |
} |
|
5149 |
new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++]; |
|
5150 |
ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size); |
|
5151 |
new_sqlp->sql_next = sqlp; |
|
5152 |
new_sqlp->sql_sq = sqp; |
|
5153 |
*prev_sqlpp = new_sqlp; |
|
5154 |
} |
|
5155 |
||
5156 |
/* |
|
5157 |
* Walk the write side queues until we hit either the driver |
|
5158 |
* or a twist in the stream (_SAMESTR will return false in both |
|
5159 |
* these cases) then turn around and walk the read side queues |
|
5160 |
* back up to the stream head. |
|
5161 |
*/ |
|
5162 |
static void |
|
5163 |
sqlist_insertall(sqlist_t *sqlist, queue_t *q) |
|
5164 |
{ |
|
5165 |
while (q != NULL) { |
|
5166 |
sqlist_insert(sqlist, q->q_syncq); |
|
5167 |
||
5168 |
if (_SAMESTR(q)) |
|
5169 |
q = q->q_next; |
|
5170 |
else if (!(q->q_flag & QREADR)) |
|
5171 |
q = _RD(q); |
|
5172 |
else |
|
5173 |
q = NULL; |
|
5174 |
} |
|
5175 |
} |
|
5176 |
||
5177 |
/* |
|
5178 |
* Allocate and build a list of all syncqs in a stream and the syncq(s) |
|
5179 |
* associated with the "q" parameter. The resulting list is sorted in a |
|
5180 |
* canonical order and is free of duplicates. |
|
5181 |
* Assumes the passed queue is a _RD(q). |
|
5182 |
*/ |
|
5183 |
static sqlist_t * |
|
5184 |
sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist) |
|
5185 |
{ |
|
5186 |
sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP); |
|
5187 |
||
5188 |
/* |
|
5189 |
* start with the current queue/qpair |
|
5190 |
*/ |
|
5191 |
ASSERT(q->q_flag & QREADR); |
|
5192 |
||
5193 |
sqlist_insert(sqlist, q->q_syncq); |
|
5194 |
sqlist_insert(sqlist, _WR(q)->q_syncq); |
|
5195 |
||
5196 |
sqlist_insertall(sqlist, stp->sd_wrq); |
|
5197 |
if (do_twist) |
|
5198 |
sqlist_insertall(sqlist, stp->sd_mate->sd_wrq); |
|
5199 |
||
5200 |
return (sqlist); |
|
5201 |
} |
|
5202 |
||
5203 |
static sqlist_t * |
|
5204 |
sqlist_alloc(struct stdata *stp, int kmflag) |
|
5205 |
{ |
|
5206 |
size_t sqlist_size; |
|
5207 |
sqlist_t *sqlist; |
|
5208 |
||
5209 |
/* |
|
5210 |
* Allocate 2 syncql_t's for each pushed module. Note that |
|
5211 |
* the sqlist_t structure already has 4 syncql_t's built in: |
|
5212 |
* 2 for the stream head, and 2 for the driver/other stream head. |
|
5213 |
*/ |
|
5214 |
sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt + |
|
5215 |
sizeof (sqlist_t); |
|
5216 |
if (STRMATED(stp)) |
|
5217 |
sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt; |
|
5218 |
sqlist = kmem_alloc(sqlist_size, kmflag); |
|
5219 |
||
5220 |
sqlist->sqlist_head = NULL; |
|
5221 |
sqlist->sqlist_size = sqlist_size; |
|
5222 |
sqlist->sqlist_index = 0; |
|
5223 |
||
5224 |
return (sqlist); |
|
5225 |
} |
|
5226 |
||
5227 |
/* |
|
5228 |
* Free the list created by sqlist_alloc() |
|
5229 |
*/ |
|
5230 |
static void |
|
5231 |
sqlist_free(sqlist_t *sqlist) |
|
5232 |
{ |
|
5233 |
kmem_free(sqlist, sqlist->sqlist_size); |
|
5234 |
} |
|
5235 |
||
5236 |
/* |
|
5237 |
* Prevent any new entries into any syncq in this stream. |
|
5238 |
* Used by freezestr. |
|
5239 |
*/ |
|
5240 |
void |
|
5241 |
strblock(queue_t *q) |
|
5242 |
{ |
|
5243 |
struct stdata *stp; |
|
5244 |
syncql_t *sql; |
|
5245 |
sqlist_t *sqlist; |
|
5246 |
||
5247 |
q = _RD(q); |
|
5248 |
||
5249 |
stp = STREAM(q); |
|
5250 |
ASSERT(stp != NULL); |
|
5251 |
||
5252 |
/* |
|
5253 |
* Get a sorted list with all the duplicates removed containing |
|
5254 |
* all the syncqs referenced by this stream. |
|
5255 |
*/ |
|
5256 |
sqlist = sqlist_build(q, stp, B_FALSE); |
|
5257 |
for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) |
|
5258 |
blocksq(sql->sql_sq, SQ_FROZEN, -1); |
|
5259 |
sqlist_free(sqlist); |
|
5260 |
} |
|
5261 |
||
5262 |
/* |
|
5263 |
* Release the block on new entries into this stream |
|
5264 |
*/ |
|
5265 |
void |
|
5266 |
strunblock(queue_t *q) |
|
5267 |
{ |
|
5268 |
struct stdata *stp; |
|
5269 |
syncql_t *sql; |
|
5270 |
sqlist_t *sqlist; |
|
5271 |
int drain_needed; |
|
5272 |
||
5273 |
q = _RD(q); |
|
5274 |
||
5275 |
/* |
|
5276 |
* Get a sorted list with all the duplicates removed containing |
|
5277 |
* all the syncqs referenced by this stream. |
|
5278 |
* Have to drop the SQ_FROZEN flag on all the syncqs before |
|
5279 |
* starting to drain them; otherwise the draining might |
|
5280 |
* cause a freezestr in some module on the stream (which |
|
5281 |
* would deadlock.) |
|
5282 |
*/ |
|
5283 |
stp = STREAM(q); |
|
5284 |
ASSERT(stp != NULL); |
|
5285 |
sqlist = sqlist_build(q, stp, B_FALSE); |
|
5286 |
drain_needed = 0; |
|
5287 |
for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) |
|
5288 |
drain_needed += dropsq(sql->sql_sq, SQ_FROZEN); |
|
5289 |
if (drain_needed) { |
|
5290 |
for (sql = sqlist->sqlist_head; sql != NULL; |
|
5291 |
sql = sql->sql_next) |
|
5292 |
emptysq(sql->sql_sq); |
|
5293 |
} |
|
5294 |
sqlist_free(sqlist); |
|
5295 |
} |
|
5296 |
||
5297 |
#ifdef DEBUG |
|
5298 |
static int |
|
5299 |
qprocsareon(queue_t *rq) |
|
5300 |
{ |
|
5301 |
if (rq->q_next == NULL) |
|
5302 |
return (0); |
|
5303 |
return (_WR(rq->q_next)->q_next == _WR(rq)); |
|
5304 |
} |
|
5305 |
||
5306 |
int |
|
5307 |
qclaimed(queue_t *q) |
|
5308 |
{ |
|
5309 |
uint_t count; |
|
5310 |
||
5311 |
count = q->q_syncq->sq_count; |
|
5312 |
SUM_SQ_PUTCOUNTS(q->q_syncq, count); |
|
5313 |
return (count != 0); |
|
5314 |
} |
|
5315 |
||
5316 |
/* |
|
5317 |
* Check if anyone has frozen this stream with freezestr |
|
5318 |
*/ |
|
5319 |
int |
|
5320 |
frozenstr(queue_t *q) |
|
5321 |
{ |
|
5322 |
return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0); |
|
5323 |
} |
|
5324 |
#endif /* DEBUG */ |
|
5325 |
||
5326 |
/* |
|
5327 |
* Enter a queue. |
|
5328 |
* Obsoleted interface. Should not be used. |
|
5329 |
*/ |
|
5330 |
void |
|
5331 |
enterq(queue_t *q) |
|
5332 |
{ |
|
5333 |
entersq(q->q_syncq, SQ_CALLBACK); |
|
5334 |
} |
|
5335 |
||
5336 |
void |
|
5337 |
leaveq(queue_t *q) |
|
5338 |
{ |
|
5339 |
leavesq(q->q_syncq, SQ_CALLBACK); |
|
5340 |
} |
|
5341 |
||
5342 |
/* |
|
5343 |
* Enter a perimeter. c_inner and c_outer specifies which concurrency bits |
|
5344 |
* to check. |
|
5345 |
* Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter |
|
5346 |
* calls and the running of open, close and service procedures. |
|
5347 |
* |
|
5348 |
* if c_inner bit is set no need to grab sq_putlocks since we don't care |
|
5349 |
* if other threads have entered or are entering put entry point. |
|
5350 |
* |
|
5351 |
* if c_inner bit is set it might have been posible to use |
|
5352 |
* sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize |
|
5353 |
* open/close path for IP) but since the count may need to be decremented in |
|
5354 |
* qwait() we wouldn't know which counter to decrement. Currently counter is |
|
5355 |
* selected by current cpu_seqid and current CPU can change at any moment. XXX |
|
5356 |
* in the future we might use curthread id bits to select the counter and this |
|
5357 |
* would stay constant across routine calls. |
|
5358 |
*/ |
|
5359 |
void |
|
5360 |
entersq(syncq_t *sq, int entrypoint) |
|
5361 |
{ |
|
5362 |
uint16_t count = 0; |
|
5363 |
uint16_t flags; |
|
5364 |
uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; |
|
5365 |
uint16_t type; |
|
5366 |
uint_t c_inner = entrypoint & SQ_CI; |
|
5367 |
uint_t c_outer = entrypoint & SQ_CO; |
|
5368 |
||
5369 |
/* |
|
5370 |
* Increment ref count to keep closes out of this queue. |
|
5371 |
*/ |
|
5372 |
ASSERT(sq); |
|
5373 |
ASSERT(c_inner && c_outer); |
|
5374 |
mutex_enter(SQLOCK(sq)); |
|
5375 |
flags = sq->sq_flags; |
|
5376 |
type = sq->sq_type; |
|
5377 |
if (!(type & c_inner)) { |
|
5378 |
/* Make sure all putcounts now use slowlock. */ |
|
5379 |
count = sq->sq_count; |
|
5380 |
SQ_PUTLOCKS_ENTER(sq); |
|
5381 |
SQ_PUTCOUNT_CLRFAST_LOCKED(sq); |
|
5382 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
5383 |
sq->sq_needexcl++; |
|
5384 |
ASSERT(sq->sq_needexcl != 0); /* wraparound */ |
|
5385 |
waitflags |= SQ_MESSAGES; |
|
5386 |
} |
|
5387 |
/* |
|
5388 |
* Wait until we can enter the inner perimeter. |
|
5389 |
* If we want exclusive access we wait until sq_count is 0. |
|
5390 |
* We have to do this before entering the outer perimeter in order |
|
5391 |
* to preserve put/close message ordering. |
|
5392 |
*/ |
|
5393 |
while ((flags & waitflags) || (!(type & c_inner) && count != 0)) { |
|
5394 |
sq->sq_flags = flags | SQ_WANTWAKEUP; |
|
5395 |
if (!(type & c_inner)) { |
|
5396 |
SQ_PUTLOCKS_EXIT(sq); |
|
5397 |
} |
|
5398 |
cv_wait(&sq->sq_wait, SQLOCK(sq)); |
|
5399 |
if (!(type & c_inner)) { |
|
5400 |
count = sq->sq_count; |
|
5401 |
SQ_PUTLOCKS_ENTER(sq); |
|
5402 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
5403 |
} |
|
5404 |
flags = sq->sq_flags; |
|
5405 |
} |
|
5406 |
||
5407 |
if (!(type & c_inner)) { |
|
5408 |
ASSERT(sq->sq_needexcl > 0); |
|
5409 |
sq->sq_needexcl--; |
|
5410 |
if (sq->sq_needexcl == 0) { |
|
5411 |
SQ_PUTCOUNT_SETFAST_LOCKED(sq); |
|
5412 |
} |
|
5413 |
} |
|
5414 |
||
5415 |
/* Check if we need to enter the outer perimeter */ |
|
5416 |
if (!(type & c_outer)) { |
|
5417 |
/* |
|
5418 |
* We have to enter the outer perimeter exclusively before |
|
5419 |
* we can increment sq_count to avoid deadlock. This implies |
|
5420 |
* that we have to re-check sq_flags and sq_count. |
|
5421 |
* |
|
5422 |
* is it possible to have c_inner set when c_outer is not set? |
|
5423 |
*/ |
|
5424 |
if (!(type & c_inner)) { |
|
5425 |
SQ_PUTLOCKS_EXIT(sq); |
|
5426 |
} |
|
5427 |
mutex_exit(SQLOCK(sq)); |
|
5428 |
outer_enter(sq->sq_outer, SQ_GOAWAY); |
|
5429 |
mutex_enter(SQLOCK(sq)); |
|
5430 |
flags = sq->sq_flags; |
|
5431 |
/* |
|
5432 |
* there should be no need to recheck sq_putcounts |
|
5433 |
* because outer_enter() has already waited for them to clear |
|
5434 |
* after setting SQ_WRITER. |
|
5435 |
*/ |
|
5436 |
count = sq->sq_count; |
|
5437 |
#ifdef DEBUG |
|
5438 |
/* |
|
5439 |
* SUMCHECK_SQ_PUTCOUNTS should return the sum instead |
|
5440 |
* of doing an ASSERT internally. Others should do |
|
5441 |
* something like |
|
5442 |
* ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0); |
|
5443 |
* without the need to #ifdef DEBUG it. |
|
5444 |
*/ |
|
5445 |
SUMCHECK_SQ_PUTCOUNTS(sq, 0); |
|
5446 |
#endif |
|
5447 |
while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) || |
|
5448 |
(!(type & c_inner) && count != 0)) { |
|
5449 |
sq->sq_flags = flags | SQ_WANTWAKEUP; |
|
5450 |
cv_wait(&sq->sq_wait, SQLOCK(sq)); |
|
5451 |
count = sq->sq_count; |
|
5452 |
flags = sq->sq_flags; |
|
5453 |
} |
|
5454 |
} |
|
5455 |
||
5456 |
sq->sq_count++; |
|
5457 |
ASSERT(sq->sq_count != 0); /* Wraparound */ |
|
5458 |
if (!(type & c_inner)) { |
|
5459 |
/* Exclusive entry */ |
|
5460 |
ASSERT(sq->sq_count == 1); |
|
5461 |
sq->sq_flags |= SQ_EXCL; |
|
5462 |
if (type & c_outer) { |
|
5463 |
SQ_PUTLOCKS_EXIT(sq); |
|
5464 |
} |
|
5465 |
} |
|
5466 |
mutex_exit(SQLOCK(sq)); |
|
5467 |
} |
|
5468 |
||
5469 |
/* |
|
5470 |
* leave a syncq. announce to framework that closes may proceed. |
|
5471 |
* c_inner and c_outer specifies which concurrency bits |
|
5472 |
* to check. |
|
5473 |
* |
|
5474 |
* must never be called from driver or module put entry point. |
|
5475 |
* |
|
5476 |
* no need to grab sq_putlocks here. See comment in strsubr.h that explains when |
|
5477 |
* sq_putlocks are used. |
|
5478 |
*/ |
|
5479 |
void |
|
5480 |
leavesq(syncq_t *sq, int entrypoint) |
|
5481 |
{ |
|
5482 |
uint16_t flags; |
|
5483 |
uint16_t type; |
|
5484 |
uint_t c_outer = entrypoint & SQ_CO; |
|
5485 |
#ifdef DEBUG |
|
5486 |
uint_t c_inner = entrypoint & SQ_CI; |
|
5487 |
#endif |
|
5488 |
||
5489 |
/* |
|
5490 |
* decrement ref count, drain the syncq if possible, and wake up |
|
5491 |
* any waiting close. |
|
5492 |
*/ |
|
5493 |
ASSERT(sq); |
|
5494 |
ASSERT(c_inner && c_outer); |
|
5495 |
mutex_enter(SQLOCK(sq)); |
|
5496 |
flags = sq->sq_flags; |
|
5497 |
type = sq->sq_type; |
|
5498 |
if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) { |
|
5499 |
||
5500 |
if (flags & SQ_WANTWAKEUP) { |
|
5501 |
flags &= ~SQ_WANTWAKEUP; |
|
5502 |
cv_broadcast(&sq->sq_wait); |
|
5503 |
} |
|
5504 |
if (flags & SQ_WANTEXWAKEUP) { |
|
5505 |
flags &= ~SQ_WANTEXWAKEUP; |
|
5506 |
cv_broadcast(&sq->sq_exitwait); |
|
5507 |
} |
|
5508 |
||
5509 |
if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) { |
|
5510 |
/* |
|
5511 |
* The syncq needs to be drained. "Exit" the syncq |
|
5512 |
* before calling drain_syncq. |
|
5513 |
*/ |
|
5514 |
ASSERT(sq->sq_count != 0); |
|
5515 |
sq->sq_count--; |
|
5516 |
ASSERT((flags & SQ_EXCL) || (type & c_inner)); |
|
5517 |
sq->sq_flags = flags & ~SQ_EXCL; |
|
5518 |
drain_syncq(sq); |
|
5519 |
ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); |
|
5520 |
/* Check if we need to exit the outer perimeter */ |
|
5521 |
/* XXX will this ever be true? */ |
|
5522 |
if (!(type & c_outer)) |
|
5523 |
outer_exit(sq->sq_outer); |
|
5524 |
return; |
|
5525 |
} |
|
5526 |
} |
|
5527 |
ASSERT(sq->sq_count != 0); |
|
5528 |
sq->sq_count--; |
|
5529 |
ASSERT((flags & SQ_EXCL) || (type & c_inner)); |
|
5530 |
sq->sq_flags = flags & ~SQ_EXCL; |
|
5531 |
mutex_exit(SQLOCK(sq)); |
|
5532 |
||
5533 |
/* Check if we need to exit the outer perimeter */ |
|
5534 |
if (!(sq->sq_type & c_outer)) |
|
5535 |
outer_exit(sq->sq_outer); |
|
5536 |
} |
|
5537 |
||
5538 |
/* |
|
5539 |
* Prevent q_next from changing in this stream by incrementing sq_count. |
|
5540 |
* |
|
5541 |
* no need to grab sq_putlocks here. See comment in strsubr.h that explains when |
|
5542 |
* sq_putlocks are used. |
|
5543 |
*/ |
|
5544 |
void |
|
5545 |
claimq(queue_t *qp) |
|
5546 |
{ |
|
5547 |
syncq_t *sq = qp->q_syncq; |
|
5548 |
||
5549 |
mutex_enter(SQLOCK(sq)); |
|
5550 |
sq->sq_count++; |
|
5551 |
ASSERT(sq->sq_count != 0); /* Wraparound */ |
|
5552 |
mutex_exit(SQLOCK(sq)); |
|
5553 |
} |
|
5554 |
||
5555 |
/* |
|
5556 |
* Undo claimq. |
|
5557 |
* |
|
5558 |
* no need to grab sq_putlocks here. See comment in strsubr.h that explains when |
|
5559 |
* sq_putlocks are used. |
|
5560 |
*/ |
|
5561 |
void |
|
5562 |
releaseq(queue_t *qp) |
|
5563 |
{ |
|
5564 |
syncq_t *sq = qp->q_syncq; |
|
5565 |
uint16_t flags; |
|
5566 |
||
5567 |
mutex_enter(SQLOCK(sq)); |
|
5568 |
ASSERT(sq->sq_count > 0); |
|
5569 |
sq->sq_count--; |
|
5570 |
||
5571 |
flags = sq->sq_flags; |
|
5572 |
if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) { |
|
5573 |
if (flags & SQ_WANTWAKEUP) { |
|
5574 |
flags &= ~SQ_WANTWAKEUP; |
|
5575 |
cv_broadcast(&sq->sq_wait); |
|
5576 |
} |
|
5577 |
sq->sq_flags = flags; |
|
5578 |
if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { |
|
5579 |
/* |
|
5580 |
* To prevent potential recursive invocation of |
|
5581 |
* drain_syncq we do not call drain_syncq if count is |
|
5582 |
* non-zero. |
|
5583 |
*/ |
|
5584 |
if (sq->sq_count == 0) { |
|
5585 |
drain_syncq(sq); |
|
5586 |
return; |
|
5587 |
} else |
|
5588 |
sqenable(sq); |
|
5589 |
} |
|
5590 |
} |
|
5591 |
mutex_exit(SQLOCK(sq)); |
|
5592 |
} |
|
5593 |
||
5594 |
/* |
|
5595 |
* Prevent q_next from changing in this stream by incrementing sd_refcnt. |
|
5596 |
*/ |
|
5597 |
void |
|
5598 |
claimstr(queue_t *qp) |
|
5599 |
{ |
|
5600 |
struct stdata *stp = STREAM(qp); |
|
5601 |
||
5602 |
mutex_enter(&stp->sd_reflock); |
|
5603 |
stp->sd_refcnt++; |
|
5604 |
ASSERT(stp->sd_refcnt != 0); /* Wraparound */ |
|
5605 |
mutex_exit(&stp->sd_reflock); |
|
5606 |
} |
|
5607 |
||
5608 |
/* |
|
5609 |
* Undo claimstr. |
|
5610 |
*/ |
|
5611 |
void |
|
5612 |
releasestr(queue_t *qp) |
|
5613 |
{ |
|
5614 |
struct stdata *stp = STREAM(qp); |
|
5615 |
||
5616 |
mutex_enter(&stp->sd_reflock); |
|
5617 |
ASSERT(stp->sd_refcnt != 0); |
|
166
519e5268bee7
4657000 releasestr() should only wakeup waiters when sq_refcnt is zero.
xy158873
parents:
0
diff
changeset
|
5618 |
if (--stp->sd_refcnt == 0) |
519e5268bee7
4657000 releasestr() should only wakeup waiters when sq_refcnt is zero.
xy158873
parents:
0
diff
changeset
|
5619 |
cv_broadcast(&stp->sd_refmonitor); |
0 | 5620 |
mutex_exit(&stp->sd_reflock); |
5621 |
} |
|
5622 |
||
5623 |
static syncq_t * |
|
5624 |
new_syncq(void) |
|
5625 |
{ |
|
5626 |
return (kmem_cache_alloc(syncq_cache, KM_SLEEP)); |
|
5627 |
} |
|
5628 |
||
5629 |
static void |
|
5630 |
free_syncq(syncq_t *sq) |
|
5631 |
{ |
|
5632 |
ASSERT(sq->sq_head == NULL); |
|
5633 |
ASSERT(sq->sq_outer == NULL); |
|
5634 |
ASSERT(sq->sq_callbpend == NULL); |
|
5635 |
ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) || |
|
5636 |
(sq->sq_onext == sq && sq->sq_oprev == sq)); |
|
5637 |
||
5638 |
if (sq->sq_ciputctrl != NULL) { |
|
5639 |
ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); |
|
5640 |
SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, |
|
5641 |
sq->sq_nciputctrl, 0); |
|
5642 |
ASSERT(ciputctrl_cache != NULL); |
|
5643 |
kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); |
|
5644 |
} |
|
5645 |
||
5646 |
sq->sq_tail = NULL; |
|
5647 |
sq->sq_evhead = NULL; |
|
5648 |
sq->sq_evtail = NULL; |
|
5649 |
sq->sq_ciputctrl = NULL; |
|
5650 |
sq->sq_nciputctrl = 0; |
|
5651 |
sq->sq_count = 0; |
|
5652 |
sq->sq_rmqcount = 0; |
|
5653 |
sq->sq_callbflags = 0; |
|
5654 |
sq->sq_cancelid = 0; |
|
5655 |
sq->sq_next = NULL; |
|
5656 |
sq->sq_needexcl = 0; |
|
5657 |
sq->sq_svcflags = 0; |
|
5658 |
sq->sq_nqueues = 0; |
|
5659 |
sq->sq_pri = 0; |
|
5660 |
sq->sq_onext = NULL; |
|
5661 |
sq->sq_oprev = NULL; |
|
5662 |
sq->sq_flags = 0; |
|
5663 |
sq->sq_type = 0; |
|
5664 |
sq->sq_servcount = 0; |
|
5665 |
||
5666 |
kmem_cache_free(syncq_cache, sq); |
|
5667 |
} |
|
5668 |
||
5669 |
/* Outer perimeter code */ |
|
5670 |
||
5671 |
/* |
|
5672 |
* The outer syncq uses the fields and flags in the syncq slightly |
|
5673 |
* differently from the inner syncqs. |
|
5674 |
* sq_count Incremented when there are pending or running |
|
5675 |
* writers at the outer perimeter to prevent the set of |
|
5676 |
* inner syncqs that belong to the outer perimeter from |
|
5677 |
* changing. |
|
5678 |
* sq_head/tail List of deferred qwriter(OUTER) operations. |
|
5679 |
* |
|
5680 |
* SQ_BLOCKED Set to prevent traversing of sq_next,sq_prev while |
|
5681 |
* inner syncqs are added to or removed from the |
|
5682 |
* outer perimeter. |
|
5683 |
* SQ_QUEUED sq_head/tail has messages or eventsqueued. |
|
5684 |
* |
|
5685 |
* SQ_WRITER A thread is currently traversing all the inner syncqs |
|
5686 |
* setting the SQ_WRITER flag. |
|
5687 |
*/ |
|
5688 |
||
5689 |
/* |
|
5690 |
* Get write access at the outer perimeter. |
|
5691 |
* Note that read access is done by entersq, putnext, and put by simply |
|
5692 |
* incrementing sq_count in the inner syncq. |
|
5693 |
* |
|
5694 |
* Waits until "flags" is no longer set in the outer to prevent multiple |
|
5695 |
* threads from having write access at the same time. SQ_WRITER has to be part |
|
5696 |
* of "flags". |
|
5697 |
* |
|
5698 |
* Increases sq_count on the outer syncq to keep away outer_insert/remove |
|
5699 |
* until the outer_exit is finished. |
|
5700 |
* |
|
5701 |
* outer_enter is vulnerable to starvation since it does not prevent new |
|
5702 |
* threads from entering the inner syncqs while it is waiting for sq_count to |
|
5703 |
* go to zero. |
|
5704 |
*/ |
|
5705 |
void |
|
5706 |
outer_enter(syncq_t *outer, uint16_t flags) |
|
5707 |
{ |
|
5708 |
syncq_t *sq; |
|
5709 |
int wait_needed; |
|
5710 |
uint16_t count; |
|
5711 |
||
5712 |
ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && |
|
5713 |
outer->sq_oprev != NULL); |
|
5714 |
ASSERT(flags & SQ_WRITER); |
|
5715 |
||
5716 |
retry: |
|
5717 |
mutex_enter(SQLOCK(outer)); |
|
5718 |
while (outer->sq_flags & flags) { |
|
5719 |
outer->sq_flags |= SQ_WANTWAKEUP; |
|
5720 |
cv_wait(&outer->sq_wait, SQLOCK(outer)); |
|
5721 |
} |
|
5722 |
||
5723 |
ASSERT(!(outer->sq_flags & SQ_WRITER)); |
|
5724 |
outer->sq_flags |= SQ_WRITER; |
|
5725 |
outer->sq_count++; |
|
5726 |
ASSERT(outer->sq_count != 0); /* wraparound */ |
|
5727 |
wait_needed = 0; |
|
5728 |
/* |
|
5729 |
* Set SQ_WRITER on all the inner syncqs while holding |
|
5730 |
* the SQLOCK on the outer syncq. This ensures that the changing |
|
5731 |
* of SQ_WRITER is atomic under the outer SQLOCK. |
|
5732 |
*/ |
|
5733 |
for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { |
|
5734 |
mutex_enter(SQLOCK(sq)); |
|
5735 |
count = sq->sq_count; |
|
5736 |
SQ_PUTLOCKS_ENTER(sq); |
|
5737 |
sq->sq_flags |= SQ_WRITER; |
|
5738 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
5739 |
if (count != 0) |
|
5740 |
wait_needed = 1; |
|
5741 |
SQ_PUTLOCKS_EXIT(sq); |
|
5742 |
mutex_exit(SQLOCK(sq)); |
|
5743 |
} |
|
5744 |
mutex_exit(SQLOCK(outer)); |
|
5745 |
||
5746 |
/* |
|
5747 |
* Get everybody out of the syncqs sequentially. |
|
5748 |
* Note that we don't actually need to aqiure the PUTLOCKS, since |
|
5749 |
* we have already cleared the fastbit, and set QWRITER. By |
|
5750 |
* definition, the count can not increase since putnext will |
|
5751 |
* take the slowlock path (and the purpose of aquiring the |
|
5752 |
* putlocks was to make sure it didn't increase while we were |
|
5753 |
* waiting). |
|
5754 |
* |
|
5755 |
* Note that we still aquire the PUTLOCKS to be safe. |
|
5756 |
*/ |
|
5757 |
if (wait_needed) { |
|
5758 |
for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { |
|
5759 |
mutex_enter(SQLOCK(sq)); |
|
5760 |
count = sq->sq_count; |
|
5761 |
SQ_PUTLOCKS_ENTER(sq); |
|
5762 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
5763 |
while (count != 0) { |
|
5764 |
sq->sq_flags |= SQ_WANTWAKEUP; |
|
5765 |
SQ_PUTLOCKS_EXIT(sq); |
|
5766 |
cv_wait(&sq->sq_wait, SQLOCK(sq)); |
|
5767 |
count = sq->sq_count; |
|
5768 |
SQ_PUTLOCKS_ENTER(sq); |
|
5769 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
5770 |
} |
|
5771 |
SQ_PUTLOCKS_EXIT(sq); |
|
5772 |
mutex_exit(SQLOCK(sq)); |
|
5773 |
} |
|
5774 |
/* |
|
5775 |
* Verify that none of the flags got set while we |
|
5776 |
* were waiting for the sq_counts to drop. |
|
5777 |
* If this happens we exit and retry entering the |
|
5778 |
* outer perimeter. |
|
5779 |
*/ |
|
5780 |
mutex_enter(SQLOCK(outer)); |
|
5781 |
if (outer->sq_flags & (flags & ~SQ_WRITER)) { |
|
5782 |
mutex_exit(SQLOCK(outer)); |
|
5783 |
outer_exit(outer); |
|
5784 |
goto retry; |
|
5785 |
} |
|
5786 |
mutex_exit(SQLOCK(outer)); |
|
5787 |
} |
|
5788 |
} |
|
5789 |
||
5790 |
/* |
|
5791 |
* Drop the write access at the outer perimeter. |
|
5792 |
* Read access is dropped implicitly (by putnext, put, and leavesq) by |
|
5793 |
* decrementing sq_count. |
|
5794 |
*/ |
|
5795 |
void |
|
5796 |
outer_exit(syncq_t *outer) |
|
5797 |
{ |
|
5798 |
syncq_t *sq; |
|
5799 |
int drain_needed; |
|
5800 |
uint16_t flags; |
|
5801 |
||
5802 |
ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && |
|
5803 |
outer->sq_oprev != NULL); |
|
5804 |
ASSERT(MUTEX_NOT_HELD(SQLOCK(outer))); |
|
5805 |
||
5806 |
/* |
|
5807 |
* Atomically (from the perspective of threads calling become_writer) |
|
5808 |
* drop the write access at the outer perimeter by holding |
|
5809 |
* SQLOCK(outer) across all the dropsq calls and the resetting of |
|
5810 |
* SQ_WRITER. |
|
5811 |
* This defines a locking order between the outer perimeter |
|
5812 |
* SQLOCK and the inner perimeter SQLOCKs. |
|
5813 |
*/ |
|
5814 |
mutex_enter(SQLOCK(outer)); |
|
5815 |
flags = outer->sq_flags; |
|
5816 |
ASSERT(outer->sq_flags & SQ_WRITER); |
|
5817 |
if (flags & SQ_QUEUED) { |
|
5818 |
write_now(outer); |
|
5819 |
flags = outer->sq_flags; |
|
5820 |
} |
|
5821 |
||
5822 |
/* |
|
5823 |
* sq_onext is stable since sq_count has not yet been decreased. |
|
5824 |
* Reset the SQ_WRITER flags in all syncqs. |
|
5825 |
* After dropping SQ_WRITER on the outer syncq we empty all the |
|
5826 |
* inner syncqs. |
|
5827 |
*/ |
|
5828 |
drain_needed = 0; |
|
5829 |
for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) |
|
5830 |
drain_needed += dropsq(sq, SQ_WRITER); |
|
5831 |
ASSERT(!(outer->sq_flags & SQ_QUEUED)); |
|
5832 |
flags &= ~SQ_WRITER; |
|
5833 |
if (drain_needed) { |
|
5834 |
outer->sq_flags = flags; |
|
5835 |
mutex_exit(SQLOCK(outer)); |
|
5836 |
for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) |
|
5837 |
emptysq(sq); |
|
5838 |
mutex_enter(SQLOCK(outer)); |
|
5839 |
flags = outer->sq_flags; |
|
5840 |
} |
|
5841 |
if (flags & SQ_WANTWAKEUP) { |
|
5842 |
flags &= ~SQ_WANTWAKEUP; |
|
5843 |
cv_broadcast(&outer->sq_wait); |
|
5844 |
} |
|
5845 |
outer->sq_flags = flags; |
|
5846 |
ASSERT(outer->sq_count > 0); |
|
5847 |
outer->sq_count--; |
|
5848 |
mutex_exit(SQLOCK(outer)); |
|
5849 |
} |
|
5850 |
||
5851 |
/* |
|
5852 |
* Add another syncq to an outer perimeter. |
|
5853 |
* Block out all other access to the outer perimeter while it is being |
|
5854 |
* changed using blocksq. |
|
5855 |
* Assumes that the caller has *not* done an outer_enter. |
|
5856 |
* |
|
5857 |
* Vulnerable to starvation in blocksq. |
|
5858 |
*/ |
|
5859 |
static void |
|
5860 |
outer_insert(syncq_t *outer, syncq_t *sq) |
|
5861 |
{ |
|
5862 |
ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && |
|
5863 |
outer->sq_oprev != NULL); |
|
5864 |
ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && |
|
5865 |
sq->sq_oprev == NULL); /* Can't be in an outer perimeter */ |
|
5866 |
||
5867 |
/* Get exclusive access to the outer perimeter list */ |
|
5868 |
blocksq(outer, SQ_BLOCKED, 0); |
|
5869 |
ASSERT(outer->sq_flags & SQ_BLOCKED); |
|
5870 |
ASSERT(!(outer->sq_flags & SQ_WRITER)); |
|
5871 |
||
5872 |
mutex_enter(SQLOCK(sq)); |
|
5873 |
sq->sq_outer = outer; |
|
5874 |
outer->sq_onext->sq_oprev = sq; |
|
5875 |
sq->sq_onext = outer->sq_onext; |
|
5876 |
outer->sq_onext = sq; |
|
5877 |
sq->sq_oprev = outer; |
|
5878 |
mutex_exit(SQLOCK(sq)); |
|
5879 |
unblocksq(outer, SQ_BLOCKED, 1); |
|
5880 |
} |
|
5881 |
||
5882 |
/* |
|
5883 |
* Remove a syncq from an outer perimeter. |
|
5884 |
* Block out all other access to the outer perimeter while it is being |
|
5885 |
* changed using blocksq. |
|
5886 |
* Assumes that the caller has *not* done an outer_enter. |
|
5887 |
* |
|
5888 |
* Vulnerable to starvation in blocksq. |
|
5889 |
*/ |
|
5890 |
static void |
|
5891 |
outer_remove(syncq_t *outer, syncq_t *sq) |
|
5892 |
{ |
|
5893 |
ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && |
|
5894 |
outer->sq_oprev != NULL); |
|
5895 |
ASSERT(sq->sq_outer == outer); |
|
5896 |
||
5897 |
/* Get exclusive access to the outer perimeter list */ |
|
5898 |
blocksq(outer, SQ_BLOCKED, 0); |
|
5899 |
ASSERT(outer->sq_flags & SQ_BLOCKED); |
|
5900 |
ASSERT(!(outer->sq_flags & SQ_WRITER)); |
|
5901 |
||
5902 |
mutex_enter(SQLOCK(sq)); |
|
5903 |
sq->sq_outer = NULL; |
|
5904 |
sq->sq_onext->sq_oprev = sq->sq_oprev; |
|
5905 |
sq->sq_oprev->sq_onext = sq->sq_onext; |
|
5906 |
sq->sq_oprev = sq->sq_onext = NULL; |
|
5907 |
mutex_exit(SQLOCK(sq)); |
|
5908 |
unblocksq(outer, SQ_BLOCKED, 1); |
|
5909 |
} |
|
5910 |
||
5911 |
/* |
|
5912 |
* Queue a deferred qwriter(OUTER) callback for this outer perimeter. |
|
5913 |
* If this is the first callback for this outer perimeter then add |
|
5914 |
* this outer perimeter to the list of outer perimeters that |
|
5915 |
* the qwriter_outer_thread will process. |
|
5916 |
* |
|
5917 |
* Increments sq_count in the outer syncq to prevent the membership |
|
5918 |
* of the outer perimeter (in terms of inner syncqs) to change while |
|
5919 |
* the callback is pending. |
|
5920 |
*/ |
|
5921 |
static void |
|
5922 |
queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp) |
|
5923 |
{ |
|
5924 |
ASSERT(MUTEX_HELD(SQLOCK(outer))); |
|
5925 |
||
5926 |
mp->b_prev = (mblk_t *)func; |
|
5927 |
mp->b_queue = q; |
|
5928 |
mp->b_next = NULL; |
|
5929 |
outer->sq_count++; /* Decremented when dequeued */ |
|
5930 |
ASSERT(outer->sq_count != 0); /* Wraparound */ |
|
5931 |
if (outer->sq_evhead == NULL) { |
|
5932 |
/* First message. */ |
|
5933 |
outer->sq_evhead = outer->sq_evtail = mp; |
|
5934 |
outer->sq_flags |= SQ_EVENTS; |
|
5935 |
mutex_exit(SQLOCK(outer)); |
|
5936 |
STRSTAT(qwr_outer); |
|
5937 |
(void) taskq_dispatch(streams_taskq, |
|
5938 |
(task_func_t *)qwriter_outer_service, outer, TQ_SLEEP); |
|
5939 |
} else { |
|
5940 |
ASSERT(outer->sq_flags & SQ_EVENTS); |
|
5941 |
outer->sq_evtail->b_next = mp; |
|
5942 |
outer->sq_evtail = mp; |
|
5943 |
mutex_exit(SQLOCK(outer)); |
|
5944 |
} |
|
5945 |
} |
|
5946 |
||
5947 |
/* |
|
5948 |
* Try and upgrade to write access at the outer perimeter. If this can |
|
5949 |
* not be done without blocking then queue the callback to be done |
|
5950 |
* by the qwriter_outer_thread. |
|
5951 |
* |
|
5952 |
* This routine can only be called from put or service procedures plus |
|
5953 |
* asynchronous callback routines that have properly entered to |
|
5954 |
* queue (with entersq.) Thus qwriter(OUTER) assumes the caller has one claim |
|
5955 |
* on the syncq associated with q. |
|
5956 |
*/ |
|
5957 |
void |
|
5958 |
qwriter_outer(queue_t *q, mblk_t *mp, void (*func)()) |
|
5959 |
{ |
|
5960 |
syncq_t *osq, *sq, *outer; |
|
5961 |
int failed; |
|
5962 |
uint16_t flags; |
|
5963 |
||
5964 |
osq = q->q_syncq; |
|
5965 |
outer = osq->sq_outer; |
|
5966 |
if (outer == NULL) |
|
5967 |
panic("qwriter(PERIM_OUTER): no outer perimeter"); |
|
5968 |
ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && |
|
5969 |
outer->sq_oprev != NULL); |
|
5970 |
||
5971 |
mutex_enter(SQLOCK(outer)); |
|
5972 |
flags = outer->sq_flags; |
|
5973 |
/* |
|
5974 |
* If some thread is traversing sq_next, or if we are blocked by |
|
5975 |
* outer_insert or outer_remove, or if the we already have queued |
|
5976 |
* callbacks, then queue this callback for later processing. |
|
5977 |
* |
|
5978 |
* Also queue the qwriter for an interrupt thread in order |
|
5979 |
* to reduce the time spent running at high IPL. |
|
5980 |
* to identify there are events. |
|
5981 |
*/ |
|
5982 |
if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) { |
|
5983 |
/* |
|
5984 |
* Queue the become_writer request. |
|
5985 |
* The queueing is atomic under SQLOCK(outer) in order |
|
5986 |
* to synchronize with outer_exit. |
|
5987 |
* queue_writer will drop the outer SQLOCK |
|
5988 |
*/ |
|
5989 |
if (flags & SQ_BLOCKED) { |
|
5990 |
/* Must set SQ_WRITER on inner perimeter */ |
|
5991 |
mutex_enter(SQLOCK(osq)); |
|
5992 |
osq->sq_flags |= SQ_WRITER; |
|
5993 |
mutex_exit(SQLOCK(osq)); |
|
5994 |
} else { |
|
5995 |
if (!(flags & SQ_WRITER)) { |
|
5996 |
/* |
|
5997 |
* The outer could have been SQ_BLOCKED thus |
|
5998 |
* SQ_WRITER might not be set on the inner. |
|
5999 |
*/ |
|
6000 |
mutex_enter(SQLOCK(osq)); |
|
6001 |
osq->sq_flags |= SQ_WRITER; |
|
6002 |
mutex_exit(SQLOCK(osq)); |
|
6003 |
} |
|
6004 |
ASSERT(osq->sq_flags & SQ_WRITER); |
|
6005 |
} |
|
6006 |
queue_writer(outer, func, q, mp); |
|
6007 |
return; |
|
6008 |
} |
|
6009 |
/* |
|
6010 |
* We are half-way to exclusive access to the outer perimeter. |
|
6011 |
* Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove |
|
6012 |
* while the inner syncqs are traversed. |
|
6013 |
*/ |
|
6014 |
outer->sq_count++; |
|
6015 |
ASSERT(outer->sq_count != 0); /* wraparound */ |
|
6016 |
flags |= SQ_WRITER; |
|
6017 |
/* |
|
6018 |
* Check if we can run the function immediately. Mark all |
|
6019 |
* syncqs with the writer flag to prevent new entries into |
|
6020 |
* put and service procedures. |
|
6021 |
* |
|
6022 |
* Set SQ_WRITER on all the inner syncqs while holding |
|
6023 |
* the SQLOCK on the outer syncq. This ensures that the changing |
|
6024 |
* of SQ_WRITER is atomic under the outer SQLOCK. |
|
6025 |
*/ |
|
6026 |
failed = 0; |
|
6027 |
for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { |
|
6028 |
uint16_t count; |
|
6029 |
uint_t maxcnt = (sq == osq) ? 1 : 0; |
|
6030 |
||
6031 |
mutex_enter(SQLOCK(sq)); |
|
6032 |
count = sq->sq_count; |
|
6033 |
SQ_PUTLOCKS_ENTER(sq); |
|
6034 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
6035 |
if (sq->sq_count > maxcnt) |
|
6036 |
failed = 1; |
|
6037 |
sq->sq_flags |= SQ_WRITER; |
|
6038 |
SQ_PUTLOCKS_EXIT(sq); |
|
6039 |
mutex_exit(SQLOCK(sq)); |
|
6040 |
} |
|
6041 |
if (failed) { |
|
6042 |
/* |
|
6043 |
* Some other thread has a read claim on the outer perimeter. |
|
6044 |
* Queue the callback for deferred processing. |
|
6045 |
* |
|
6046 |
* queue_writer will set SQ_QUEUED before we drop SQ_WRITER |
|
6047 |
* so that other qwriter(OUTER) calls will queue their |
|
6048 |
* callbacks as well. queue_writer increments sq_count so we |
|
6049 |
* decrement to compensate for the our increment. |
|
6050 |
* |
|
6051 |
* Dropping SQ_WRITER enables the writer thread to work |
|
6052 |
* on this outer perimeter. |
|
6053 |
*/ |
|
6054 |
outer->sq_flags = flags; |
|
6055 |
queue_writer(outer, func, q, mp); |
|
6056 |
/* queue_writer dropper the lock */ |
|
6057 |
mutex_enter(SQLOCK(outer)); |
|
6058 |
ASSERT(outer->sq_count > 0); |
|
6059 |
outer->sq_count--; |
|
6060 |
ASSERT(outer->sq_flags & SQ_WRITER); |
|
6061 |
flags = outer->sq_flags; |
|
6062 |
flags &= ~SQ_WRITER; |
|
6063 |
if (flags & SQ_WANTWAKEUP) { |
|
6064 |
flags &= ~SQ_WANTWAKEUP; |
|
6065 |
cv_broadcast(&outer->sq_wait); |
|
6066 |
} |
|
6067 |
outer->sq_flags = flags; |
|
6068 |
mutex_exit(SQLOCK(outer)); |
|
6069 |
return; |
|
6070 |
} else { |
|
6071 |
outer->sq_flags = flags; |
|
6072 |
mutex_exit(SQLOCK(outer)); |
|
6073 |
} |
|
6074 |
||
6075 |
/* Can run it immediately */ |
|
6076 |
(*func)(q, mp); |
|
6077 |
||
6078 |
outer_exit(outer); |
|
6079 |
} |
|
6080 |
||
6081 |
/* |
|
6082 |
* Dequeue all writer callbacks from the outer perimeter and run them. |
|
6083 |
*/ |
|
6084 |
static void |
|
6085 |
write_now(syncq_t *outer) |
|
6086 |
{ |
|
6087 |
mblk_t *mp; |
|
6088 |
queue_t *q; |
|
6089 |
void (*func)(); |
|
6090 |
||
6091 |
ASSERT(MUTEX_HELD(SQLOCK(outer))); |
|
6092 |
ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && |
|
6093 |
outer->sq_oprev != NULL); |
|
6094 |
while ((mp = outer->sq_evhead) != NULL) { |
|
6095 |
/* |
|
6096 |
* queues cannot be placed on the queuelist on the outer |
|
6097 |
* perimiter. |
|
6098 |
*/ |
|
6099 |
ASSERT(!(outer->sq_flags & SQ_MESSAGES)); |
|
6100 |
ASSERT((outer->sq_flags & SQ_EVENTS)); |
|
6101 |
||
6102 |
outer->sq_evhead = mp->b_next; |
|
6103 |
if (outer->sq_evhead == NULL) { |
|
6104 |
outer->sq_evtail = NULL; |
|
6105 |
outer->sq_flags &= ~SQ_EVENTS; |
|
6106 |
} |
|
6107 |
ASSERT(outer->sq_count != 0); |
|
6108 |
outer->sq_count--; /* Incremented when enqueued. */ |
|
6109 |
mutex_exit(SQLOCK(outer)); |
|
6110 |
/* |
|
6111 |
* Drop the message if the queue is closing. |
|
6112 |
* Make sure that the queue is "claimed" when the callback |
|
6113 |
* is run in order to satisfy various ASSERTs. |
|
6114 |
*/ |
|
6115 |
q = mp->b_queue; |
|
6116 |
func = (void (*)())mp->b_prev; |
|
6117 |
ASSERT(func != NULL); |
|
6118 |
mp->b_next = mp->b_prev = NULL; |
|
6119 |
if (q->q_flag & QWCLOSE) { |
|
6120 |
freemsg(mp); |
|
6121 |
} else { |
|
6122 |
claimq(q); |
|
6123 |
(*func)(q, mp); |
|
6124 |
releaseq(q); |
|
6125 |
} |
|
6126 |
mutex_enter(SQLOCK(outer)); |
|
6127 |
} |
|
6128 |
ASSERT(MUTEX_HELD(SQLOCK(outer))); |
|
6129 |
} |
|
6130 |
||
6131 |
/* |
|
6132 |
* The list of messages on the inner syncq is effectively hashed |
|
6133 |
* by destination queue. These destination queues are doubly |
|
6134 |
* linked lists (hopefully) in priority order. Messages are then |
|
6135 |
* put on the queue referenced by the q_sqhead/q_sqtail elements. |
|
6136 |
* Additional messages are linked together by the b_next/b_prev |
|
6137 |
* elements in the mblk, with (similar to putq()) the first message |
|
6138 |
* having a NULL b_prev and the last message having a NULL b_next. |
|
6139 |
* |
|
6140 |
* Events, such as qwriter callbacks, are put onto a list in FIFO |
|
6141 |
* order referenced by sq_evhead, and sq_evtail. This is a singly |
|
6142 |
* linked list, and messages here MUST be processed in the order queued. |
|
6143 |
*/ |
|
6144 |
||
6145 |
/* |
|
6146 |
* Run the events on the syncq event list (sq_evhead). |
|
6147 |
* Assumes there is only one claim on the syncq, it is |
|
6148 |
* already exclusive (SQ_EXCL set), and the SQLOCK held. |
|
6149 |
* Messages here are processed in order, with the SQ_EXCL bit |
|
6150 |
* held all the way through till the last message is processed. |
|
6151 |
*/ |
|
6152 |
void |
|
6153 |
sq_run_events(syncq_t *sq) |
|
6154 |
{ |
|
6155 |
mblk_t *bp; |
|
6156 |
queue_t *qp; |
|
6157 |
uint16_t flags = sq->sq_flags; |
|
6158 |
void (*func)(); |
|
6159 |
||
6160 |
ASSERT(MUTEX_HELD(SQLOCK(sq))); |
|
6161 |
ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && |
|
6162 |
sq->sq_oprev == NULL) || |
|
6163 |
(sq->sq_outer != NULL && sq->sq_onext != NULL && |
|
6164 |
sq->sq_oprev != NULL)); |
|
6165 |
||
6166 |
ASSERT(flags & SQ_EXCL); |
|
6167 |
ASSERT(sq->sq_count == 1); |
|
6168 |
||
6169 |
/* |
|
6170 |
* We need to process all of the events on this list. It |
|
6171 |
* is possible that new events will be added while we are |
|
6172 |
* away processing a callback, so on every loop, we start |
|
6173 |
* back at the beginning of the list. |
|
6174 |
*/ |
|
6175 |
/* |
|
6176 |
* We have to reaccess sq_evhead since there is a |
|
6177 |
* possibility of a new entry while we were running |
|
6178 |
* the callback. |
|
6179 |
*/ |
|
6180 |
for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) { |
|
6181 |
ASSERT(bp->b_queue->q_syncq == sq); |
|
6182 |
ASSERT(sq->sq_flags & SQ_EVENTS); |
|
6183 |
||
6184 |
qp = bp->b_queue; |
|
6185 |
func = (void (*)())bp->b_prev; |
|
6186 |
ASSERT(func != NULL); |
|
6187 |
||
6188 |
/* |
|
6189 |
* Messages from the event queue must be taken off in |
|
6190 |
* FIFO order. |
|
6191 |
*/ |
|
6192 |
ASSERT(sq->sq_evhead == bp); |
|
6193 |
sq->sq_evhead = bp->b_next; |
|
6194 |
||
6195 |
if (bp->b_next == NULL) { |
|
6196 |
/* Deleting last */ |
|
6197 |
ASSERT(sq->sq_evtail == bp); |
|
6198 |
sq->sq_evtail = NULL; |
|
6199 |
sq->sq_flags &= ~SQ_EVENTS; |
|
6200 |
} |
|
6201 |
bp->b_prev = bp->b_next = NULL; |
|
6202 |
ASSERT(bp->b_datap->db_ref != 0); |
|
6203 |
||
6204 |
mutex_exit(SQLOCK(sq)); |
|
6205 |
||
6206 |
(*func)(qp, bp); |
|
6207 |
||
6208 |
mutex_enter(SQLOCK(sq)); |
|
6209 |
/* |
|
6210 |
* re-read the flags, since they could have changed. |
|
6211 |
*/ |
|
6212 |
flags = sq->sq_flags; |
|
6213 |
ASSERT(flags & SQ_EXCL); |
|
6214 |
} |
|
6215 |
ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL); |
|
6216 |
ASSERT(!(sq->sq_flags & SQ_EVENTS)); |
|
6217 |
||
6218 |
if (flags & SQ_WANTWAKEUP) { |
|
6219 |
flags &= ~SQ_WANTWAKEUP; |
|
6220 |
cv_broadcast(&sq->sq_wait); |
|
6221 |
} |
|
6222 |
if (flags & SQ_WANTEXWAKEUP) { |
|
6223 |
flags &= ~SQ_WANTEXWAKEUP; |
|
6224 |
cv_broadcast(&sq->sq_exitwait); |
|
6225 |
} |
|
6226 |
sq->sq_flags = flags; |
|
6227 |
} |
|
6228 |
||
6229 |
/* |
|
6230 |
* Put messages on the event list. |
|
6231 |
* If we can go exclusive now, do so and process the event list, otherwise |
|
6232 |
* let the last claim service this list (or wake the sqthread). |
|
6233 |
* This procedure assumes SQLOCK is held. To run the event list, it |
|
6234 |
* must be called with no claims. |
|
6235 |
*/ |
|
6236 |
static void |
|
6237 |
sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)()) |
|
6238 |
{ |
|
6239 |
uint16_t count; |
|
6240 |
||
6241 |
ASSERT(MUTEX_HELD(SQLOCK(sq))); |
|
6242 |
ASSERT(func != NULL); |
|
6243 |
||
6244 |
/* |
|
6245 |
* This is a callback. Add it to the list of callbacks |
|
6246 |
* and see about upgrading. |
|
6247 |
*/ |
|
6248 |
mp->b_prev = (mblk_t *)func; |
|
6249 |
mp->b_queue = q; |
|
6250 |
mp->b_next = NULL; |
|
6251 |
if (sq->sq_evhead == NULL) { |
|
6252 |
sq->sq_evhead = sq->sq_evtail = mp; |
|
6253 |
sq->sq_flags |= SQ_EVENTS; |
|
6254 |
} else { |
|
6255 |
ASSERT(sq->sq_evtail != NULL); |
|
6256 |
ASSERT(sq->sq_evtail->b_next == NULL); |
|
6257 |
ASSERT(sq->sq_flags & SQ_EVENTS); |
|
6258 |
sq->sq_evtail->b_next = mp; |
|
6259 |
sq->sq_evtail = mp; |
|
6260 |
} |
|
6261 |
/* |
|
6262 |
* We have set SQ_EVENTS, so threads will have to |
|
6263 |
* unwind out of the perimiter, and new entries will |
|
6264 |
* not grab a putlock. But we still need to know |
|
6265 |
* how many threads have already made a claim to the |
|
6266 |
* syncq, so grab the putlocks, and sum the counts. |
|
6267 |
* If there are no claims on the syncq, we can upgrade |
|
6268 |
* to exclusive, and run the event list. |
|
6269 |
* NOTE: We hold the SQLOCK, so we can just grab the |
|
6270 |
* putlocks. |
|
6271 |
*/ |
|
6272 |
count = sq->sq_count; |
|
6273 |
SQ_PUTLOCKS_ENTER(sq); |
|
6274 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
6275 |
/* |
|
6276 |
* We have no claim, so we need to check if there |
|
6277 |
* are no others, then we can upgrade. |
|
6278 |
*/ |
|
6279 |
/* |
|
6280 |
* There are currently no claims on |
|
6281 |
* the syncq by this thread (at least on this entry). The thread who has |
|
6282 |
* the claim should drain syncq. |
|
6283 |
*/ |
|
6284 |
if (count > 0) { |
|
6285 |
/* |
|
6286 |
* Can't upgrade - other threads inside. |
|
6287 |
*/ |
|
6288 |
SQ_PUTLOCKS_EXIT(sq); |
|
6289 |
mutex_exit(SQLOCK(sq)); |
|
6290 |
return; |
|
6291 |
} |
|
6292 |
/* |
|
6293 |
* Need to set SQ_EXCL and make a claim on the syncq. |
|
6294 |
*/ |
|
6295 |
ASSERT((sq->sq_flags & SQ_EXCL) == 0); |
|
6296 |
sq->sq_flags |= SQ_EXCL; |
|
6297 |
ASSERT(sq->sq_count == 0); |
|
6298 |
sq->sq_count++; |
|
6299 |
SQ_PUTLOCKS_EXIT(sq); |
|
6300 |
||
6301 |
/* Process the events list */ |
|
6302 |
sq_run_events(sq); |
|
6303 |
||
6304 |
/* |
|
6305 |
* Release our claim... |
|
6306 |
*/ |
|
6307 |
sq->sq_count--; |
|
6308 |
||
6309 |
/* |
|
6310 |
* And release SQ_EXCL. |
|
6311 |
* We don't need to acquire the putlocks to release |
|
6312 |
* SQ_EXCL, since we are exclusive, and hold the SQLOCK. |
|
6313 |
*/ |
|
6314 |
sq->sq_flags &= ~SQ_EXCL; |
|
6315 |
||
6316 |
/* |
|
6317 |
* sq_run_events should have released SQ_EXCL |
|
6318 |
*/ |
|
6319 |
ASSERT(!(sq->sq_flags & SQ_EXCL)); |
|
6320 |
||
6321 |
/* |
|
6322 |
* If anything happened while we were running the |
|
6323 |
* events (or was there before), we need to process |
|
6324 |
* them now. We shouldn't be exclusive sine we |
|
6325 |
* released the perimiter above (plus, we asserted |
|
6326 |
* for it). |
|
6327 |
*/ |
|
6328 |
if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED)) |
|
6329 |
drain_syncq(sq); |
|
6330 |
else |
|
6331 |
mutex_exit(SQLOCK(sq)); |
|
6332 |
} |
|
6333 |
||
6334 |
/* |
|
6335 |
* Perform delayed processing. The caller has to make sure that it is safe |
|
6336 |
* to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are |
|
6337 |
* set.) |
|
6338 |
* |
|
6339 |
* Assume that the caller has NO claims on the syncq. However, a claim |
|
6340 |
* on the syncq does not indicate that a thread is draining the syncq. |
|
6341 |
* There may be more claims on the syncq than there are threads draining |
|
6342 |
* (i.e. #_threads_draining <= sq_count) |
|
6343 |
* |
|
6344 |
* drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set |
|
6345 |
* in order to preserve qwriter(OUTER) ordering constraints. |
|
6346 |
* |
|
6347 |
* sq_putcount only needs to be checked when dispatching the queued |
|
6348 |
* writer call for CIPUT sync queue, but this is handled in sq_run_events. |
|
6349 |
*/ |
|
6350 |
void |
|
6351 |
drain_syncq(syncq_t *sq) |
|
6352 |
{ |
|
6353 |
queue_t *qp; |
|
6354 |
uint16_t count; |
|
6355 |
uint16_t type = sq->sq_type; |
|
6356 |
uint16_t flags = sq->sq_flags; |
|
6357 |
boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE; |
|
6358 |
||
6359 |
TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, |
|
6360 |
"drain_syncq start:%p", sq); |
|
6361 |
ASSERT(MUTEX_HELD(SQLOCK(sq))); |
|
6362 |
ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && |
|
6363 |
sq->sq_oprev == NULL) || |
|
6364 |
(sq->sq_outer != NULL && sq->sq_onext != NULL && |
|
6365 |
sq->sq_oprev != NULL)); |
|
6366 |
||
6367 |
/* |
|
6368 |
* Drop SQ_SERVICE flag. |
|
6369 |
*/ |
|
6370 |
if (bg_service) |
|
6371 |
sq->sq_svcflags &= ~SQ_SERVICE; |
|
6372 |
||
6373 |
/* |
|
6374 |
* If SQ_EXCL is set, someone else is processing this syncq - let him |
|
6375 |
* finish the job. |
|
6376 |
*/ |
|
6377 |
if (flags & SQ_EXCL) { |
|
6378 |
if (bg_service) { |
|
6379 |
ASSERT(sq->sq_servcount != 0); |
|
6380 |
sq->sq_servcount--; |
|
6381 |
} |
|
6382 |
mutex_exit(SQLOCK(sq)); |
|
6383 |
return; |
|
6384 |
} |
|
6385 |
||
6386 |
/* |
|
6387 |
* This routine can be called by a background thread if |
|
6388 |
* it was scheduled by a hi-priority thread. SO, if there are |
|
6389 |
* NOT messages queued, return (remember, we have the SQLOCK, |
|
6390 |
* and it cannot change until we release it). Wakeup any waiters also. |
|
6391 |
*/ |
|
6392 |
if (!(flags & SQ_QUEUED)) { |
|
6393 |
if (flags & SQ_WANTWAKEUP) { |
|
6394 |
flags &= ~SQ_WANTWAKEUP; |
|
6395 |
cv_broadcast(&sq->sq_wait); |
|
6396 |
} |
|
6397 |
if (flags & SQ_WANTEXWAKEUP) { |
|
6398 |
flags &= ~SQ_WANTEXWAKEUP; |
|
6399 |
cv_broadcast(&sq->sq_exitwait); |
|
6400 |
} |
|
6401 |
sq->sq_flags = flags; |
|
6402 |
if (bg_service) { |
|
6403 |
ASSERT(sq->sq_servcount != 0); |
|
6404 |
sq->sq_servcount--; |
|
6405 |
} |
|
6406 |
mutex_exit(SQLOCK(sq)); |
|
6407 |
return; |
|
6408 |
} |
|
6409 |
||
6410 |
/* |
|
6411 |
* If this is not a concurrent put perimiter, we need to |
|
6412 |
* become exclusive to drain. Also, if not CIPUT, we would |
|
6413 |
* not have acquired a putlock, so we don't need to check |
|
6414 |
* the putcounts. If not entering with a claim, we test |
|
6415 |
* for sq_count == 0. |
|
6416 |
*/ |
|
6417 |
type = sq->sq_type; |
|
6418 |
if (!(type & SQ_CIPUT)) { |
|
6419 |
if (sq->sq_count > 1) { |
|
6420 |
if (bg_service) { |
|
6421 |
ASSERT(sq->sq_servcount != 0); |
|
6422 |
sq->sq_servcount--; |
|
6423 |
} |
|
6424 |
mutex_exit(SQLOCK(sq)); |
|
6425 |
return; |
|
6426 |
} |
|
6427 |
sq->sq_flags |= SQ_EXCL; |
|
6428 |
} |
|
6429 |
||
6430 |
/* |
|
6431 |
* This is where we make a claim to the syncq. |
|
6432 |
* This can either be done by incrementing a putlock, or |
|
6433 |
* the sq_count. But since we already have the SQLOCK |
|
6434 |
* here, we just bump the sq_count. |
|
6435 |
* |
|
6436 |
* Note that after we make a claim, we need to let the code |
|
6437 |
* fall through to the end of this routine to clean itself |
|
6438 |
* up. A return in the while loop will put the syncq in a |
|
6439 |
* very bad state. |
|
6440 |
*/ |
|
6441 |
sq->sq_count++; |
|
6442 |
ASSERT(sq->sq_count != 0); /* wraparound */ |
|
6443 |
||
6444 |
while ((flags = sq->sq_flags) & SQ_QUEUED) { |
|
6445 |
/* |
|
6446 |
* If we are told to stayaway or went exclusive, |
|
6447 |
* we are done. |
|
6448 |
*/ |
|
6449 |
if (flags & (SQ_STAYAWAY)) { |
|
6450 |
break; |
|
6451 |
} |
|
6452 |
||
6453 |
/* |
|
6454 |
* If there are events to run, do so. |
|
6455 |
* We have one claim to the syncq, so if there are |
|
6456 |
* more than one, other threads are running. |
|
6457 |
*/ |
|
6458 |
if (sq->sq_evhead != NULL) { |
|
6459 |
ASSERT(sq->sq_flags & SQ_EVENTS); |
|
6460 |
||
6461 |
count = sq->sq_count; |
|
6462 |
SQ_PUTLOCKS_ENTER(sq); |
|
6463 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
6464 |
if (count > 1) { |
|
6465 |
SQ_PUTLOCKS_EXIT(sq); |
|
6466 |
/* Can't upgrade - other threads inside */ |
|
6467 |
break; |
|
6468 |
} |
|
6469 |
ASSERT((flags & SQ_EXCL) == 0); |
|
6470 |
sq->sq_flags = flags | SQ_EXCL; |
|
6471 |
SQ_PUTLOCKS_EXIT(sq); |
|
6472 |
/* |
|
6473 |
* we have the only claim, run the events, |
|
6474 |
* sq_run_events will clear the SQ_EXCL flag. |
|
6475 |
*/ |
|
6476 |
sq_run_events(sq); |
|
6477 |
||
6478 |
/* |
|
6479 |
* If this is a CIPUT perimiter, we need |
|
6480 |
* to drop the SQ_EXCL flag so we can properly |
|
6481 |
* continue draining the syncq. |
|
6482 |
*/ |
|
6483 |
if (type & SQ_CIPUT) { |
|
6484 |
ASSERT(sq->sq_flags & SQ_EXCL); |
|
6485 |
sq->sq_flags &= ~SQ_EXCL; |
|
6486 |
} |
|
6487 |
||
6488 |
/* |
|
6489 |
* And go back to the beginning just in case |
|
6490 |
* anything changed while we were away. |
|
6491 |
*/ |
|
6492 |
ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT)); |
|
6493 |
continue; |
|
6494 |
} |
|
6495 |
||
6496 |
ASSERT(sq->sq_evhead == NULL); |
|
6497 |
ASSERT(!(sq->sq_flags & SQ_EVENTS)); |
|
6498 |
||
6499 |
/* |
|
6500 |
* Find the queue that is not draining. |
|
6501 |
* |
|
6502 |
* q_draining is protected by QLOCK which we do not hold. |
|
6503 |
* But if it was set, then a thread was draining, and if it gets |
|
6504 |
* cleared, then it was because the thread has successfully |
|
6505 |
* drained the syncq, or a GOAWAY state occured. For the GOAWAY |
|
6506 |
* state to happen, a thread needs the SQLOCK which we hold, and |
|
6507 |
* if there was such a flag, we whould have already seen it. |
|
6508 |
*/ |
|
6509 |
||
6510 |
for (qp = sq->sq_head; |
|
6511 |
qp != NULL && (qp->q_draining || |
|
6512 |
(qp->q_sqflags & Q_SQDRAINING)); |
|
6513 |
qp = qp->q_sqnext) |
|
6514 |
; |
|
6515 |
||
6516 |
if (qp == NULL) |
|
6517 |
break; |
|
6518 |
||
6519 |
/* |
|
6520 |
* We have a queue to work on, and we hold the |
|
6521 |
* SQLOCK and one claim, call qdrain_syncq. |
|
6522 |
* This means we need to release the SQLOCK and |
|
6523 |
* aquire the QLOCK (OK since we have a claim). |
|
6524 |
* Note that qdrain_syncq will actually dequeue |
|
6525 |
* this queue from the sq_head list when it is |
|
6526 |
* convinced all the work is done and release |
|
6527 |
* the QLOCK before returning. |
|
6528 |
*/ |
|
6529 |
qp->q_sqflags |= Q_SQDRAINING; |
|
6530 |
mutex_exit(SQLOCK(sq)); |
|
6531 |
mutex_enter(QLOCK(qp)); |
|
6532 |
qdrain_syncq(sq, qp); |
|
6533 |
mutex_enter(SQLOCK(sq)); |
|
6534 |
||
6535 |
/* The queue is drained */ |
|
6536 |
ASSERT(qp->q_sqflags & Q_SQDRAINING); |
|
6537 |
qp->q_sqflags &= ~Q_SQDRAINING; |
|
6538 |
/* |
|
6539 |
* NOTE: After this point qp should not be used since it may be |
|
6540 |
* closed. |
|
6541 |
*/ |
|
6542 |
} |
|
6543 |
||
6544 |
ASSERT(MUTEX_HELD(SQLOCK(sq))); |
|
6545 |
flags = sq->sq_flags; |
|
6546 |
||
6547 |
/* |
|
6548 |
* sq->sq_head cannot change because we hold the |
|
6549 |
* sqlock. However, a thread CAN decide that it is no longer |
|
6550 |
* going to drain that queue. However, this should be due to |
|
6551 |
* a GOAWAY state, and we should see that here. |
|
6552 |
* |
|
6553 |
* This loop is not very efficient. One solution may be adding a second |
|
6554 |
* pointer to the "draining" queue, but it is difficult to do when |
|
6555 |
* queues are inserted in the middle due to priority ordering. Another |
|
6556 |
* possibility is to yank the queue out of the sq list and put it onto |
|
6557 |
* the "draining list" and then put it back if it can't be drained. |
|
6558 |
*/ |
|
6559 |
||
6560 |
ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) || |
|
6561 |
(type & SQ_CI) || sq->sq_head->q_draining); |
|
6562 |
||
6563 |
/* Drop SQ_EXCL for non-CIPUT perimiters */ |
|
6564 |
if (!(type & SQ_CIPUT)) |
|
6565 |
flags &= ~SQ_EXCL; |
|
6566 |
ASSERT((flags & SQ_EXCL) == 0); |
|
6567 |
||
6568 |
/* Wake up any waiters. */ |
|
6569 |
if (flags & SQ_WANTWAKEUP) { |
|
6570 |
flags &= ~SQ_WANTWAKEUP; |
|
6571 |
cv_broadcast(&sq->sq_wait); |
|
6572 |
} |
|
6573 |
if (flags & SQ_WANTEXWAKEUP) { |
|
6574 |
flags &= ~SQ_WANTEXWAKEUP; |
|
6575 |
cv_broadcast(&sq->sq_exitwait); |
|
6576 |
} |
|
6577 |
sq->sq_flags = flags; |
|
6578 |
||
6579 |
ASSERT(sq->sq_count != 0); |
|
6580 |
/* Release our claim. */ |
|
6581 |
sq->sq_count--; |
|
6582 |
||
6583 |
if (bg_service) { |
|
6584 |
ASSERT(sq->sq_servcount != 0); |
|
6585 |
sq->sq_servcount--; |
|
6586 |
} |
|
6587 |
||
6588 |
mutex_exit(SQLOCK(sq)); |
|
6589 |
||
6590 |
TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, |
|
6591 |
"drain_syncq end:%p", sq); |
|
6592 |
} |
|
6593 |
||
6594 |
||
6595 |
/* |
|
6596 |
* |
|
6597 |
* qdrain_syncq can be called (currently) from only one of two places: |
|
6598 |
* drain_syncq |
|
6599 |
* putnext (or some variation of it). |
|
6600 |
* and eventually |
|
6601 |
* qwait(_sig) |
|
6602 |
* |
|
6603 |
* If called from drain_syncq, we found it in the list |
|
6604 |
* of queue's needing service, so there is work to be done (or it |
|
6605 |
* wouldn't be on the list). |
|
6606 |
* |
|
6607 |
* If called from some putnext variation, it was because the |
|
6608 |
* perimiter is open, but messages are blocking a putnext and |
|
6609 |
* there is not a thread working on it. Now a thread could start |
|
6610 |
* working on it while we are getting ready to do so ourself, but |
|
6611 |
* the thread would set the q_draining flag, and we can spin out. |
|
6612 |
* |
|
6613 |
* As for qwait(_sig), I think I shall let it continue to call |
|
6614 |
* drain_syncq directly (after all, it will get here eventually). |
|
6615 |
* |
|
6616 |
* qdrain_syncq has to terminate when: |
|
6617 |
* - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering |
|
6618 |
* - SQ_EVENTS gets set to preserve qwriter(INNER) ordering |
|
6619 |
* |
|
6620 |
* ASSUMES: |
|
6621 |
* One claim |
|
6622 |
* QLOCK held |
|
6623 |
* SQLOCK not held |
|
6624 |
* Will release QLOCK before returning |
|
6625 |
*/ |
|
6626 |
void |
|
6627 |
qdrain_syncq(syncq_t *sq, queue_t *q) |
|
6628 |
{ |
|
6629 |
mblk_t *bp; |
|
6630 |
boolean_t do_clr; |
|
6631 |
#ifdef DEBUG |
|
6632 |
uint16_t count; |
|
6633 |
#endif |
|
6634 |
||
6635 |
TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, |
|
6636 |
"drain_syncq start:%p", sq); |
|
6637 |
ASSERT(q->q_syncq == sq); |
|
6638 |
ASSERT(MUTEX_HELD(QLOCK(q))); |
|
6639 |
ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); |
|
6640 |
/* |
|
6641 |
* For non-CIPUT perimiters, we should be called with the |
|
6642 |
* exclusive bit set already. For non-CIPUT perimiters we |
|
6643 |
* will be doing a concurrent drain, so it better not be set. |
|
6644 |
*/ |
|
6645 |
ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT))); |
|
6646 |
ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL))); |
|
6647 |
ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL)); |
|
6648 |
/* |
|
6649 |
* All outer pointers are set, or none of them are |
|
6650 |
*/ |
|
6651 |
ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && |
|
6652 |
sq->sq_oprev == NULL) || |
|
6653 |
(sq->sq_outer != NULL && sq->sq_onext != NULL && |
|
6654 |
sq->sq_oprev != NULL)); |
|
6655 |
#ifdef DEBUG |
|
6656 |
count = sq->sq_count; |
|
6657 |
/* |
|
6658 |
* This is OK without the putlocks, because we have one |
|
6659 |
* claim either from the sq_count, or a putcount. We could |
|
6660 |
* get an erroneous value from other counts, but ours won't |
|
6661 |
* change, so one way or another, we will have at least a |
|
6662 |
* value of one. |
|
6663 |
*/ |
|
6664 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
6665 |
ASSERT(count >= 1); |
|
6666 |
#endif /* DEBUG */ |
|
6667 |
||
6668 |
/* |
|
6669 |
* The first thing to do here, is find out if a thread is already |
|
6670 |
* draining this queue or the queue is closing. If so, we are done, |
|
6671 |
* just return. Also, if there are no messages, we are done as well. |
|
6672 |
* Note that we check the q_sqhead since there is s window of |
|
6673 |
* opportunity for us to enter here because Q_SQQUEUED was set, but is |
|
6674 |
* not anymore. |
|
6675 |
*/ |
|
6676 |
if (q->q_draining || (q->q_sqhead == NULL)) { |
|
6677 |
mutex_exit(QLOCK(q)); |
|
6678 |
return; |
|
6679 |
} |
|
6680 |
||
6681 |
/* |
|
6682 |
* If the perimiter is exclusive, there is nothing we can |
|
6683 |
* do right now, go away. |
|
6684 |
* Note that there is nothing to prevent this case from changing |
|
6685 |
* right after this check, but the spin-out will catch it. |
|
6686 |
*/ |
|
6687 |
||
6688 |
/* Tell other threads that we are draining this queue */ |
|
6689 |
q->q_draining = 1; /* Protected by QLOCK */ |
|
6690 |
||
6691 |
for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) { |
|
6692 |
||
6693 |
/* |
|
6694 |
* Because we can enter this routine just because |
|
6695 |
* a putnext is blocked, we need to spin out if |
|
6696 |
* the perimiter wants to go exclusive as well |
|
6697 |
* as just blocked. We need to spin out also if |
|
6698 |
* events are queued on the syncq. |
|
6699 |
* Don't check for SQ_EXCL, because non-CIPUT |
|
6700 |
* perimiters would set it, and it can't become |
|
6701 |
* exclusive while we hold a claim. |
|
6702 |
*/ |
|
6703 |
if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) { |
|
6704 |
break; |
|
6705 |
} |
|
6706 |
||
6707 |
#ifdef DEBUG |
|
6708 |
/* |
|
6709 |
* Since we are in qdrain_syncq, we already know the queue, |
|
6710 |
* but for sanity, we want to check this against the qp that |
|
6711 |
* was passed in by bp->b_queue. |
|
6712 |
*/ |
|
6713 |
||
6714 |
ASSERT(bp->b_queue == q); |
|
6715 |
ASSERT(bp->b_queue->q_syncq == sq); |
|
6716 |
bp->b_queue = NULL; |
|
6717 |
||
6718 |
/* |
|
6719 |
* We would have the following check in the DEBUG code: |
|
6720 |
* |
|
6721 |
* if (bp->b_prev != NULL) { |
|
6722 |
* ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp); |
|
6723 |
* } |
|
6724 |
* |
|
6725 |
* This can't be done, however, since IP modifies qinfo |
|
6726 |
* structure at run-time (switching between IPv4 qinfo and IPv6 |
|
6727 |
* qinfo), invalidating the check. |
|
6728 |
* So the assignment to func is left here, but the ASSERT itself |
|
6729 |
* is removed until the whole issue is resolved. |
|
6730 |
*/ |
|
6731 |
#endif |
|
6732 |
ASSERT(q->q_sqhead == bp); |
|
6733 |
q->q_sqhead = bp->b_next; |
|
6734 |
bp->b_prev = bp->b_next = NULL; |
|
6735 |
ASSERT(q->q_syncqmsgs > 0); |
|
6736 |
mutex_exit(QLOCK(q)); |
|
6737 |
||
6738 |
ASSERT(bp->b_datap->db_ref != 0); |
|
6739 |
||
6740 |
(void) (*q->q_qinfo->qi_putp)(q, bp); |
|
6741 |
||
6742 |
mutex_enter(QLOCK(q)); |
|
6743 |
/* |
|
6744 |
* We should decrement q_syncqmsgs only after executing the |
|
6745 |
* put procedure to avoid a possible race with putnext(). |
|
6746 |
* In putnext() though it sees Q_SQQUEUED is set, there is |
|
6747 |
* an optimization which allows putnext to call the put |
|
6748 |
* procedure directly if (q_syncqmsgs == 0) and thus |
|
6749 |
* a message reodering could otherwise occur. |
|
6750 |
*/ |
|
6751 |
q->q_syncqmsgs--; |
|
6752 |
||
6753 |
/* |
|
6754 |
* Clear QFULL in the next service procedure queue if |
|
6755 |
* this is the last message destined to that queue. |
|
6756 |
* |
|
6757 |
* It would make better sense to have some sort of |
|
6758 |
* tunable for the low water mark, but these symantics |
|
6759 |
* are not yet defined. So, alas, we use a constant. |
|
6760 |
*/ |
|
6761 |
do_clr = (q->q_syncqmsgs == 0); |
|
6762 |
mutex_exit(QLOCK(q)); |
|
6763 |
||
6764 |
if (do_clr) |
|
6765 |
clr_qfull(q); |
|
6766 |
||
6767 |
mutex_enter(QLOCK(q)); |
|
6768 |
/* |
|
6769 |
* Always clear SQ_EXCL when CIPUT in order to handle |
|
6770 |
* qwriter(INNER). |
|
6771 |
*/ |
|
6772 |
/* |
|
6773 |
* The putp() can call qwriter and get exclusive access |
|
6774 |
* IFF this is the only claim. So, we need to test for |
|
6775 |
* this possibility so we can aquire the mutex and clear |
|
6776 |
* the bit. |
|
6777 |
*/ |
|
6778 |
if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) { |
|
6779 |
mutex_enter(SQLOCK(sq)); |
|
6780 |
sq->sq_flags &= ~SQ_EXCL; |
|
6781 |
mutex_exit(SQLOCK(sq)); |
|
6782 |
} |
|
6783 |
} |
|
6784 |
||
6785 |
/* |
|
6786 |
* We should either have no queues on the syncq, or we were |
|
6787 |
* told to goaway by a waiter (which we will wake up at the |
|
6788 |
* end of this function). |
|
6789 |
*/ |
|
6790 |
ASSERT((q->q_sqhead == NULL) || |
|
6791 |
(sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS))); |
|
6792 |
||
6793 |
ASSERT(MUTEX_HELD(QLOCK(q))); |
|
6794 |
ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); |
|
6795 |
||
6796 |
/* |
|
6797 |
* Remove the q from the syncq list if all the messages are |
|
6798 |
* drained. |
|
6799 |
*/ |
|
6800 |
if (q->q_sqhead == NULL) { |
|
6801 |
mutex_enter(SQLOCK(sq)); |
|
6802 |
if (q->q_sqflags & Q_SQQUEUED) |
|
6803 |
SQRM_Q(sq, q); |
|
6804 |
mutex_exit(SQLOCK(sq)); |
|
6805 |
/* |
|
6806 |
* Since the queue is removed from the list, reset its priority. |
|
6807 |
*/ |
|
6808 |
q->q_spri = 0; |
|
6809 |
} |
|
6810 |
||
6811 |
/* |
|
6812 |
* Remember, the q_draining flag is used to let another |
|
6813 |
* thread know that there is a thread currently draining |
|
6814 |
* the messages for a queue. Since we are now done with |
|
6815 |
* this queue (even if there may be messages still there), |
|
6816 |
* we need to clear this flag so some thread will work |
|
6817 |
* on it if needed. |
|
6818 |
*/ |
|
6819 |
ASSERT(q->q_draining); |
|
6820 |
q->q_draining = 0; |
|
6821 |
||
6822 |
/* called with a claim, so OK to drop all locks. */ |
|
6823 |
mutex_exit(QLOCK(q)); |
|
6824 |
||
6825 |
TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, |
|
6826 |
"drain_syncq end:%p", sq); |
|
6827 |
} |
|
6828 |
/* END OF QDRAIN_SYNCQ */ |
|
6829 |
||
6830 |
||
6831 |
/* |
|
6832 |
* This is the mate to qdrain_syncq, except that it is putting the |
|
6833 |
* message onto the the queue instead draining. Since the |
|
6834 |
* message is destined for the queue that is selected, there is |
|
6835 |
* no need to identify the function because the message is |
|
6836 |
* intended for the put routine for the queue. But this |
|
6837 |
* routine will do it anyway just in case (but only for debug kernels). |
|
6838 |
* |
|
6839 |
* After the message is enqueued on the syncq, it calls putnext_tail() |
|
6840 |
* which will schedule a background thread to actually process the message. |
|
6841 |
* |
|
6842 |
* Assumes that there is a claim on the syncq (sq->sq_count > 0) and |
|
6843 |
* SQLOCK(sq) and QLOCK(q) are not held. |
|
6844 |
*/ |
|
6845 |
void |
|
6846 |
qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp) |
|
6847 |
{ |
|
6848 |
queue_t *fq = NULL; |
|
6849 |
||
6850 |
ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); |
|
6851 |
ASSERT(MUTEX_NOT_HELD(QLOCK(q))); |
|
6852 |
ASSERT(sq->sq_count > 0); |
|
6853 |
ASSERT(q->q_syncq == sq); |
|
6854 |
ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && |
|
6855 |
sq->sq_oprev == NULL) || |
|
6856 |
(sq->sq_outer != NULL && sq->sq_onext != NULL && |
|
6857 |
sq->sq_oprev != NULL)); |
|
6858 |
||
6859 |
mutex_enter(QLOCK(q)); |
|
6860 |
||
6861 |
/* |
|
6862 |
* Set QFULL in next service procedure queue (that cares) if not |
|
6863 |
* already set and if there are already more messages on the syncq |
|
6864 |
* than sq_max_size. If sq_max_size is 0, no flow control will be |
|
6865 |
* asserted on any syncq. |
|
6866 |
* |
|
6867 |
* The fq here is the next queue with a service procedure. |
|
6868 |
* This is where we would fail canputnext, so this is where we |
|
6869 |
* need to set QFULL. |
|
6870 |
* |
|
6871 |
* LOCKING HIERARCHY: In the case when fq != q we need to |
|
6872 |
* a) Take QLOCK(fq) to set QFULL flag and |
|
6873 |
* b) Take sd_reflock in the case of the hot stream to update |
|
6874 |
* sd_refcnt. |
|
6875 |
* We already have QLOCK at this point. To avoid cross-locks with |
|
6876 |
* freezestr() which grabs all QLOCKs and with strlock() which grabs |
|
6877 |
* both SQLOCK and sd_reflock, we need to drop respective locks first. |
|
6878 |
*/ |
|
6879 |
if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) && |
|
6880 |
(q->q_syncqmsgs > sq_max_size)) { |
|
6881 |
if ((fq = q->q_nfsrv) == q) { |
|
6882 |
fq->q_flag |= QFULL; |
|
6883 |
} else { |
|
6884 |
mutex_exit(QLOCK(q)); |
|
6885 |
mutex_enter(QLOCK(fq)); |
|
6886 |
fq->q_flag |= QFULL; |
|
6887 |
mutex_exit(QLOCK(fq)); |
|
6888 |
mutex_enter(QLOCK(q)); |
|
6889 |
} |
|
6890 |
} |
|
6891 |
||
6892 |
#ifdef DEBUG |
|
6893 |
/* |
|
6894 |
* This is used for debug in the qfill_syncq/qdrain_syncq case |
|
6895 |
* to trace the queue that the message is intended for. Note |
|
6896 |
* that the original use was to identify the queue and function |
|
6897 |
* to call on the drain. In the new syncq, we have the context |
|
6898 |
* of the queue that we are draining, so call it's putproc and |
|
6899 |
* don't rely on the saved values. But for debug this is still |
|
6900 |
* usefull information. |
|
6901 |
*/ |
|
6902 |
mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp; |
|
6903 |
mp->b_queue = q; |
|
6904 |
mp->b_next = NULL; |
|
6905 |
#endif |
|
6906 |
ASSERT(q->q_syncq == sq); |
|
6907 |
/* |
|
6908 |
* Enqueue the message on the list. |
|
6909 |
* SQPUT_MP() accesses q_syncqmsgs. We are already holding QLOCK to |
|
6910 |
* protect it. So its ok to acquire SQLOCK after SQPUT_MP(). |
|
6911 |
*/ |
|
6912 |
SQPUT_MP(q, mp); |
|
6913 |
mutex_enter(SQLOCK(sq)); |
|
6914 |
||
6915 |
/* |
|
6916 |
* And queue on syncq for scheduling, if not already queued. |
|
6917 |
* Note that we need the SQLOCK for this, and for testing flags |
|
6918 |
* at the end to see if we will drain. So grab it now, and |
|
6919 |
* release it before we call qdrain_syncq or return. |
|
6920 |
*/ |
|
6921 |
if (!(q->q_sqflags & Q_SQQUEUED)) { |
|
6922 |
q->q_spri = curthread->t_pri; |
|
6923 |
SQPUT_Q(sq, q); |
|
6924 |
} |
|
6925 |
#ifdef DEBUG |
|
6926 |
else { |
|
6927 |
/* |
|
6928 |
* All of these conditions MUST be true! |
|
6929 |
*/ |
|
6930 |
ASSERT(sq->sq_tail != NULL); |
|
6931 |
if (sq->sq_tail == sq->sq_head) { |
|
6932 |
ASSERT((q->q_sqprev == NULL) && |
|
6933 |
(q->q_sqnext == NULL)); |
|
6934 |
} else { |
|
6935 |
ASSERT((q->q_sqprev != NULL) || |
|
6936 |
(q->q_sqnext != NULL)); |
|
6937 |
} |
|
6938 |
ASSERT(sq->sq_flags & SQ_QUEUED); |
|
6939 |
ASSERT(q->q_syncqmsgs != 0); |
|
6940 |
ASSERT(q->q_sqflags & Q_SQQUEUED); |
|
6941 |
} |
|
6942 |
#endif |
|
6943 |
mutex_exit(QLOCK(q)); |
|
6944 |
/* |
|
6945 |
* SQLOCK is still held, so sq_count can be safely decremented. |
|
6946 |
*/ |
|
6947 |
sq->sq_count--; |
|
6948 |
||
6949 |
putnext_tail(sq, q, 0); |
|
6950 |
/* Should not reference sq or q after this point. */ |
|
6951 |
} |
|
6952 |
||
6953 |
/* End of qfill_syncq */ |
|
6954 |
||
6955 |
/* |
|
6956 |
* Remove all messages from a syncq (if qp is NULL) or remove all messages |
|
6957 |
* that would be put into qp by drain_syncq. |
|
6958 |
* Used when deleting the syncq (qp == NULL) or when detaching |
|
6959 |
* a queue (qp != NULL). |
|
6960 |
* Return non-zero if one or more messages were freed. |
|
6961 |
* |
|
6962 |
* no need to grab sq_putlocks here. See comment in strsubr.h that explains when |
|
6963 |
* sq_putlocks are used. |
|
6964 |
* |
|
6965 |
* NOTE: This function assumes that it is called from the close() context and |
|
6966 |
* that all the queues in the syncq are going aay. For this reason it doesn't |
|
6967 |
* acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is |
|
6968 |
* currently valid, but it is useful to rethink this function to behave properly |
|
6969 |
* in other cases. |
|
6970 |
*/ |
|
6971 |
int |
|
6972 |
flush_syncq(syncq_t *sq, queue_t *qp) |
|
6973 |
{ |
|
6974 |
mblk_t *bp, *mp_head, *mp_next, *mp_prev; |
|
6975 |
queue_t *q; |
|
6976 |
int ret = 0; |
|
6977 |
||
6978 |
mutex_enter(SQLOCK(sq)); |
|
6979 |
||
6980 |
/* |
|
6981 |
* Before we leave, we need to make sure there are no |
|
6982 |
* events listed for this queue. All events for this queue |
|
6983 |
* will just be freed. |
|
6984 |
*/ |
|
6985 |
if (qp != NULL && sq->sq_evhead != NULL) { |
|
6986 |
ASSERT(sq->sq_flags & SQ_EVENTS); |
|
6987 |
||
6988 |
mp_prev = NULL; |
|
6989 |
for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) { |
|
6990 |
mp_next = bp->b_next; |
|
6991 |
if (bp->b_queue == qp) { |
|
6992 |
/* Delete this message */ |
|
6993 |
if (mp_prev != NULL) { |
|
6994 |
mp_prev->b_next = mp_next; |
|
6995 |
/* |
|
6996 |
* Update sq_evtail if the last element |
|
6997 |
* is removed. |
|
6998 |
*/ |
|
6999 |
if (bp == sq->sq_evtail) { |
|
7000 |
ASSERT(mp_next == NULL); |
|
7001 |
sq->sq_evtail = mp_prev; |
|
7002 |
} |
|
7003 |
} else |
|
7004 |
sq->sq_evhead = mp_next; |
|
7005 |
if (sq->sq_evhead == NULL) |
|
7006 |
sq->sq_flags &= ~SQ_EVENTS; |
|
7007 |
bp->b_prev = bp->b_next = NULL; |
|
7008 |
freemsg(bp); |
|
7009 |
ret++; |
|
7010 |
} else { |
|
7011 |
mp_prev = bp; |
|
7012 |
} |
|
7013 |
} |
|
7014 |
} |
|
7015 |
||
7016 |
/* |
|
7017 |
* Walk sq_head and: |
|
7018 |
* - match qp if qp is set, remove it's messages |
|
7019 |
* - all if qp is not set |
|
7020 |
*/ |
|
7021 |
q = sq->sq_head; |
|
7022 |
while (q != NULL) { |
|
7023 |
ASSERT(q->q_syncq == sq); |
|
7024 |
if ((qp == NULL) || (qp == q)) { |
|
7025 |
/* |
|
7026 |
* Yank the messages as a list off the queue |
|
7027 |
*/ |
|
7028 |
mp_head = q->q_sqhead; |
|
7029 |
/* |
|
7030 |
* We do not have QLOCK(q) here (which is safe due to |
|
7031 |
* assumptions mentioned above). To obtain the lock we |
|
7032 |
* need to release SQLOCK which may allow lots of things |
|
7033 |
* to change upon us. This place requires more analysis. |
|
7034 |
*/ |
|
7035 |
q->q_sqhead = q->q_sqtail = NULL; |
|
7036 |
ASSERT(mp_head->b_queue && |
|
7037 |
mp_head->b_queue->q_syncq == sq); |
|
7038 |
||
7039 |
/* |
|
7040 |
* Free each of the messages. |
|
7041 |
*/ |
|
7042 |
for (bp = mp_head; bp != NULL; bp = mp_next) { |
|
7043 |
mp_next = bp->b_next; |
|
7044 |
bp->b_prev = bp->b_next = NULL; |
|
7045 |
freemsg(bp); |
|
7046 |
ret++; |
|
7047 |
} |
|
7048 |
/* |
|
7049 |
* Now remove the queue from the syncq. |
|
7050 |
*/ |
|
7051 |
ASSERT(q->q_sqflags & Q_SQQUEUED); |
|
7052 |
SQRM_Q(sq, q); |
|
7053 |
q->q_spri = 0; |
|
7054 |
q->q_syncqmsgs = 0; |
|
7055 |
||
7056 |
/* |
|
7057 |
* If qp was specified, we are done with it and are |
|
7058 |
* going to drop SQLOCK(sq) and return. We wakeup syncq |
|
7059 |
* waiters while we still have the SQLOCK. |
|
7060 |
*/ |
|
7061 |
if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) { |
|
7062 |
sq->sq_flags &= ~SQ_WANTWAKEUP; |
|
7063 |
cv_broadcast(&sq->sq_wait); |
|
7064 |
} |
|
7065 |
/* Drop SQLOCK across clr_qfull */ |
|
7066 |
mutex_exit(SQLOCK(sq)); |
|
7067 |
||
7068 |
/* |
|
7069 |
* We avoid doing the test that drain_syncq does and |
|
7070 |
* unconditionally clear qfull for every flushed |
|
7071 |
* message. Since flush_syncq is only called during |
|
7072 |
* close this should not be a problem. |
|
7073 |
*/ |
|
7074 |
clr_qfull(q); |
|
7075 |
if (qp != NULL) { |
|
7076 |
return (ret); |
|
7077 |
} else { |
|
7078 |
mutex_enter(SQLOCK(sq)); |
|
7079 |
/* |
|
7080 |
* The head was removed by SQRM_Q above. |
|
7081 |
* reread the new head and flush it. |
|
7082 |
*/ |
|
7083 |
q = sq->sq_head; |
|
7084 |
} |
|
7085 |
} else { |
|
7086 |
q = q->q_sqnext; |
|
7087 |
} |
|
7088 |
ASSERT(MUTEX_HELD(SQLOCK(sq))); |
|
7089 |
} |
|
7090 |
||
7091 |
if (sq->sq_flags & SQ_WANTWAKEUP) { |
|
7092 |
sq->sq_flags &= ~SQ_WANTWAKEUP; |
|
7093 |
cv_broadcast(&sq->sq_wait); |
|
7094 |
} |
|
7095 |
||
7096 |
mutex_exit(SQLOCK(sq)); |
|
7097 |
return (ret); |
|
7098 |
} |
|
7099 |
||
7100 |
/* |
|
7101 |
* Propagate all messages from a syncq to the next syncq that are associated |
|
7102 |
* with the specified queue. If the queue is attached to a driver or if the |
|
7103 |
* messages have been added due to a qwriter(PERIM_INNER), free the messages. |
|
7104 |
* |
|
7105 |
* Assumes that the stream is strlock()'ed. We don't come here if there |
|
7106 |
* are no messages to propagate. |
|
7107 |
* |
|
7108 |
* NOTE : If the queue is attached to a driver, all the messages are freed |
|
7109 |
* as there is no point in propagating the messages from the driver syncq |
|
7110 |
* to the closing stream head which will in turn get freed later. |
|
7111 |
*/ |
|
7112 |
static int |
|
7113 |
propagate_syncq(queue_t *qp) |
|
7114 |
{ |
|
7115 |
mblk_t *bp, *head, *tail, *prev, *next; |
|
7116 |
syncq_t *sq; |
|
7117 |
queue_t *nqp; |
|
7118 |
syncq_t *nsq; |
|
7119 |
boolean_t isdriver; |
|
7120 |
int moved = 0; |
|
7121 |
uint16_t flags; |
|
7122 |
pri_t priority = curthread->t_pri; |
|
7123 |
#ifdef DEBUG |
|
7124 |
void (*func)(); |
|
7125 |
#endif |
|
7126 |
||
7127 |
sq = qp->q_syncq; |
|
7128 |
ASSERT(MUTEX_HELD(SQLOCK(sq))); |
|
7129 |
/* debug macro */ |
|
7130 |
SQ_PUTLOCKS_HELD(sq); |
|
7131 |
/* |
|
7132 |
* As entersq() does not increment the sq_count for |
|
7133 |
* the write side, check sq_count for non-QPERQ |
|
7134 |
* perimeters alone. |
|
7135 |
*/ |
|
7136 |
ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1)); |
|
7137 |
||
7138 |
/* |
|
7139 |
* propagate_syncq() can be called because of either messages on the |
|
7140 |
* queue syncq or because on events on the queue syncq. Do actual |
|
7141 |
* message propagations if there are any messages. |
|
7142 |
*/ |
|
7143 |
if (qp->q_syncqmsgs) { |
|
7144 |
isdriver = (qp->q_flag & QISDRV); |
|
7145 |
||
7146 |
if (!isdriver) { |
|
7147 |
nqp = qp->q_next; |
|
7148 |
nsq = nqp->q_syncq; |
|
7149 |
ASSERT(MUTEX_HELD(SQLOCK(nsq))); |
|
7150 |
/* debug macro */ |
|
7151 |
SQ_PUTLOCKS_HELD(nsq); |
|
7152 |
#ifdef DEBUG |
|
7153 |
func = (void (*)())nqp->q_qinfo->qi_putp; |
|
7154 |
#endif |
|
7155 |
} |
|
7156 |
||
7157 |
SQRM_Q(sq, qp); |
|
7158 |
priority = MAX(qp->q_spri, priority); |
|
7159 |
qp->q_spri = 0; |
|
7160 |
head = qp->q_sqhead; |
|
7161 |
tail = qp->q_sqtail; |
|
7162 |
qp->q_sqhead = qp->q_sqtail = NULL; |
|
7163 |
qp->q_syncqmsgs = 0; |
|
7164 |
||
7165 |
/* |
|
7166 |
* Walk the list of messages, and free them if this is a driver, |
|
7167 |
* otherwise reset the b_prev and b_queue value to the new putp. |
|
7168 |
* Afterward, we will just add the head to the end of the next |
|
7169 |
* syncq, and point the tail to the end of this one. |
|
7170 |
*/ |
|
7171 |
||
7172 |
for (bp = head; bp != NULL; bp = next) { |
|
7173 |
next = bp->b_next; |
|
7174 |
if (isdriver) { |
|
7175 |
bp->b_prev = bp->b_next = NULL; |
|
7176 |
freemsg(bp); |
|
7177 |
continue; |
|
7178 |
} |
|
7179 |
/* Change the q values for this message */ |
|
7180 |
bp->b_queue = nqp; |
|
7181 |
#ifdef DEBUG |
|
7182 |
bp->b_prev = (mblk_t *)func; |
|
7183 |
#endif |
|
7184 |
moved++; |
|
7185 |
} |
|
7186 |
/* |
|
7187 |
* Attach list of messages to the end of the new queue (if there |
|
7188 |
* is a list of messages). |
|
7189 |
*/ |
|
7190 |
||
7191 |
if (!isdriver && head != NULL) { |
|
7192 |
ASSERT(tail != NULL); |
|
7193 |
if (nqp->q_sqhead == NULL) { |
|
7194 |
nqp->q_sqhead = head; |
|
7195 |
} else { |
|
7196 |
ASSERT(nqp->q_sqtail != NULL); |
|
7197 |
nqp->q_sqtail->b_next = head; |
|
7198 |
} |
|
7199 |
nqp->q_sqtail = tail; |
|
7200 |
/* |
|
7201 |
* When messages are moved from high priority queue to |
|
7202 |
* another queue, the destination queue priority is |
|
7203 |
* upgraded. |
|
7204 |
*/ |
|
7205 |
||
7206 |
if (priority > nqp->q_spri) |
|
7207 |
nqp->q_spri = priority; |
|
7208 |
||
7209 |
SQPUT_Q(nsq, nqp); |
|
7210 |
||
7211 |
nqp->q_syncqmsgs += moved; |
|
7212 |
ASSERT(nqp->q_syncqmsgs != 0); |
|
7213 |
} |
|
7214 |
} |
|
7215 |
||
7216 |
/* |
|
7217 |
* Before we leave, we need to make sure there are no |
|
7218 |
* events listed for this queue. All events for this queue |
|
7219 |
* will just be freed. |
|
7220 |
*/ |
|
7221 |
if (sq->sq_evhead != NULL) { |
|
7222 |
ASSERT(sq->sq_flags & SQ_EVENTS); |
|
7223 |
prev = NULL; |
|
7224 |
for (bp = sq->sq_evhead; bp != NULL; bp = next) { |
|
7225 |
next = bp->b_next; |
|
7226 |
if (bp->b_queue == qp) { |
|
7227 |
/* Delete this message */ |
|
7228 |
if (prev != NULL) { |
|
7229 |
prev->b_next = next; |
|
7230 |
/* |
|
7231 |
* Update sq_evtail if the last element |
|
7232 |
* is removed. |
|
7233 |
*/ |
|
7234 |
if (bp == sq->sq_evtail) { |
|
7235 |
ASSERT(next == NULL); |
|
7236 |
sq->sq_evtail = prev; |
|
7237 |
} |
|
7238 |
} else |
|
7239 |
sq->sq_evhead = next; |
|
7240 |
if (sq->sq_evhead == NULL) |
|
7241 |
sq->sq_flags &= ~SQ_EVENTS; |
|
7242 |
bp->b_prev = bp->b_next = NULL; |
|
7243 |
freemsg(bp); |
|
7244 |
} else { |
|
7245 |
prev = bp; |
|
7246 |
} |
|
7247 |
} |
|
7248 |
} |
|
7249 |
||
7250 |
flags = sq->sq_flags; |
|
7251 |
||
7252 |
/* Wake up any waiter before leaving. */ |
|
7253 |
if (flags & SQ_WANTWAKEUP) { |
|
7254 |
flags &= ~SQ_WANTWAKEUP; |
|
7255 |
cv_broadcast(&sq->sq_wait); |
|
7256 |
} |
|
7257 |
sq->sq_flags = flags; |
|
7258 |
||
7259 |
return (moved); |
|
7260 |
} |
|
7261 |
||
7262 |
/* |
|
7263 |
* Try and upgrade to exclusive access at the inner perimeter. If this can |
|
7264 |
* not be done without blocking then request will be queued on the syncq |
|
7265 |
* and drain_syncq will run it later. |
|
7266 |
* |
|
7267 |
* This routine can only be called from put or service procedures plus |
|
7268 |
* asynchronous callback routines that have properly entered to |
|
7269 |
* queue (with entersq.) Thus qwriter_inner assumes the caller has one claim |
|
7270 |
* on the syncq associated with q. |
|
7271 |
*/ |
|
7272 |
void |
|
7273 |
qwriter_inner(queue_t *q, mblk_t *mp, void (*func)()) |
|
7274 |
{ |
|
7275 |
syncq_t *sq = q->q_syncq; |
|
7276 |
uint16_t count; |
|
7277 |
||
7278 |
mutex_enter(SQLOCK(sq)); |
|
7279 |
count = sq->sq_count; |
|
7280 |
SQ_PUTLOCKS_ENTER(sq); |
|
7281 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
7282 |
ASSERT(count >= 1); |
|
7283 |
ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC)); |
|
7284 |
||
7285 |
if (count == 1) { |
|
7286 |
/* |
|
7287 |
* Can upgrade. This case also handles nested qwriter calls |
|
7288 |
* (when the qwriter callback function calls qwriter). In that |
|
7289 |
* case SQ_EXCL is already set. |
|
7290 |
*/ |
|
7291 |
sq->sq_flags |= SQ_EXCL; |
|
7292 |
SQ_PUTLOCKS_EXIT(sq); |
|
7293 |
mutex_exit(SQLOCK(sq)); |
|
7294 |
(*func)(q, mp); |
|
7295 |
/* |
|
7296 |
* Assumes that leavesq, putnext, and drain_syncq will reset |
|
7297 |
* SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on |
|
7298 |
* until putnext, leavesq, or drain_syncq drops it. |
|
7299 |
* That way we handle nested qwriter(INNER) without dropping |
|
7300 |
* SQ_EXCL until the outermost qwriter callback routine is |
|
7301 |
* done. |
|
7302 |
*/ |
|
7303 |
return; |
|
7304 |
} |
|
7305 |
SQ_PUTLOCKS_EXIT(sq); |
|
7306 |
sqfill_events(sq, q, mp, func); |
|
7307 |
} |
|
7308 |
||
7309 |
/* |
|
7310 |
* Synchronous callback support functions |
|
7311 |
*/ |
|
7312 |
||
7313 |
/* |
|
7314 |
* Allocate a callback parameter structure. |
|
7315 |
* Assumes that caller initializes the flags and the id. |
|
7316 |
* Acquires SQLOCK(sq) if non-NULL is returned. |
|
7317 |
*/ |
|
7318 |
callbparams_t * |
|
7319 |
callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags) |
|
7320 |
{ |
|
7321 |
callbparams_t *cbp; |
|
7322 |
size_t size = sizeof (callbparams_t); |
|
7323 |
||
7324 |
cbp = kmem_alloc(size, kmflags & ~KM_PANIC); |
|
7325 |
||
7326 |
/* |
|
7327 |
* Only try tryhard allocation if the caller is ready to panic. |
|
7328 |
* Otherwise just fail. |
|
7329 |
*/ |
|
7330 |
if (cbp == NULL) { |
|
7331 |
if (kmflags & KM_PANIC) |
|
7332 |
cbp = kmem_alloc_tryhard(sizeof (callbparams_t), |
|
7333 |
&size, kmflags); |
|
7334 |
else |
|
7335 |
return (NULL); |
|
7336 |
} |
|
7337 |
||
7338 |
ASSERT(size >= sizeof (callbparams_t)); |
|
7339 |
cbp->cbp_size = size; |
|
7340 |
cbp->cbp_sq = sq; |
|
7341 |
cbp->cbp_func = func; |
|
7342 |
cbp->cbp_arg = arg; |
|
7343 |
mutex_enter(SQLOCK(sq)); |
|
7344 |
cbp->cbp_next = sq->sq_callbpend; |
|
7345 |
sq->sq_callbpend = cbp; |
|
7346 |
return (cbp); |
|
7347 |
} |
|
7348 |
||
7349 |
void |
|
7350 |
callbparams_free(syncq_t *sq, callbparams_t *cbp) |
|
7351 |
{ |
|
7352 |
callbparams_t **pp, *p; |
|
7353 |
||
7354 |
ASSERT(MUTEX_HELD(SQLOCK(sq))); |
|
7355 |
||
7356 |
for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { |
|
7357 |
if (p == cbp) { |
|
7358 |
*pp = p->cbp_next; |
|
7359 |
kmem_free(p, p->cbp_size); |
|
7360 |
return; |
|
7361 |
} |
|
7362 |
} |
|
7363 |
(void) (STRLOG(0, 0, 0, SL_CONSOLE, |
|
7364 |
"callbparams_free: not found\n")); |
|
7365 |
} |
|
7366 |
||
7367 |
void |
|
7368 |
callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag) |
|
7369 |
{ |
|
7370 |
callbparams_t **pp, *p; |
|
7371 |
||
7372 |
ASSERT(MUTEX_HELD(SQLOCK(sq))); |
|
7373 |
||
7374 |
for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { |
|
7375 |
if (p->cbp_id == id && p->cbp_flags == flag) { |
|
7376 |
*pp = p->cbp_next; |
|
7377 |
kmem_free(p, p->cbp_size); |
|
7378 |
return; |
|
7379 |
} |
|
7380 |
} |
|
7381 |
(void) (STRLOG(0, 0, 0, SL_CONSOLE, |
|
7382 |
"callbparams_free_id: not found\n")); |
|
7383 |
} |
|
7384 |
||
7385 |
/* |
|
7386 |
* Callback wrapper function used by once-only callbacks that can be |
|
7387 |
* cancelled (qtimeout and qbufcall) |
|
7388 |
* Contains inline version of entersq(sq, SQ_CALLBACK) that can be |
|
7389 |
* cancelled by the qun* functions. |
|
7390 |
*/ |
|
7391 |
void |
|
7392 |
qcallbwrapper(void *arg) |
|
7393 |
{ |
|
7394 |
callbparams_t *cbp = arg; |
|
7395 |
syncq_t *sq; |
|
7396 |
uint16_t count = 0; |
|
7397 |
uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; |
|
7398 |
uint16_t type; |
|
7399 |
||
7400 |
sq = cbp->cbp_sq; |
|
7401 |
mutex_enter(SQLOCK(sq)); |
|
7402 |
type = sq->sq_type; |
|
7403 |
if (!(type & SQ_CICB)) { |
|
7404 |
count = sq->sq_count; |
|
7405 |
SQ_PUTLOCKS_ENTER(sq); |
|
7406 |
SQ_PUTCOUNT_CLRFAST_LOCKED(sq); |
|
7407 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
7408 |
sq->sq_needexcl++; |
|
7409 |
ASSERT(sq->sq_needexcl != 0); /* wraparound */ |
|
7410 |
waitflags |= SQ_MESSAGES; |
|
7411 |
} |
|
7412 |
/* Can not handle exlusive entry at outer perimeter */ |
|
7413 |
ASSERT(type & SQ_COCB); |
|
7414 |
||
7415 |
while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) { |
|
7416 |
if ((sq->sq_callbflags & cbp->cbp_flags) && |
|
7417 |
(sq->sq_cancelid == cbp->cbp_id)) { |
|
7418 |
/* timeout has been cancelled */ |
|
7419 |
sq->sq_callbflags |= SQ_CALLB_BYPASSED; |
|
7420 |
callbparams_free(sq, cbp); |
|
7421 |
if (!(type & SQ_CICB)) { |
|
7422 |
ASSERT(sq->sq_needexcl > 0); |
|
7423 |
sq->sq_needexcl--; |
|
7424 |
if (sq->sq_needexcl == 0) { |
|
7425 |
SQ_PUTCOUNT_SETFAST_LOCKED(sq); |
|
7426 |
} |
|
7427 |
SQ_PUTLOCKS_EXIT(sq); |
|
7428 |
} |
|
7429 |
mutex_exit(SQLOCK(sq)); |
|
7430 |
return; |
|
7431 |
} |
|
7432 |
sq->sq_flags |= SQ_WANTWAKEUP; |
|
7433 |
if (!(type & SQ_CICB)) { |
|
7434 |
SQ_PUTLOCKS_EXIT(sq); |
|
7435 |
} |
|
7436 |
cv_wait(&sq->sq_wait, SQLOCK(sq)); |
|
7437 |
if (!(type & SQ_CICB)) { |
|
7438 |
count = sq->sq_count; |
|
7439 |
SQ_PUTLOCKS_ENTER(sq); |
|
7440 |
SUM_SQ_PUTCOUNTS(sq, count); |
|
7441 |
} |
|
7442 |
} |
|
7443 |
||
7444 |
sq->sq_count++; |
|
7445 |
ASSERT(sq->sq_count != 0); /* Wraparound */ |
|
7446 |
if (!(type & SQ_CICB)) { |
|
7447 |
ASSERT(count == 0); |
|
7448 |
sq->sq_flags |= SQ_EXCL; |
|
7449 |
ASSERT(sq->sq_needexcl > 0); |
|
7450 |
sq->sq_needexcl--; |
|
7451 |
if (sq->sq_needexcl == 0) { |
|
7452 |
SQ_PUTCOUNT_SETFAST_LOCKED(sq); |
|
7453 |
} |
|
7454 |
SQ_PUTLOCKS_EXIT(sq); |
|
7455 |
} |
|
7456 |
||
7457 |
mutex_exit(SQLOCK(sq)); |
|
7458 |
||
7459 |
cbp->cbp_func(cbp->cbp_arg); |
|
7460 |
||
7461 |
/* |
|
7462 |
* We drop the lock only for leavesq to re-acquire it. |
|
7463 |
* Possible optimization is inline of leavesq. |
|
7464 |
*/ |
|
7465 |
mutex_enter(SQLOCK(sq)); |
|
7466 |
callbparams_free(sq, cbp); |
|
7467 |
mutex_exit(SQLOCK(sq)); |
|
7468 |
leavesq(sq, SQ_CALLBACK); |
|
7469 |
} |
|
7470 |
||
7471 |
/* |
|
7472 |
* no need to grab sq_putlocks here. See comment in strsubr.h that |
|
7473 |
* explains when sq_putlocks are used. |
|
7474 |
* |
|
7475 |
* sq_count (or one of the sq_putcounts) has already been |
|
7476 |
* decremented by the caller, and if SQ_QUEUED, we need to call |
|
7477 |
* drain_syncq (the global syncq drain). |
|
7478 |
* If putnext_tail is called with the SQ_EXCL bit set, we are in |
|
7479 |
* one of two states, non-CIPUT perimiter, and we need to clear |
|
7480 |
* it, or we went exclusive in the put procedure. In any case, |
|
7481 |
* we want to clear the bit now, and it is probably easier to do |
|
7482 |
* this at the beginning of this function (remember, we hold |
|
7483 |
* the SQLOCK). Lastly, if there are other messages queued |
|
7484 |
* on the syncq (and not for our destination), enable the syncq |
|
7485 |
* for background work. |
|
7486 |
*/ |
|
7487 |
||
7488 |
/* ARGSUSED */ |
|
7489 |
void |
|
7490 |
putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags) |
|
7491 |
{ |
|
7492 |
uint16_t flags = sq->sq_flags; |
|
7493 |
||
7494 |
ASSERT(MUTEX_HELD(SQLOCK(sq))); |
|
7495 |
ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); |
|
7496 |
||
7497 |
/* Clear SQ_EXCL if set in passflags */ |
|
7498 |
if (passflags & SQ_EXCL) { |
|
7499 |
flags &= ~SQ_EXCL; |
|
7500 |
} |
|
7501 |
if (flags & SQ_WANTWAKEUP) { |
|
7502 |
flags &= ~SQ_WANTWAKEUP; |
|
7503 |
cv_broadcast(&sq->sq_wait); |
|
7504 |
} |
|
7505 |
if (flags & SQ_WANTEXWAKEUP) { |
|
7506 |
flags &= ~SQ_WANTEXWAKEUP; |
|
7507 |
cv_broadcast(&sq->sq_exitwait); |
|
7508 |
} |
|
7509 |
sq->sq_flags = flags; |
|
7510 |
||
7511 |
/* |
|
7512 |
* We have cleared SQ_EXCL if we were asked to, and started |
|
7513 |
* the wakeup process for waiters. If there are no writers |
|
7514 |
* then we need to drain the syncq if we were told to, or |
|
7515 |
* enable the background thread to do it. |
|
7516 |
*/ |
|
7517 |
if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) { |
|
7518 |
if ((passflags & SQ_QUEUED) || |
|
7519 |
(sq->sq_svcflags & SQ_DISABLED)) { |
|
7520 |
/* drain_syncq will take care of events in the list */ |
|
7521 |
drain_syncq(sq); |
|
7522 |
return; |
|
7523 |
} else if (flags & SQ_QUEUED) { |
|
7524 |
sqenable(sq); |
|
7525 |
} |
|
7526 |
} |
|
7527 |
/* Drop the SQLOCK on exit */ |
|
7528 |
mutex_exit(SQLOCK(sq)); |
|
7529 |
TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END, |
|
7530 |
"putnext_end:(%p, %p, %p) done", NULL, qp, sq); |
|
7531 |
} |
|
7532 |
||
7533 |
void |
|
7534 |
set_qend(queue_t *q) |
|
7535 |
{ |
|
7536 |
mutex_enter(QLOCK(q)); |
|
7537 |
if (!O_SAMESTR(q)) |
|
7538 |
q->q_flag |= QEND; |
|
7539 |
else |
|
7540 |
q->q_flag &= ~QEND; |
|
7541 |
mutex_exit(QLOCK(q)); |
|
7542 |
q = _OTHERQ(q); |
|
7543 |
mutex_enter(QLOCK(q)); |
|
7544 |
if (!O_SAMESTR(q)) |
|
7545 |
q->q_flag |= QEND; |
|
7546 |
else |
|
7547 |
q->q_flag &= ~QEND; |
|
7548 |
mutex_exit(QLOCK(q)); |
|
7549 |
} |
|
7550 |
||
7551 |
||
7552 |
void |
|
7553 |
clr_qfull(queue_t *q) |
|
7554 |
{ |
|
7555 |
queue_t *oq = q; |
|
7556 |
||
7557 |
q = q->q_nfsrv; |
|
7558 |
/* Fast check if there is any work to do before getting the lock. */ |
|
7559 |
if ((q->q_flag & (QFULL|QWANTW)) == 0) { |
|
7560 |
return; |
|
7561 |
} |
|
7562 |
||
7563 |
/* |
|
7564 |
* Do not reset QFULL (and backenable) if the q_count is the reason |
|
7565 |
* for QFULL being set. |
|
7566 |
*/ |
|
7567 |
mutex_enter(QLOCK(q)); |
|
7568 |
/* |
|
5360
96c808cd96bc
6526408 udp application hangs forever when using SO_SNDBUF with value 0 due to QFULL flag never cleared
rk129064
parents:
3932
diff
changeset
|
7569 |
* If queue is empty i.e q_mblkcnt is zero, queue can not be full. |
96c808cd96bc
6526408 udp application hangs forever when using SO_SNDBUF with value 0 due to QFULL flag never cleared
rk129064
parents:
3932
diff
changeset
|
7570 |
* Hence clear the QFULL. |
96c808cd96bc
6526408 udp application hangs forever when using SO_SNDBUF with value 0 due to QFULL flag never cleared
rk129064
parents:
3932
diff
changeset
|
7571 |
* If both q_count and q_mblkcnt are less than the hiwat mark, |
96c808cd96bc
6526408 udp application hangs forever when using SO_SNDBUF with value 0 due to QFULL flag never cleared
rk129064
parents:
3932
diff
changeset
|
7572 |
* clear the QFULL. |
96c808cd96bc
6526408 udp application hangs forever when using SO_SNDBUF with value 0 due to QFULL flag never cleared
rk129064
parents:
3932
diff
changeset
|
7573 |
*/ |
96c808cd96bc
6526408 udp application hangs forever when using SO_SNDBUF with value 0 due to QFULL flag never cleared
rk129064
parents:
3932
diff
changeset
|
7574 |
if (q->q_mblkcnt == 0 || ((q->q_count < q->q_hiwat) && |
96c808cd96bc
6526408 udp application hangs forever when using SO_SNDBUF with value 0 due to QFULL flag never cleared
rk129064
parents:
3932
diff
changeset
|
7575 |
(q->q_mblkcnt < q->q_hiwat))) { |
0 | 7576 |
q->q_flag &= ~QFULL; |
7577 |
/* |
|
7578 |
* A little more confusing, how about this way: |
|
7579 |
* if someone wants to write, |
|
7580 |
* AND |
|
7581 |
* both counts are less than the lowat mark |
|
7582 |
* OR |
|
7583 |
* the lowat mark is zero |
|
7584 |
* THEN |
|
7585 |
* backenable |
|
7586 |
*/ |
|
7587 |
if ((q->q_flag & QWANTW) && |
|
7588 |
(((q->q_count < q->q_lowat) && |
|
7589 |
(q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) { |
|
7590 |
q->q_flag &= ~QWANTW; |
|
7591 |
mutex_exit(QLOCK(q)); |
|
7592 |
backenable(oq, 0); |
|
7593 |
} else |
|
7594 |
mutex_exit(QLOCK(q)); |
|
7595 |
} else |
|
7596 |
mutex_exit(QLOCK(q)); |
|
7597 |
} |
|
7598 |
||
7599 |
/* |
|
7600 |
* Set the forward service procedure pointer. |
|
7601 |
* |
|
7602 |
* Called at insert-time to cache a queue's next forward service procedure in |
|
7603 |
* q_nfsrv; used by canput() and canputnext(). If the queue to be inserted |
|
7604 |
* has a service procedure then q_nfsrv points to itself. If the queue to be |
|
7605 |
* inserted does not have a service procedure, then q_nfsrv points to the next |
|
7606 |
* queue forward that has a service procedure. If the queue is at the logical |
|
7607 |
* end of the stream (driver for write side, stream head for the read side) |
|
7608 |
* and does not have a service procedure, then q_nfsrv also points to itself. |
|
7609 |
*/ |
|
7610 |
void |
|
7611 |
set_nfsrv_ptr( |
|
7612 |
queue_t *rnew, /* read queue pointer to new module */ |
|
7613 |
queue_t *wnew, /* write queue pointer to new module */ |
|
7614 |
queue_t *prev_rq, /* read queue pointer to the module above */ |
|
7615 |
queue_t *prev_wq) /* write queue pointer to the module above */ |
|
7616 |
{ |
|
7617 |
queue_t *qp; |
|
7618 |
||
7619 |
if (prev_wq->q_next == NULL) { |
|
7620 |
/* |
|
7621 |
* Insert the driver, initialize the driver and stream head. |
|
7622 |
* In this case, prev_rq/prev_wq should be the stream head. |
|
7623 |
* _I_INSERT does not allow inserting a driver. Make sure |
|
7624 |
* that it is not an insertion. |
|
7625 |
*/ |
|
7626 |
ASSERT(!(rnew->q_flag & _QINSERTING)); |
|
7627 |
wnew->q_nfsrv = wnew; |
|
7628 |
if (rnew->q_qinfo->qi_srvp) |
|
7629 |
rnew->q_nfsrv = rnew; |
|
7630 |
else |
|
7631 |
rnew->q_nfsrv = prev_rq; |
|
7632 |
prev_rq->q_nfsrv = prev_rq; |
|
7633 |
prev_wq->q_nfsrv = prev_wq; |
|
7634 |
} else { |
|
7635 |
/* |
|
7636 |
* set up read side q_nfsrv pointer. This MUST be done |
|
7637 |
* before setting the write side, because the setting of |
|
7638 |
* the write side for a fifo may depend on it. |
|
7639 |
* |
|
7640 |
* Suppose we have a fifo that only has pipemod pushed. |
|
7641 |
* pipemod has no read or write service procedures, so |
|
7642 |
* nfsrv for both pipemod queues points to prev_rq (the |
|
7643 |
* stream read head). Now push bufmod (which has only a |
|
7644 |
* read service procedure). Doing the write side first, |
|
7645 |
* wnew->q_nfsrv is set to pipemod's writeq nfsrv, which |
|
7646 |
* is WRONG; the next queue forward from wnew with a |
|
7647 |
* service procedure will be rnew, not the stream read head. |
|
7648 |
* Since the downstream queue (which in the case of a fifo |
|
7649 |
* is the read queue rnew) can affect upstream queues, it |
|
7650 |
* needs to be done first. Setting up the read side first |
|
7651 |
* sets nfsrv for both pipemod queues to rnew and then |
|
7652 |
* when the write side is set up, wnew-q_nfsrv will also |
|
7653 |
* point to rnew. |
|
7654 |
*/ |
|
7655 |
if (rnew->q_qinfo->qi_srvp) { |
|
7656 |
/* |
|
7657 |
* use _OTHERQ() because, if this is a pipe, next |
|
7658 |
* module may have been pushed from other end and |
|
7659 |
* q_next could be a read queue. |
|
7660 |
*/ |
|
7661 |
qp = _OTHERQ(prev_wq->q_next); |
|
7662 |
while (qp && qp->q_nfsrv != qp) { |
|
7663 |
qp->q_nfsrv = rnew; |
|
7664 |
qp = backq(qp); |
|
7665 |
} |
|
7666 |
rnew->q_nfsrv = rnew; |
|
7667 |
} else |
|
7668 |
rnew->q_nfsrv = prev_rq->q_nfsrv; |
|
7669 |
||
7670 |
/* set up write side q_nfsrv pointer */ |
|
7671 |
if (wnew->q_qinfo->qi_srvp) { |
|
7672 |
wnew->q_nfsrv = wnew; |
|
7673 |
||
7674 |
/* |
|
7675 |
* For insertion, need to update nfsrv of the modules |
|
7676 |
* above which do not have a service routine. |
|
7677 |
*/ |
|
7678 |
if (rnew->q_flag & _QINSERTING) { |
|
7679 |
for (qp = prev_wq; |
|
7680 |
qp != NULL && qp->q_nfsrv != qp; |
|
7681 |
qp = backq(qp)) { |
|
7682 |
qp->q_nfsrv = wnew->q_nfsrv; |
|
7683 |
} |
|
7684 |
} |
|
7685 |
} else { |
|
7686 |
if (prev_wq->q_next == prev_rq) |
|
7687 |
/* |
|
7688 |
* Since prev_wq/prev_rq are the middle of a |
|
7689 |
* fifo, wnew/rnew will also be the middle of |
|
7690 |
* a fifo and wnew's nfsrv is same as rnew's. |
|
7691 |
*/ |
|
7692 |
wnew->q_nfsrv = rnew->q_nfsrv; |
|
7693 |
else |
|
7694 |
wnew->q_nfsrv = prev_wq->q_next->q_nfsrv; |
|
7695 |
} |
|
7696 |
} |
|
7697 |
} |
|
7698 |
||
7699 |
/* |
|
7700 |
* Reset the forward service procedure pointer; called at remove-time. |
|
7701 |
*/ |
|
7702 |
void |
|
7703 |
reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp) |
|
7704 |
{ |
|
7705 |
queue_t *tmp_qp; |
|
7706 |
||
7707 |
/* Reset the write side q_nfsrv pointer for _I_REMOVE */ |
|
7708 |
if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) { |
|
7709 |
for (tmp_qp = backq(wqp); |
|
7710 |
tmp_qp != NULL && tmp_qp->q_nfsrv == wqp; |
|
7711 |
tmp_qp = backq(tmp_qp)) { |
|
7712 |
tmp_qp->q_nfsrv = wqp->q_nfsrv; |
|
7713 |
} |
|
7714 |
} |
|
7715 |
||
7716 |
/* reset the read side q_nfsrv pointer */ |
|
7717 |
if (rqp->q_qinfo->qi_srvp) { |
|
7718 |
if (wqp->q_next) { /* non-driver case */ |
|
7719 |
tmp_qp = _OTHERQ(wqp->q_next); |
|
7720 |
while (tmp_qp && tmp_qp->q_nfsrv == rqp) { |
|
7721 |
/* Note that rqp->q_next cannot be NULL */ |
|
7722 |
ASSERT(rqp->q_next != NULL); |
|
7723 |
tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv; |
|
7724 |
tmp_qp = backq(tmp_qp); |
|
7725 |
} |
|
7726 |
} |
|
7727 |
} |
|
7728 |
} |
|
7729 |
||
7730 |
/* |
|
7731 |
* This routine should be called after all stream geometry changes to update |
|
7732 |
* the stream head cached struio() rd/wr queue pointers. Note must be called |
|
7733 |
* with the streamlock()ed. |
|
7734 |
* |
|
7735 |
* Note: only enables Synchronous STREAMS for a side of a Stream which has |
|
7736 |
* an explicit synchronous barrier module queue. That is, a queue that |
|
7737 |
* has specified a struio() type. |
|
7738 |
*/ |
|
7739 |
static void |
|
7740 |
strsetuio(stdata_t *stp) |
|
7741 |
{ |
|
7742 |
queue_t *wrq; |
|
7743 |
||
7744 |
if (stp->sd_flag & STPLEX) { |
|
7745 |
/* |
|
7746 |
* Not stremahead, but a mux, so no Synchronous STREAMS. |
|
7747 |
*/ |
|
7748 |
stp->sd_struiowrq = NULL; |
|
7749 |
stp->sd_struiordq = NULL; |
|
7750 |
return; |
|
7751 |
} |
|
7752 |
/* |
|
7753 |
* Scan the write queue(s) while synchronous |
|
7754 |
* until we find a qinfo uio type specified. |
|
7755 |
*/ |
|
7756 |
wrq = stp->sd_wrq->q_next; |
|
7757 |
while (wrq) { |
|
7758 |
if (wrq->q_struiot == STRUIOT_NONE) { |
|
7759 |
wrq = 0; |
|
7760 |
break; |
|
7761 |
} |
|
7762 |
if (wrq->q_struiot != STRUIOT_DONTCARE) |
|
7763 |
break; |
|
7764 |
if (! _SAMESTR(wrq)) { |
|
7765 |
wrq = 0; |
|
7766 |
break; |
|
7767 |
} |
|
7768 |
wrq = wrq->q_next; |
|
7769 |
} |
|
7770 |
stp->sd_struiowrq = wrq; |
|
7771 |
/* |
|
7772 |
* Scan the read queue(s) while synchronous |
|
7773 |
* until we find a qinfo uio type specified. |
|
7774 |
*/ |
|
7775 |
wrq = stp->sd_wrq->q_next; |
|
7776 |
while (wrq) { |
|
7777 |
if (_RD(wrq)->q_struiot == STRUIOT_NONE) { |
|
7778 |
wrq = 0; |
|
7779 |
break; |
|
7780 |
} |
|
7781 |
if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE) |
|
7782 |
break; |
|
7783 |
if (! _SAMESTR(wrq)) { |
|
7784 |
wrq = 0; |
|
7785 |
break; |
|
7786 |
} |
|
7787 |
wrq = wrq->q_next; |
|
7788 |
} |
|
7789 |
stp->sd_struiordq = wrq ? _RD(wrq) : 0; |
|
7790 |
} |
|
7791 |
||
7792 |
/* |
|
7793 |
* pass_wput, unblocks the passthru queues, so that |
|
7794 |
* messages can arrive at muxs lower read queue, before |
|
7795 |
* I_LINK/I_UNLINK is acked/nacked. |
|
7796 |
*/ |
|
7797 |
static void |
|
7798 |
pass_wput(queue_t *q, mblk_t *mp) |
|
7799 |
{ |
|
7800 |
syncq_t *sq; |
|
7801 |
||
7802 |
sq = _RD(q)->q_syncq; |
|
7803 |
if (sq->sq_flags & SQ_BLOCKED) |
|
7804 |
unblocksq(sq, SQ_BLOCKED, 0); |
|
7805 |
putnext(q, mp); |
|
7806 |
} |
|
7807 |
||
7808 |
/* |
|
7809 |
* Set up queues for the link/unlink. |
|
7810 |
* Create a new queue and block it and then insert it |
|
7811 |
* below the stream head on the lower stream. |
|
7812 |
* This prevents any messages from arriving during the setq |
|
7813 |
* as well as while the mux is processing the LINK/I_UNLINK. |
|
7814 |
* The blocked passq is unblocked once the LINK/I_UNLINK has |
|
7815 |
* been acked or nacked or if a message is generated and sent |
|
7816 |
* down muxs write put procedure. |
|
7817 |
* see pass_wput(). |
|
7818 |
* |
|
7819 |
* After the new queue is inserted, all messages coming from below are |
|
7820 |
* blocked. The call to strlock will ensure that all activity in the stream head |
|
7821 |
* read queue syncq is stopped (sq_count drops to zero). |
|
7822 |
*/ |
|
7823 |
static queue_t * |
|
7824 |
link_addpassthru(stdata_t *stpdown) |
|
7825 |
{ |
|
7826 |
queue_t *passq; |
|
7827 |
sqlist_t sqlist; |
|
7828 |
||
7829 |
passq = allocq(); |
|
7830 |
STREAM(passq) = STREAM(_WR(passq)) = stpdown; |
|
7831 |
/* setq might sleep in allocator - avoid holding locks. */ |
|
7832 |
setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ, |
|
7833 |
SQ_CI|SQ_CO, B_FALSE); |
|
7834 |
claimq(passq); |
|
7835 |
blocksq(passq->q_syncq, SQ_BLOCKED, 1); |
|
7836 |
insertq(STREAM(passq), passq); |
|
7837 |
||
7838 |
/* |
|
7839 |
* Use strlock() to wait for the stream head sq_count to drop to zero |
|
7840 |
* since we are going to change q_ptr in the stream head. Note that |
|
7841 |
* insertq() doesn't wait for any syncq counts to drop to zero. |
|
7842 |
*/ |
|
7843 |
sqlist.sqlist_head = NULL; |
|
7844 |
sqlist.sqlist_index = 0; |
|
7845 |
sqlist.sqlist_size = sizeof (sqlist_t); |
|
7846 |
sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq); |
|
7847 |
strlock(stpdown, &sqlist); |
|
7848 |
strunlock(stpdown, &sqlist); |
|
7849 |
||
7850 |
releaseq(passq); |
|
7851 |
return (passq); |
|
7852 |
} |
|
7853 |
||
7854 |
/* |
|
7855 |
* Let messages flow up into the mux by removing |
|
7856 |
* the passq. |
|
7857 |
*/ |
|
7858 |
static void |
|
7859 |
link_rempassthru(queue_t *passq) |
|
7860 |
{ |
|
7861 |
claimq(passq); |
|
7862 |
removeq(passq); |
|
7863 |
releaseq(passq); |
|
7864 |
freeq(passq); |
|
7865 |
} |
|
7866 |
||
7867 |
/* |
|
577 | 7868 |
* Wait for the condition variable pointed to by `cvp' to be signaled, |
7869 |
* or for `tim' milliseconds to elapse, whichever comes first. If `tim' |
|
7870 |
* is negative, then there is no time limit. If `nosigs' is non-zero, |
|
7871 |
* then the wait will be non-interruptible. |
|
7872 |
* |
|
7873 |
* Returns >0 if signaled, 0 if interrupted, or -1 upon timeout. |
|
0 | 7874 |
*/ |
7875 |
clock_t |
|
7876 |
str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs) |
|
7877 |
{ |
|
7878 |
clock_t ret, now, tick; |
|
7879 |
||
7880 |
if (tim < 0) { |
|
7881 |
if (nosigs) { |
|
7882 |
cv_wait(cvp, mp); |
|
7883 |
ret = 1; |
|
7884 |
} else { |
|
7885 |
ret = cv_wait_sig(cvp, mp); |
|
7886 |
} |
|
7887 |
} else if (tim > 0) { |
|
7888 |
/* |
|
7889 |
* convert milliseconds to clock ticks |
|
7890 |
*/ |
|
7891 |
tick = MSEC_TO_TICK_ROUNDUP(tim); |
|
7892 |
time_to_wait(&now, tick); |
|
7893 |
if (nosigs) { |
|
7894 |
ret = cv_timedwait(cvp, mp, now); |
|
7895 |
} else { |
|
7896 |
ret = cv_timedwait_sig(cvp, mp, now); |
|
7897 |
} |
|
7898 |
} else { |
|
7899 |
ret = -1; |
|
7900 |
} |
|
7901 |
return (ret); |
|
7902 |
} |
|
7903 |
||
7904 |
/* |
|
7905 |
* Wait until the stream head can determine if it is at the mark but |
|
7906 |
* don't wait forever to prevent a race condition between the "mark" state |
|
7907 |
* in the stream head and any mark state in the caller/user of this routine. |
|
7908 |
* |
|
7909 |
* This is used by sockets and for a socket it would be incorrect |
|
7910 |
* to return a failure for SIOCATMARK when there is no data in the receive |
|
7911 |
* queue and the marked urgent data is traveling up the stream. |
|
7912 |
* |
|
7913 |
* This routine waits until the mark is known by waiting for one of these |
|
7914 |
* three events: |
|
7915 |
* The stream head read queue becoming non-empty (including an EOF) |
|
7916 |
* The STRATMARK flag being set. (Due to a MSGMARKNEXT message.) |
|
7917 |
* The STRNOTATMARK flag being set (which indicates that the transport |
|
7918 |
* has sent a MSGNOTMARKNEXT message to indicate that it is not at |
|
7919 |
* the mark). |
|
7920 |
* |
|
7921 |
* The routine returns 1 if the stream is at the mark; 0 if it can |
|
7922 |
* be determined that the stream is not at the mark. |
|
7923 |
* If the wait times out and it can't determine |
|
7924 |
* whether or not the stream might be at the mark the routine will return -1. |
|
7925 |
* |
|
7926 |
* Note: This routine should only be used when a mark is pending i.e., |
|
7927 |
* in the socket case the SIGURG has been posted. |
|
7928 |
* Note2: This can not wakeup just because synchronous streams indicate |
|
7929 |
* that data is available since it is not possible to use the synchronous |
|
7930 |
* streams interfaces to determine the b_flag value for the data queued below |
|
7931 |
* the stream head. |
|
7932 |
*/ |
|
7933 |
int |
|
7934 |
strwaitmark(vnode_t *vp) |
|
7935 |
{ |
|
7936 |
struct stdata *stp = vp->v_stream; |
|
7937 |
queue_t *rq = _RD(stp->sd_wrq); |
|
7938 |
int mark; |
|
7939 |
||
7940 |
mutex_enter(&stp->sd_lock); |
|
7941 |
while (rq->q_first == NULL && |
|
7942 |
!(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) { |
|
7943 |
stp->sd_flag |= RSLEEP; |
|
7944 |
||
7945 |
/* Wait for 100 milliseconds for any state change. */ |
|
7946 |
if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) { |
|
7947 |
mutex_exit(&stp->sd_lock); |
|
7948 |
return (-1); |
|
7949 |
} |
|
7950 |
} |
|
7951 |
if (stp->sd_flag & STRATMARK) |
|
7952 |
mark = 1; |
|
7953 |
else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK)) |
|
7954 |
mark = 1; |
|
7955 |
else |
|
7956 |
mark = 0; |
|
7957 |
||
7958 |
mutex_exit(&stp->sd_lock); |
|
7959 |
return (mark); |
|
7960 |
} |
|
7961 |
||
7962 |
/* |
|
7963 |
* Set a read side error. If persist is set change the socket error |
|
7964 |
* to persistent. If errfunc is set install the function as the exported |
|
7965 |
* error handler. |
|
7966 |
*/ |
|
7967 |
void |
|
7968 |
strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) |
|
7969 |
{ |
|
7970 |
struct stdata *stp = vp->v_stream; |
|
7971 |
||
7972 |
mutex_enter(&stp->sd_lock); |
|
7973 |
stp->sd_rerror = error; |
|
7974 |
if (error == 0 && errfunc == NULL) |
|
7975 |
stp->sd_flag &= ~STRDERR; |
|
7976 |
else |
|
7977 |
stp->sd_flag |= STRDERR; |
|
7978 |
if (persist) { |
|
7979 |
stp->sd_flag &= ~STRDERRNONPERSIST; |
|
7980 |
} else { |
|
7981 |
stp->sd_flag |= STRDERRNONPERSIST; |
|
7982 |
} |
|
7983 |
stp->sd_rderrfunc = errfunc; |
|
7984 |
if (error != 0 || errfunc != NULL) { |
|
7985 |
cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ |
|
7986 |
cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ |
|
7987 |
cv_broadcast(&stp->sd_monitor); /* ioctllers */ |
|
7988 |
||
7989 |
mutex_exit(&stp->sd_lock); |
|
7990 |
pollwakeup(&stp->sd_pollist, POLLERR); |
|
7991 |
mutex_enter(&stp->sd_lock); |
|
7992 |
||
7993 |
if (stp->sd_sigflags & S_ERROR) |
|
7994 |
strsendsig(stp->sd_siglist, S_ERROR, 0, error); |
|
7995 |
} |
|
7996 |
mutex_exit(&stp->sd_lock); |
|
7997 |
} |
|
7998 |
||
7999 |
/* |
|
8000 |
* Set a write side error. If persist is set change the socket error |
|
8001 |
* to persistent. |
|
8002 |
*/ |
|
8003 |
void |
|
8004 |
strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) |
|
8005 |
{ |
|
8006 |
struct stdata *stp = vp->v_stream; |
|
8007 |
||
8008 |
mutex_enter(&stp->sd_lock); |
|
8009 |
stp->sd_werror = error; |
|
8010 |
if (error == 0 && errfunc == NULL) |
|
8011 |
stp->sd_flag &= ~STWRERR; |
|
8012 |
else |
|
8013 |
stp->sd_flag |= STWRERR; |
|
8014 |
if (persist) { |
|
8015 |
stp->sd_flag &= ~STWRERRNONPERSIST; |
|
8016 |
} else { |
|
8017 |
stp->sd_flag |= STWRERRNONPERSIST; |
|
8018 |
} |
|
8019 |
stp->sd_wrerrfunc = errfunc; |
|
8020 |
if (error != 0 || errfunc != NULL) { |
|
8021 |
cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ |
|
8022 |
cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ |
|
8023 |
cv_broadcast(&stp->sd_monitor); /* ioctllers */ |
|
8024 |
||
8025 |
mutex_exit(&stp->sd_lock); |
|
8026 |
pollwakeup(&stp->sd_pollist, POLLERR); |
|
8027 |
mutex_enter(&stp->sd_lock); |
|
8028 |
||
8029 |
if (stp->sd_sigflags & S_ERROR) |
|
8030 |
strsendsig(stp->sd_siglist, S_ERROR, 0, error); |
|
8031 |
} |
|
8032 |
mutex_exit(&stp->sd_lock); |
|
8033 |
} |
|
8034 |
||
8035 |
/* |
|
8036 |
* Make the stream return 0 (EOF) when all data has been read. |
|
8037 |
* No effect on write side. |
|
8038 |
*/ |
|
8039 |
void |
|
8040 |
strseteof(vnode_t *vp, int eof) |
|
8041 |
{ |
|
8042 |
struct stdata *stp = vp->v_stream; |
|
8043 |
||
8044 |
mutex_enter(&stp->sd_lock); |
|
8045 |
if (!eof) { |
|
8046 |
stp->sd_flag &= ~STREOF; |
|
8047 |
mutex_exit(&stp->sd_lock); |
|
8048 |
return; |
|
8049 |
} |
|
8050 |
stp->sd_flag |= STREOF; |
|
8051 |
if (stp->sd_flag & RSLEEP) { |
|
8052 |
stp->sd_flag &= ~RSLEEP; |
|
8053 |
cv_broadcast(&_RD(stp->sd_wrq)->q_wait); |
|
8054 |
} |
|
8055 |
||
8056 |
mutex_exit(&stp->sd_lock); |
|
8057 |
pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM); |
|
8058 |
mutex_enter(&stp->sd_lock); |
|
8059 |
||
8060 |
if (stp->sd_sigflags & (S_INPUT|S_RDNORM)) |
|
8061 |
strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0); |
|
8062 |
mutex_exit(&stp->sd_lock); |
|
8063 |
} |
|
8064 |
||
8065 |
void |
|
8066 |
strflushrq(vnode_t *vp, int flag) |
|
8067 |
{ |
|
8068 |
struct stdata *stp = vp->v_stream; |
|
8069 |
||
8070 |
mutex_enter(&stp->sd_lock); |
|
8071 |
flushq(_RD(stp->sd_wrq), flag); |
|
8072 |
mutex_exit(&stp->sd_lock); |
|
8073 |
} |
|
8074 |
||
8075 |
void |
|
8076 |
strsetrputhooks(vnode_t *vp, uint_t flags, |
|
8077 |
msgfunc_t protofunc, msgfunc_t miscfunc) |
|
8078 |
{ |
|
8079 |
struct stdata *stp = vp->v_stream; |
|
8080 |
||
8081 |
mutex_enter(&stp->sd_lock); |
|
8082 |
||
8083 |
if (protofunc == NULL) |
|
8084 |
stp->sd_rprotofunc = strrput_proto; |
|
8085 |
else |
|
8086 |
stp->sd_rprotofunc = protofunc; |
|
8087 |
||
8088 |
if (miscfunc == NULL) |
|
8089 |
stp->sd_rmiscfunc = strrput_misc; |
|
8090 |
else |
|
8091 |
stp->sd_rmiscfunc = miscfunc; |
|
8092 |
||
8093 |
if (flags & SH_CONSOL_DATA) |
|
8094 |
stp->sd_rput_opt |= SR_CONSOL_DATA; |
|
8095 |
else |
|
8096 |
stp->sd_rput_opt &= ~SR_CONSOL_DATA; |
|
8097 |
||
8098 |
if (flags & SH_SIGALLDATA) |
|
8099 |
stp->sd_rput_opt |= SR_SIGALLDATA; |
|
8100 |
else |
|
8101 |
stp->sd_rput_opt &= ~SR_SIGALLDATA; |
|
8102 |
||
8103 |
if (flags & SH_IGN_ZEROLEN) |
|
8104 |
stp->sd_rput_opt |= SR_IGN_ZEROLEN; |
|
8105 |
else |
|
8106 |
stp->sd_rput_opt &= ~SR_IGN_ZEROLEN; |
|
8107 |
||
8108 |
mutex_exit(&stp->sd_lock); |
|
8109 |
} |
|
8110 |
||
8111 |
void |
|
8112 |
strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime) |
|
8113 |
{ |
|
8114 |
struct stdata *stp = vp->v_stream; |
|
8115 |
||
8116 |
mutex_enter(&stp->sd_lock); |
|
8117 |
stp->sd_closetime = closetime; |
|
8118 |
||
8119 |
if (flags & SH_SIGPIPE) |
|
8120 |
stp->sd_wput_opt |= SW_SIGPIPE; |
|
8121 |
else |
|
8122 |
stp->sd_wput_opt &= ~SW_SIGPIPE; |
|
8123 |
if (flags & SH_RECHECK_ERR) |
|
8124 |
stp->sd_wput_opt |= SW_RECHECK_ERR; |
|
8125 |
else |
|
8126 |
stp->sd_wput_opt &= ~SW_RECHECK_ERR; |
|
8127 |
||
8128 |
mutex_exit(&stp->sd_lock); |
|
8129 |
} |
|
8130 |
||
898 | 8131 |
void |
8132 |
strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc) |
|
8133 |
{ |
|
8134 |
struct stdata *stp = vp->v_stream; |
|
8135 |
||
8136 |
mutex_enter(&stp->sd_lock); |
|
8137 |
||
8138 |
stp->sd_rputdatafunc = rdatafunc; |
|
8139 |
stp->sd_wputdatafunc = wdatafunc; |
|
8140 |
||
8141 |
mutex_exit(&stp->sd_lock); |
|
8142 |
} |
|
8143 |
||
0 | 8144 |
/* Used within framework when the queue is already locked */ |
8145 |
void |
|
8146 |
qenable_locked(queue_t *q) |
|
8147 |
{ |
|
8148 |
stdata_t *stp = STREAM(q); |
|
8149 |
||
8150 |
ASSERT(MUTEX_HELD(QLOCK(q))); |
|
8151 |
||
8152 |
if (!q->q_qinfo->qi_srvp) |
|
8153 |
return; |
|
8154 |
||
8155 |
/* |
|
8156 |
* Do not place on run queue if already enabled or closing. |
|
8157 |
*/ |
|
8158 |
if (q->q_flag & (QWCLOSE|QENAB)) |
|
8159 |
return; |
|
8160 |
||
8161 |
/* |
|
8162 |
* mark queue enabled and place on run list if it is not already being |
|
8163 |
* serviced. If it is serviced, the runservice() function will detect |
|
8164 |
* that QENAB is set and call service procedure before clearing |
|
8165 |
* QINSERVICE flag. |
|
8166 |
*/ |
|
8167 |
q->q_flag |= QENAB; |
|
8168 |
if (q->q_flag & QINSERVICE) |
|
8169 |
return; |
|
8170 |
||
8171 |
/* Record the time of qenable */ |
|
8172 |
q->q_qtstamp = lbolt; |
|
8173 |
||
8174 |
/* |
|
8175 |
* Put the queue in the stp list and schedule it for background |
|
8176 |
* processing if it is not already scheduled or if stream head does not |
|
8177 |
* intent to process it in the foreground later by setting |
|
8178 |
* STRS_WILLSERVICE flag. |
|
8179 |
*/ |
|
8180 |
mutex_enter(&stp->sd_qlock); |
|
8181 |
/* |
|
8182 |
* If there are already something on the list, stp flags should show |
|
8183 |
* intention to drain it. |
|
8184 |
*/ |
|
8185 |
IMPLY(STREAM_NEEDSERVICE(stp), |
|
8186 |
(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))); |
|
8187 |
||
8188 |
ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link); |
|
8189 |
stp->sd_nqueues++; |
|
8190 |
||
8191 |
/* |
|
8192 |
* If no one will drain this stream we are the first producer and |
|
8193 |
* need to schedule it for background thread. |
|
8194 |
*/ |
|
8195 |
if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) { |
|
8196 |
/* |
|
8197 |
* No one will service this stream later, so we have to |
|
8198 |
* schedule it now. |
|
8199 |
*/ |
|
8200 |
STRSTAT(stenables); |
|
8201 |
stp->sd_svcflags |= STRS_SCHEDULED; |
|
8202 |
stp->sd_servid = (void *)taskq_dispatch(streams_taskq, |
|
8203 |
(task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE); |
|
8204 |
||
8205 |
if (stp->sd_servid == NULL) { |
|
8206 |
/* |
|
8207 |
* Task queue failed so fail over to the backup |
|
8208 |
* servicing thread. |
|
8209 |
*/ |
|
8210 |
STRSTAT(taskqfails); |
|
8211 |
/* |
|
8212 |
* It is safe to clear STRS_SCHEDULED flag because it |
|
8213 |
* was set by this thread above. |
|
8214 |
*/ |
|
8215 |
stp->sd_svcflags &= ~STRS_SCHEDULED; |
|
8216 |
||
8217 |
/* |
|
8218 |
* Failover scheduling is protected by service_queue |
|
8219 |
* lock. |
|
8220 |
*/ |
|
8221 |
mutex_enter(&service_queue); |
|
8222 |
ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q)); |
|
8223 |
ASSERT(q->q_link == NULL); |
|
8224 |
/* |
|
8225 |
* Append the queue to qhead/qtail list. |
|
8226 |
*/ |
|
8227 |
if (qhead == NULL) |
|
8228 |
qhead = q; |
|
8229 |
else |
|
8230 |
qtail->q_link = q; |
|
8231 |
qtail = q; |
|
8232 |
/* |
|
8233 |
* Clear stp queue list. |
|
8234 |
*/ |
|
8235 |
stp->sd_qhead = stp->sd_qtail = NULL; |
|
8236 |
stp->sd_nqueues = 0; |
|
8237 |
/* |
|
8238 |
* Wakeup background queue processing thread. |
|
8239 |
*/ |
|
8240 |
cv_signal(&services_to_run); |
|
8241 |
mutex_exit(&service_queue); |
|
8242 |
} |
|
8243 |
} |
|
8244 |
mutex_exit(&stp->sd_qlock); |
|
8245 |
} |
|
8246 |
||
8247 |
static void |
|
8248 |
queue_service(queue_t *q) |
|
8249 |
{ |
|
8250 |
/* |
|
8251 |
* The queue in the list should have |
|
8252 |
* QENAB flag set and should not have |
|
8253 |
* QINSERVICE flag set. QINSERVICE is |
|
8254 |
* set when the queue is dequeued and |
|
8255 |
* qenable_locked doesn't enqueue a |
|
8256 |
* queue with QINSERVICE set. |
|
8257 |
*/ |
|
8258 |
||
8259 |
ASSERT(!(q->q_flag & QINSERVICE)); |
|
8260 |
ASSERT((q->q_flag & QENAB)); |
|
8261 |
mutex_enter(QLOCK(q)); |
|
8262 |
q->q_flag &= ~QENAB; |
|
8263 |
q->q_flag |= QINSERVICE; |
|
8264 |
mutex_exit(QLOCK(q)); |
|
8265 |
runservice(q); |
|
8266 |
} |
|
8267 |
||
8268 |
static void |
|
8269 |
syncq_service(syncq_t *sq) |
|
8270 |
{ |
|
8271 |
STRSTAT(syncqservice); |
|
8272 |
mutex_enter(SQLOCK(sq)); |
|
8273 |
ASSERT(!(sq->sq_svcflags & SQ_SERVICE)); |
|
8274 |
ASSERT(sq->sq_servcount != 0); |
|
8275 |
ASSERT(sq->sq_next == NULL); |
|
8276 |
||
8277 |
/* if we came here from the background thread, clear the flag */ |
|
8278 |
if (sq->sq_svcflags & SQ_BGTHREAD) |
|
8279 |
sq->sq_svcflags &= ~SQ_BGTHREAD; |
|
8280 |
||
8281 |
/* let drain_syncq know that it's being called in the background */ |
|
8282 |
sq->sq_svcflags |= SQ_SERVICE; |
|
8283 |
drain_syncq(sq); |
|
8284 |
} |
|
8285 |
||
8286 |
static void |
|
8287 |
qwriter_outer_service(syncq_t *outer) |
|
8288 |
{ |
|
8289 |
/* |
|
8290 |
* Note that SQ_WRITER is used on the outer perimeter |
|
8291 |
* to signal that a qwriter(OUTER) is either investigating |
|
8292 |
* running or that it is actually running a function. |
|
8293 |
*/ |
|
8294 |
outer_enter(outer, SQ_BLOCKED|SQ_WRITER); |
|
8295 |
||
8296 |
/* |
|
8297 |
* All inner syncq are empty and have SQ_WRITER set |
|
8298 |
* to block entering the outer perimeter. |
|
8299 |
* |
|
8300 |
* We do not need to explicitly call write_now since |
|
8301 |
* outer_exit does it for us. |
|
8302 |
*/ |
|
8303 |
outer_exit(outer); |
|
8304 |
} |
|
8305 |
||
8306 |
static void |
|
8307 |
mblk_free(mblk_t *mp) |
|
8308 |
{ |
|
8309 |
dblk_t *dbp = mp->b_datap; |
|
8310 |
frtn_t *frp = dbp->db_frtnp; |
|
8311 |
||
8312 |
mp->b_next = NULL; |
|
8313 |
if (dbp->db_fthdr != NULL) |
|
8314 |
str_ftfree(dbp); |
|
8315 |
||
8316 |
ASSERT(dbp->db_fthdr == NULL); |
|
8317 |
frp->free_func(frp->free_arg); |
|
8318 |
ASSERT(dbp->db_mblk == mp); |
|
8319 |
||
8320 |
if (dbp->db_credp != NULL) { |
|
8321 |
crfree(dbp->db_credp); |
|
8322 |
dbp->db_credp = NULL; |
|
8323 |
} |
|
8324 |
dbp->db_cpid = -1; |
|
8325 |
dbp->db_struioflag = 0; |
|
8326 |
dbp->db_struioun.cksum.flags = 0; |
|
8327 |
||
8328 |
kmem_cache_free(dbp->db_cache, dbp); |
|
8329 |
} |
|
8330 |
||
8331 |
/* |
|
8332 |
* Background processing of the stream queue list. |
|
8333 |
*/ |
|
8334 |
static void |
|
8335 |
stream_service(stdata_t *stp) |
|
8336 |
{ |
|
8337 |
queue_t *q; |
|
8338 |
||
8339 |
mutex_enter(&stp->sd_qlock); |
|
8340 |
||
8341 |
STR_SERVICE(stp, q); |
|
8342 |
||
8343 |
stp->sd_svcflags &= ~STRS_SCHEDULED; |
|
8344 |
stp->sd_servid = NULL; |
|
8345 |
cv_signal(&stp->sd_qcv); |
|
8346 |
mutex_exit(&stp->sd_qlock); |
|
8347 |
} |
|
8348 |
||
8349 |
/* |
|
8350 |
* Foreground processing of the stream queue list. |
|
8351 |
*/ |
|
8352 |
void |
|
8353 |
stream_runservice(stdata_t *stp) |
|
8354 |
{ |
|
8355 |
queue_t *q; |
|
8356 |
||
8357 |
mutex_enter(&stp->sd_qlock); |
|
8358 |
STRSTAT(rservice); |
|
8359 |
/* |
|
8360 |
* We are going to drain this stream queue list, so qenable_locked will |
|
8361 |
* not schedule it until we finish. |
|
8362 |
*/ |
|
8363 |
stp->sd_svcflags |= STRS_WILLSERVICE; |
|
8364 |
||
8365 |
STR_SERVICE(stp, q); |
|
8366 |
||
8367 |
stp->sd_svcflags &= ~STRS_WILLSERVICE; |
|
8368 |
mutex_exit(&stp->sd_qlock); |
|
8369 |
/* |
|
8370 |
* Help backup background thread to drain the qhead/qtail list. |
|
8371 |
*/ |
|
8372 |
while (qhead != NULL) { |
|
8373 |
STRSTAT(qhelps); |
|
8374 |
mutex_enter(&service_queue); |
|
8375 |
DQ(q, qhead, qtail, q_link); |
|
8376 |
mutex_exit(&service_queue); |
|
8377 |
if (q != NULL) |
|
8378 |
queue_service(q); |
|
8379 |
} |
|
8380 |
} |
|
8381 |
||
8382 |
void |
|
8383 |
stream_willservice(stdata_t *stp) |
|
8384 |
{ |
|
8385 |
mutex_enter(&stp->sd_qlock); |
|
8386 |
stp->sd_svcflags |= STRS_WILLSERVICE; |
|
8387 |
mutex_exit(&stp->sd_qlock); |
|
8388 |
} |
|
8389 |
||
8390 |
/* |
|
8391 |
* Replace the cred currently in the mblk with a different one. |
|
8392 |
*/ |
|
8393 |
void |
|
8394 |
mblk_setcred(mblk_t *mp, cred_t *cr) |
|
8395 |
{ |
|
8396 |
cred_t *ocr = DB_CRED(mp); |
|
8397 |
||
8398 |
ASSERT(cr != NULL); |
|
8399 |
||
8400 |
if (cr != ocr) { |
|
8401 |
crhold(mp->b_datap->db_credp = cr); |
|
8402 |
if (ocr != NULL) |
|
8403 |
crfree(ocr); |
|
8404 |
} |
|
8405 |
} |
|
8406 |
||
8407 |
int |
|
8408 |
hcksum_assoc(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, |
|
8409 |
uint32_t start, uint32_t stuff, uint32_t end, uint32_t value, |
|
8410 |
uint32_t flags, int km_flags) |
|
8411 |
{ |
|
8412 |
int rc = 0; |
|
8413 |
||
8414 |
ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); |
|
8415 |
if (mp->b_datap->db_type == M_DATA) { |
|
8416 |
/* Associate values for M_DATA type */ |
|
741
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
8417 |
DB_CKSUMSTART(mp) = (intptr_t)start; |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
8418 |
DB_CKSUMSTUFF(mp) = (intptr_t)stuff; |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
8419 |
DB_CKSUMEND(mp) = (intptr_t)end; |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
8420 |
DB_CKSUMFLAGS(mp) = flags; |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
8421 |
DB_CKSUM16(mp) = (uint16_t)value; |
0 | 8422 |
|
8423 |
} else { |
|
8424 |
pattrinfo_t pa_info; |
|
8425 |
||
8426 |
ASSERT(mmd != NULL); |
|
8427 |
||
8428 |
pa_info.type = PATTR_HCKSUM; |
|
8429 |
pa_info.len = sizeof (pattr_hcksum_t); |
|
8430 |
||
8431 |
if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) { |
|
8432 |
pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf; |
|
8433 |
||
8434 |
hck->hcksum_start_offset = start; |
|
8435 |
hck->hcksum_stuff_offset = stuff; |
|
8436 |
hck->hcksum_end_offset = end; |
|
8437 |
hck->hcksum_cksum_val.inet_cksum = (uint16_t)value; |
|
8438 |
hck->hcksum_flags = flags; |
|
741
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
8439 |
} else { |
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
8440 |
rc = -1; |
0 | 8441 |
} |
8442 |
} |
|
8443 |
return (rc); |
|
8444 |
} |
|
8445 |
||
8446 |
void |
|
8447 |
hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, |
|
8448 |
uint32_t *start, uint32_t *stuff, uint32_t *end, |
|
8449 |
uint32_t *value, uint32_t *flags) |
|
8450 |
{ |
|
8451 |
ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); |
|
8452 |
if (mp->b_datap->db_type == M_DATA) { |
|
8453 |
if (flags != NULL) { |
|
741
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
8454 |
*flags = DB_CKSUMFLAGS(mp); |
0 | 8455 |
if (*flags & HCK_PARTIALCKSUM) { |
8456 |
if (start != NULL) |
|
741
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
8457 |
*start = (uint32_t)DB_CKSUMSTART(mp); |
0 | 8458 |
if (stuff != NULL) |
741
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
8459 |
*stuff = (uint32_t)DB_CKSUMSTUFF(mp); |
0 | 8460 |
if (end != NULL) |
741
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
8461 |
*end = (uint32_t)DB_CKSUMEND(mp); |
0 | 8462 |
if (value != NULL) |
741
40027a3621ac
PSARC 2005/082 Yosemite: UDP Performance Enhancement
masputra
parents:
577
diff
changeset
|
8463 |
*value = (uint32_t)DB_CKSUM16(mp); |
3115 | 8464 |
} else if ((*flags & HW_LSO) && (value != NULL)) |
8465 |
*value = (uint32_t)DB_LSOMSS(mp); |
|
0 | 8466 |
} |
8467 |
} else { |
|
8468 |
pattrinfo_t hck_attr = {PATTR_HCKSUM}; |
|
8469 |
||
8470 |
ASSERT(mmd != NULL); |
|
8471 |
||
8472 |
/* get hardware checksum attribute */ |
|
8473 |
if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) { |
|
8474 |
pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf; |
|
8475 |
||
8476 |
ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t)); |
|
8477 |
if (flags != NULL) |
|
8478 |
*flags = hck->hcksum_flags; |
|
8479 |
if (start != NULL) |
|
8480 |
*start = hck->hcksum_start_offset; |
|
8481 |
if (stuff != NULL) |
|
8482 |
*stuff = hck->hcksum_stuff_offset; |
|
8483 |
if (end != NULL) |
|
8484 |
*end = hck->hcksum_end_offset; |
|
8485 |
if (value != NULL) |
|
8486 |
*value = (uint32_t) |
|
8487 |
hck->hcksum_cksum_val.inet_cksum; |
|
8488 |
} |
|
8489 |
} |
|
8490 |
} |
|
8491 |
||
8492 |
/* |
|
8493 |
* Checksum buffer *bp for len bytes with psum partial checksum, |
|
8494 |
* or 0 if none, and return the 16 bit partial checksum. |
|
8495 |
*/ |
|
8496 |
unsigned |
|
8497 |
bcksum(uchar_t *bp, int len, unsigned int psum) |
|
8498 |
{ |
|
8499 |
int odd = len & 1; |
|
8500 |
extern unsigned int ip_ocsum(); |
|
8501 |
||
8502 |
if (((intptr_t)bp & 1) == 0 && !odd) { |
|
8503 |
/* |
|
8504 |
* Bp is 16 bit aligned and len is multiple of 16 bit word. |
|
8505 |
*/ |
|
8506 |
return (ip_ocsum((ushort_t *)bp, len >> 1, psum)); |
|
8507 |
} |
|
8508 |
if (((intptr_t)bp & 1) != 0) { |
|
8509 |
/* |
|
8510 |
* Bp isn't 16 bit aligned. |
|
8511 |
*/ |
|
8512 |
unsigned int tsum; |
|
8513 |
||
8514 |
#ifdef _LITTLE_ENDIAN |
|
8515 |
psum += *bp; |
|
8516 |
#else |
|
8517 |
psum += *bp << 8; |
|
8518 |
#endif |
|
8519 |
len--; |
|
8520 |
bp++; |
|
8521 |
tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0); |
|
8522 |
psum += (tsum << 8) & 0xffff | (tsum >> 8); |
|
8523 |
if (len & 1) { |
|
8524 |
bp += len - 1; |
|
8525 |
#ifdef _LITTLE_ENDIAN |
|
8526 |
psum += *bp << 8; |
|
8527 |
#else |
|
8528 |
psum += *bp; |
|
8529 |
#endif |
|
8530 |
} |
|
8531 |
} else { |
|
8532 |
/* |
|
8533 |
* Bp is 16 bit aligned. |
|
8534 |
*/ |
|
8535 |
psum = ip_ocsum((ushort_t *)bp, len >> 1, psum); |
|
8536 |
if (odd) { |
|
8537 |
bp += len - 1; |
|
8538 |
#ifdef _LITTLE_ENDIAN |
|
8539 |
psum += *bp; |
|
8540 |
#else |
|
8541 |
psum += *bp << 8; |
|
8542 |
#endif |
|
8543 |
} |
|
8544 |
} |
|
8545 |
/* |
|
8546 |
* Normalize psum to 16 bits before returning the new partial |
|
8547 |
* checksum. The max psum value before normalization is 0x3FDFE. |
|
8548 |
*/ |
|
8549 |
return ((psum >> 16) + (psum & 0xFFFF)); |
|
8550 |
} |
|
8551 |
||
8552 |
boolean_t |
|
8553 |
is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd) |
|
8554 |
{ |
|
8555 |
boolean_t rc; |
|
8556 |
||
8557 |
ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); |
|
8558 |
if (DB_TYPE(mp) == M_DATA) { |
|
8559 |
rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0); |
|
8560 |
} else { |
|
8561 |
pattrinfo_t zcopy_attr = {PATTR_ZCOPY}; |
|
8562 |
||
8563 |
ASSERT(mmd != NULL); |
|
8564 |
rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL); |
|
8565 |
} |
|
8566 |
return (rc); |
|
8567 |
} |
|
8568 |
||
8569 |
void |
|
8570 |
freemsgchain(mblk_t *mp) |
|
8571 |
{ |
|
8572 |
mblk_t *next; |
|
8573 |
||
8574 |
while (mp != NULL) { |
|
8575 |
next = mp->b_next; |
|
8576 |
mp->b_next = NULL; |
|
8577 |
||
8578 |
freemsg(mp); |
|
8579 |
mp = next; |
|
8580 |
} |
|
8581 |
} |
|
8582 |
||
8583 |
mblk_t * |
|
8584 |
copymsgchain(mblk_t *mp) |
|
8585 |
{ |
|
8586 |
mblk_t *nmp = NULL; |
|
8587 |
mblk_t **nmpp = &nmp; |
|
8588 |
||
8589 |
for (; mp != NULL; mp = mp->b_next) { |
|
8590 |
if ((*nmpp = copymsg(mp)) == NULL) { |
|
8591 |
freemsgchain(nmp); |
|
8592 |
return (NULL); |
|
8593 |
} |
|
8594 |
||
8595 |
nmpp = &((*nmpp)->b_next); |
|
8596 |
} |
|
8597 |
||
8598 |
return (nmp); |
|
8599 |
} |
|
8600 |
||
8601 |
/* NOTE: Do not add code after this point. */ |
|
8602 |
#undef QLOCK |
|
8603 |
||
8604 |
/* |
|
8605 |
* replacement for QLOCK macro for those that can't use it. |
|
8606 |
*/ |
|
8607 |
kmutex_t * |
|
8608 |
QLOCK(queue_t *q) |
|
8609 |
{ |
|
8610 |
return (&(q)->q_lock); |
|
8611 |
} |
|
8612 |
||
8613 |
/* |
|
8614 |
* Dummy runqueues/queuerun functions functions for backwards compatibility. |
|
8615 |
*/ |
|
8616 |
#undef runqueues |
|
8617 |
void |
|
8618 |
runqueues(void) |
|
8619 |
{ |
|
8620 |
} |
|
8621 |
||
8622 |
#undef queuerun |
|
8623 |
void |
|
8624 |
queuerun(void) |
|
8625 |
{ |
|
8626 |
} |
|
3448 | 8627 |
|
8628 |
/* |
|
8629 |
* Initialize the STR stack instance, which tracks autopush and persistent |
|
8630 |
* links. |
|
8631 |
*/ |
|
8632 |
/* ARGSUSED */ |
|
8633 |
static void * |
|
8634 |
str_stack_init(netstackid_t stackid, netstack_t *ns) |
|
8635 |
{ |
|
8636 |
str_stack_t *ss; |
|
8637 |
int i; |
|
8638 |
||
8639 |
ss = (str_stack_t *)kmem_zalloc(sizeof (*ss), KM_SLEEP); |
|
8640 |
ss->ss_netstack = ns; |
|
8641 |
||
8642 |
/* |
|
8643 |
* set up autopush |
|
8644 |
*/ |
|
8645 |
sad_initspace(ss); |
|
8646 |
||
8647 |
/* |
|
8648 |
* set up mux_node structures. |
|
8649 |
*/ |
|
8650 |
ss->ss_devcnt = devcnt; /* In case it should change before free */ |
|
8651 |
ss->ss_mux_nodes = kmem_zalloc((sizeof (struct mux_node) * |
|
8652 |
ss->ss_devcnt), KM_SLEEP); |
|
8653 |
for (i = 0; i < ss->ss_devcnt; i++) |
|
8654 |
ss->ss_mux_nodes[i].mn_imaj = i; |
|
8655 |
return (ss); |
|
8656 |
} |
|
8657 |
||
8658 |
/* |
|
8659 |
* Note: run at zone shutdown and not destroy so that the PLINKs are |
|
8660 |
* gone by the time other cleanup happens from the destroy callbacks. |
|
8661 |
*/ |
|
8662 |
static void |
|
8663 |
str_stack_shutdown(netstackid_t stackid, void *arg) |
|
8664 |
{ |
|
8665 |
str_stack_t *ss = (str_stack_t *)arg; |
|
8666 |
int i; |
|
8667 |
cred_t *cr; |
|
8668 |
||
8669 |
cr = zone_get_kcred(netstackid_to_zoneid(stackid)); |
|
8670 |
ASSERT(cr != NULL); |
|
8671 |
||
8672 |
/* Undo all the I_PLINKs for this zone */ |
|
8673 |
for (i = 0; i < ss->ss_devcnt; i++) { |
|
8674 |
struct mux_edge *ep; |
|
8675 |
ldi_handle_t lh; |
|
8676 |
ldi_ident_t li; |
|
8677 |
int ret; |
|
8678 |
int rval; |
|
8679 |
dev_t rdev; |
|
8680 |
||
8681 |
ep = ss->ss_mux_nodes[i].mn_outp; |
|
8682 |
if (ep == NULL) |
|
8683 |
continue; |
|
8684 |
ret = ldi_ident_from_major((major_t)i, &li); |
|
8685 |
if (ret != 0) { |
|
8686 |
continue; |
|
8687 |
} |
|
8688 |
rdev = ep->me_dev; |
|
8689 |
ret = ldi_open_by_dev(&rdev, OTYP_CHR, FREAD|FWRITE, |
|
8690 |
cr, &lh, li); |
|
8691 |
if (ret != 0) { |
|
8692 |
ldi_ident_release(li); |
|
8693 |
continue; |
|
8694 |
} |
|
8695 |
||
8696 |
ret = ldi_ioctl(lh, I_PUNLINK, (intptr_t)MUXID_ALL, FKIOCTL, |
|
8697 |
cr, &rval); |
|
8698 |
if (ret) { |
|
8699 |
(void) ldi_close(lh, FREAD|FWRITE, cr); |
|
8700 |
ldi_ident_release(li); |
|
8701 |
continue; |
|
8702 |
} |
|
8703 |
(void) ldi_close(lh, FREAD|FWRITE, cr); |
|
8704 |
||
8705 |
/* Close layered handles */ |
|
8706 |
ldi_ident_release(li); |
|
8707 |
} |
|
8708 |
crfree(cr); |
|
8709 |
||
8710 |
sad_freespace(ss); |
|
8711 |
||
8712 |
kmem_free(ss->ss_mux_nodes, sizeof (struct mux_node) * ss->ss_devcnt); |
|
8713 |
ss->ss_mux_nodes = NULL; |
|
8714 |
} |
|
8715 |
||
8716 |
/* |
|
8717 |
* Free the structure; str_stack_shutdown did the other cleanup work. |
|
8718 |
*/ |
|
8719 |
/* ARGSUSED */ |
|
8720 |
static void |
|
8721 |
str_stack_fini(netstackid_t stackid, void *arg) |
|
8722 |
{ |
|
8723 |
str_stack_t *ss = (str_stack_t *)arg; |
|
8724 |
||
8725 |
kmem_free(ss, sizeof (*ss)); |
|
8726 |
} |