author | maybee |
Fri, 11 Nov 2005 19:27:17 -0800 | |
changeset 896 | f5270e6bd04d |
parent 849 | 8d799fd81a9b |
child 1544 | 938876158511 |
permissions | -rw-r--r-- |
789 | 1 |
/* |
2 |
* CDDL HEADER START |
|
3 |
* |
|
4 |
* The contents of this file are subject to the terms of the |
|
5 |
* Common Development and Distribution License, Version 1.0 only |
|
6 |
* (the "License"). You may not use this file except in compliance |
|
7 |
* with the License. |
|
8 |
* |
|
9 |
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE |
|
10 |
* or http://www.opensolaris.org/os/licensing. |
|
11 |
* See the License for the specific language governing permissions |
|
12 |
* and limitations under the License. |
|
13 |
* |
|
14 |
* When distributing Covered Code, include this CDDL HEADER in each |
|
15 |
* file and include the License file at usr/src/OPENSOLARIS.LICENSE. |
|
16 |
* If applicable, add the following below this CDDL HEADER, with the |
|
17 |
* fields enclosed by brackets "[]" replaced with your own identifying |
|
18 |
* information: Portions Copyright [yyyy] [name of copyright owner] |
|
19 |
* |
|
20 |
* CDDL HEADER END |
|
21 |
*/ |
|
22 |
/* |
|
23 |
* Copyright 2005 Sun Microsystems, Inc. All rights reserved. |
|
24 |
* Use is subject to license terms. |
|
25 |
*/ |
|
26 |
||
27 |
#pragma ident "%Z%%M% %I% %E% SMI" |
|
28 |
||
29 |
#include <sys/zfs_context.h> |
|
30 |
#include <sys/spa.h> |
|
31 |
#include <sys/txg.h> |
|
32 |
#include <sys/spa_impl.h> |
|
33 |
#include <sys/vdev_impl.h> |
|
34 |
#include <sys/zio_impl.h> |
|
35 |
#include <sys/zio_compress.h> |
|
36 |
#include <sys/zio_checksum.h> |
|
37 |
||
38 |
static void zio_vdev_io_enter(zio_t *zio); |
|
39 |
static void zio_vdev_io_exit(zio_t *zio); |
|
40 |
||
41 |
/* |
|
42 |
* ========================================================================== |
|
43 |
* I/O priority table |
|
44 |
* ========================================================================== |
|
45 |
*/ |
|
46 |
uint8_t zio_priority_table[ZIO_PRIORITY_TABLE_SIZE] = { |
|
47 |
0, /* ZIO_PRIORITY_NOW */ |
|
48 |
0, /* ZIO_PRIORITY_SYNC_READ */ |
|
49 |
0, /* ZIO_PRIORITY_SYNC_WRITE */ |
|
50 |
6, /* ZIO_PRIORITY_ASYNC_READ */ |
|
51 |
4, /* ZIO_PRIORITY_ASYNC_WRITE */ |
|
52 |
4, /* ZIO_PRIORITY_FREE */ |
|
53 |
0, /* ZIO_PRIORITY_CACHE_FILL */ |
|
54 |
0, /* ZIO_PRIORITY_LOG_WRITE */ |
|
55 |
10, /* ZIO_PRIORITY_RESILVER */ |
|
56 |
20, /* ZIO_PRIORITY_SCRUB */ |
|
57 |
}; |
|
58 |
||
59 |
/* |
|
60 |
* ========================================================================== |
|
61 |
* I/O type descriptions |
|
62 |
* ========================================================================== |
|
63 |
*/ |
|
64 |
char *zio_type_name[ZIO_TYPES] = { |
|
65 |
"null", "read", "write", "free", "claim", "ioctl" }; |
|
66 |
||
67 |
/* At or above this size, force gang blocking - for testing */ |
|
68 |
uint64_t zio_gang_bang = SPA_MAXBLOCKSIZE + 1; |
|
69 |
||
70 |
typedef struct zio_sync_pass { |
|
71 |
int zp_defer_free; /* defer frees after this pass */ |
|
72 |
int zp_dontcompress; /* don't compress after this pass */ |
|
73 |
int zp_rewrite; /* rewrite new bps after this pass */ |
|
74 |
} zio_sync_pass_t; |
|
75 |
||
76 |
zio_sync_pass_t zio_sync_pass = { |
|
77 |
1, /* zp_defer_free */ |
|
78 |
4, /* zp_dontcompress */ |
|
79 |
1, /* zp_rewrite */ |
|
80 |
}; |
|
81 |
||
82 |
/* |
|
83 |
* ========================================================================== |
|
84 |
* I/O kmem caches |
|
85 |
* ========================================================================== |
|
86 |
*/ |
|
87 |
kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; |
|
88 |
||
89 |
void |
|
90 |
zio_init(void) |
|
91 |
{ |
|
92 |
size_t c; |
|
93 |
||
94 |
/* |
|
95 |
* For small buffers, we want a cache for each multiple of |
|
96 |
* SPA_MINBLOCKSIZE. For medium-size buffers, we want a cache |
|
97 |
* for each quarter-power of 2. For large buffers, we want |
|
98 |
* a cache for each multiple of PAGESIZE. |
|
99 |
*/ |
|
100 |
for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { |
|
101 |
size_t size = (c + 1) << SPA_MINBLOCKSHIFT; |
|
102 |
size_t p2 = size; |
|
103 |
size_t align = 0; |
|
104 |
||
105 |
while (p2 & (p2 - 1)) |
|
106 |
p2 &= p2 - 1; |
|
107 |
||
108 |
if (size <= 4 * SPA_MINBLOCKSIZE) { |
|
109 |
align = SPA_MINBLOCKSIZE; |
|
110 |
} else if (P2PHASE(size, PAGESIZE) == 0) { |
|
111 |
align = PAGESIZE; |
|
112 |
} else if (P2PHASE(size, p2 >> 2) == 0) { |
|
113 |
align = p2 >> 2; |
|
114 |
} |
|
115 |
||
116 |
if (align != 0) { |
|
117 |
char name[30]; |
|
118 |
(void) sprintf(name, "zio_buf_%lu", size); |
|
119 |
zio_buf_cache[c] = kmem_cache_create(name, size, |
|
849
8d799fd81a9b
6345023 /dev/zfs fails to open once ZFS module is unloaded
bonwick
parents:
789
diff
changeset
|
120 |
align, NULL, NULL, NULL, NULL, NULL, KMC_NODEBUG); |
789 | 121 |
dprintf("creating cache for size %5lx align %5lx\n", |
122 |
size, align); |
|
123 |
} |
|
124 |
} |
|
125 |
||
126 |
while (--c != 0) { |
|
127 |
ASSERT(zio_buf_cache[c] != NULL); |
|
128 |
if (zio_buf_cache[c - 1] == NULL) |
|
129 |
zio_buf_cache[c - 1] = zio_buf_cache[c]; |
|
130 |
} |
|
131 |
} |
|
132 |
||
133 |
void |
|
134 |
zio_fini(void) |
|
135 |
{ |
|
136 |
size_t c; |
|
137 |
kmem_cache_t *last_cache = NULL; |
|
138 |
||
139 |
for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { |
|
140 |
if (zio_buf_cache[c] != last_cache) { |
|
141 |
last_cache = zio_buf_cache[c]; |
|
142 |
kmem_cache_destroy(zio_buf_cache[c]); |
|
143 |
} |
|
144 |
zio_buf_cache[c] = NULL; |
|
145 |
} |
|
146 |
} |
|
147 |
||
148 |
/* |
|
149 |
* ========================================================================== |
|
150 |
* Allocate and free I/O buffers |
|
151 |
* ========================================================================== |
|
152 |
*/ |
|
153 |
void * |
|
154 |
zio_buf_alloc(size_t size) |
|
155 |
{ |
|
156 |
size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; |
|
157 |
||
158 |
ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); |
|
159 |
||
160 |
return (kmem_cache_alloc(zio_buf_cache[c], KM_SLEEP)); |
|
161 |
} |
|
162 |
||
163 |
void |
|
164 |
zio_buf_free(void *buf, size_t size) |
|
165 |
{ |
|
166 |
size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; |
|
167 |
||
168 |
ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); |
|
169 |
||
170 |
kmem_cache_free(zio_buf_cache[c], buf); |
|
171 |
} |
|
172 |
||
173 |
/* |
|
174 |
* ========================================================================== |
|
175 |
* Push and pop I/O transform buffers |
|
176 |
* ========================================================================== |
|
177 |
*/ |
|
178 |
static void |
|
179 |
zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize) |
|
180 |
{ |
|
181 |
zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); |
|
182 |
||
183 |
zt->zt_data = data; |
|
184 |
zt->zt_size = size; |
|
185 |
zt->zt_bufsize = bufsize; |
|
186 |
||
187 |
zt->zt_next = zio->io_transform_stack; |
|
188 |
zio->io_transform_stack = zt; |
|
189 |
||
190 |
zio->io_data = data; |
|
191 |
zio->io_size = size; |
|
192 |
} |
|
193 |
||
194 |
static void |
|
195 |
zio_pop_transform(zio_t *zio, void **data, uint64_t *size, uint64_t *bufsize) |
|
196 |
{ |
|
197 |
zio_transform_t *zt = zio->io_transform_stack; |
|
198 |
||
199 |
*data = zt->zt_data; |
|
200 |
*size = zt->zt_size; |
|
201 |
*bufsize = zt->zt_bufsize; |
|
202 |
||
203 |
zio->io_transform_stack = zt->zt_next; |
|
204 |
kmem_free(zt, sizeof (zio_transform_t)); |
|
205 |
||
206 |
if ((zt = zio->io_transform_stack) != NULL) { |
|
207 |
zio->io_data = zt->zt_data; |
|
208 |
zio->io_size = zt->zt_size; |
|
209 |
} |
|
210 |
} |
|
211 |
||
212 |
static void |
|
213 |
zio_clear_transform_stack(zio_t *zio) |
|
214 |
{ |
|
215 |
void *data; |
|
216 |
uint64_t size, bufsize; |
|
217 |
||
218 |
ASSERT(zio->io_transform_stack != NULL); |
|
219 |
||
220 |
zio_pop_transform(zio, &data, &size, &bufsize); |
|
221 |
while (zio->io_transform_stack != NULL) { |
|
222 |
zio_buf_free(data, bufsize); |
|
223 |
zio_pop_transform(zio, &data, &size, &bufsize); |
|
224 |
} |
|
225 |
} |
|
226 |
||
227 |
/* |
|
228 |
* ========================================================================== |
|
229 |
* Create the various types of I/O (read, write, free) |
|
230 |
* ========================================================================== |
|
231 |
*/ |
|
232 |
static zio_t * |
|
233 |
zio_create(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, |
|
234 |
void *data, uint64_t size, zio_done_func_t *done, void *private, |
|
235 |
zio_type_t type, int priority, int flags, uint8_t stage, uint32_t pipeline) |
|
236 |
{ |
|
237 |
zio_t *zio; |
|
238 |
||
239 |
ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); |
|
240 |
ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); |
|
241 |
||
242 |
zio = kmem_zalloc(sizeof (zio_t), KM_SLEEP); |
|
243 |
zio->io_parent = pio; |
|
244 |
zio->io_spa = spa; |
|
245 |
zio->io_txg = txg; |
|
246 |
if (bp != NULL) { |
|
247 |
zio->io_bp = bp; |
|
248 |
zio->io_bp_copy = *bp; |
|
249 |
zio->io_bp_orig = *bp; |
|
250 |
/* XXBP - Need to inherit this when it matters */ |
|
251 |
zio->io_dva_index = 0; |
|
252 |
} |
|
253 |
zio->io_done = done; |
|
254 |
zio->io_private = private; |
|
255 |
zio->io_type = type; |
|
256 |
zio->io_priority = priority; |
|
257 |
zio->io_stage = stage; |
|
258 |
zio->io_pipeline = pipeline; |
|
259 |
zio->io_async_stages = ZIO_ASYNC_PIPELINE_STAGES; |
|
260 |
zio->io_timestamp = lbolt64; |
|
261 |
zio->io_flags = flags; |
|
262 |
zio_push_transform(zio, data, size, size); |
|
263 |
||
264 |
if (pio == NULL) { |
|
265 |
if (!(flags & ZIO_FLAG_CONFIG_HELD)) |
|
266 |
spa_config_enter(zio->io_spa, RW_READER); |
|
267 |
zio->io_root = zio; |
|
268 |
} else { |
|
269 |
zio->io_root = pio->io_root; |
|
270 |
||
271 |
mutex_enter(&pio->io_lock); |
|
272 |
if (stage < ZIO_STAGE_READY) |
|
273 |
pio->io_children_notready++; |
|
274 |
pio->io_children_notdone++; |
|
275 |
zio->io_sibling_next = pio->io_child; |
|
276 |
zio->io_sibling_prev = NULL; |
|
277 |
if (pio->io_child != NULL) |
|
278 |
pio->io_child->io_sibling_prev = zio; |
|
279 |
pio->io_child = zio; |
|
280 |
mutex_exit(&pio->io_lock); |
|
281 |
} |
|
282 |
||
283 |
return (zio); |
|
284 |
} |
|
285 |
||
286 |
zio_t * |
|
287 |
zio_null(zio_t *pio, spa_t *spa, zio_done_func_t *done, void *private, |
|
288 |
int flags) |
|
289 |
{ |
|
290 |
zio_t *zio; |
|
291 |
||
292 |
zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, |
|
293 |
ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, ZIO_STAGE_OPEN, |
|
294 |
ZIO_WAIT_FOR_CHILDREN_PIPELINE); |
|
295 |
||
296 |
return (zio); |
|
297 |
} |
|
298 |
||
299 |
zio_t * |
|
300 |
zio_root(spa_t *spa, zio_done_func_t *done, void *private, int flags) |
|
301 |
{ |
|
302 |
return (zio_null(NULL, spa, done, private, flags)); |
|
303 |
} |
|
304 |
||
305 |
zio_t * |
|
306 |
zio_read(zio_t *pio, spa_t *spa, blkptr_t *bp, void *data, |
|
307 |
uint64_t size, zio_done_func_t *done, void *private, |
|
308 |
int priority, int flags) |
|
309 |
{ |
|
310 |
zio_t *zio; |
|
311 |
dva_t *dva; |
|
312 |
||
313 |
ASSERT3U(size, ==, BP_GET_LSIZE(bp)); |
|
314 |
||
315 |
zio = zio_create(pio, spa, bp->blk_birth, bp, data, size, done, private, |
|
316 |
ZIO_TYPE_READ, priority, flags, ZIO_STAGE_OPEN, ZIO_READ_PIPELINE); |
|
317 |
||
318 |
/* |
|
319 |
* Work off our copy of the bp so the caller can free it. |
|
320 |
*/ |
|
321 |
zio->io_bp = &zio->io_bp_copy; |
|
322 |
||
323 |
bp = zio->io_bp; |
|
324 |
dva = ZIO_GET_DVA(zio); |
|
325 |
||
326 |
if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) { |
|
327 |
uint64_t csize = BP_GET_PSIZE(bp); |
|
328 |
void *cbuf = zio_buf_alloc(csize); |
|
329 |
||
330 |
zio_push_transform(zio, cbuf, csize, csize); |
|
331 |
zio->io_pipeline |= 1U << ZIO_STAGE_READ_DECOMPRESS; |
|
332 |
} |
|
333 |
||
334 |
if (DVA_GET_GANG(dva)) { |
|
335 |
uint64_t gsize = SPA_GANGBLOCKSIZE; |
|
336 |
void *gbuf = zio_buf_alloc(gsize); |
|
337 |
||
338 |
zio_push_transform(zio, gbuf, gsize, gsize); |
|
339 |
zio->io_pipeline |= 1U << ZIO_STAGE_READ_GANG_MEMBERS; |
|
340 |
} |
|
341 |
||
342 |
return (zio); |
|
343 |
} |
|
344 |
||
345 |
zio_t * |
|
346 |
zio_write(zio_t *pio, spa_t *spa, int checksum, int compress, |
|
347 |
uint64_t txg, blkptr_t *bp, void *data, uint64_t size, |
|
348 |
zio_done_func_t *done, void *private, int priority, int flags) |
|
349 |
{ |
|
350 |
zio_t *zio; |
|
351 |
||
352 |
ASSERT(checksum >= ZIO_CHECKSUM_OFF && |
|
353 |
checksum < ZIO_CHECKSUM_FUNCTIONS); |
|
354 |
||
355 |
ASSERT(compress >= ZIO_COMPRESS_OFF && |
|
356 |
compress < ZIO_COMPRESS_FUNCTIONS); |
|
357 |
||
358 |
zio = zio_create(pio, spa, txg, bp, data, size, done, private, |
|
359 |
ZIO_TYPE_WRITE, priority, flags, |
|
360 |
ZIO_STAGE_OPEN, ZIO_WRITE_PIPELINE); |
|
361 |
||
362 |
zio->io_checksum = checksum; |
|
363 |
zio->io_compress = compress; |
|
364 |
||
365 |
if (compress != ZIO_COMPRESS_OFF) |
|
366 |
zio->io_async_stages |= 1U << ZIO_STAGE_WRITE_COMPRESS; |
|
367 |
||
368 |
if (bp->blk_birth != txg) { |
|
369 |
/* XXX the bp usually (always?) gets re-zeroed later */ |
|
370 |
BP_ZERO(bp); |
|
371 |
BP_SET_LSIZE(bp, size); |
|
372 |
BP_SET_PSIZE(bp, size); |
|
373 |
} |
|
374 |
||
375 |
return (zio); |
|
376 |
} |
|
377 |
||
378 |
zio_t * |
|
379 |
zio_rewrite(zio_t *pio, spa_t *spa, int checksum, |
|
380 |
uint64_t txg, blkptr_t *bp, void *data, uint64_t size, |
|
381 |
zio_done_func_t *done, void *private, int priority, int flags) |
|
382 |
{ |
|
383 |
zio_t *zio; |
|
384 |
||
385 |
/* XXBP - We need to re-evaluate when to insert pipeline stages */ |
|
386 |
zio = zio_create(pio, spa, txg, bp, data, size, done, private, |
|
387 |
ZIO_TYPE_WRITE, priority, flags, |
|
388 |
ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE); |
|
389 |
||
390 |
zio->io_checksum = checksum; |
|
391 |
zio->io_compress = ZIO_COMPRESS_OFF; |
|
392 |
||
393 |
return (zio); |
|
394 |
} |
|
395 |
||
396 |
static zio_t * |
|
397 |
zio_write_allocate(zio_t *pio, spa_t *spa, int checksum, |
|
398 |
uint64_t txg, blkptr_t *bp, void *data, uint64_t size, |
|
399 |
zio_done_func_t *done, void *private, int priority, int flags) |
|
400 |
{ |
|
401 |
zio_t *zio; |
|
402 |
||
403 |
BP_ZERO(bp); |
|
404 |
BP_SET_LSIZE(bp, size); |
|
405 |
BP_SET_PSIZE(bp, size); |
|
406 |
BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); |
|
407 |
||
408 |
zio = zio_create(pio, spa, txg, bp, data, size, done, private, |
|
409 |
ZIO_TYPE_WRITE, priority, flags, |
|
410 |
ZIO_STAGE_OPEN, ZIO_WRITE_ALLOCATE_PIPELINE); |
|
411 |
||
412 |
zio->io_checksum = checksum; |
|
413 |
zio->io_compress = ZIO_COMPRESS_OFF; |
|
414 |
||
415 |
return (zio); |
|
416 |
} |
|
417 |
||
418 |
zio_t * |
|
419 |
zio_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, |
|
420 |
zio_done_func_t *done, void *private) |
|
421 |
{ |
|
422 |
zio_t *zio; |
|
423 |
||
424 |
ASSERT(!BP_IS_HOLE(bp)); |
|
425 |
||
426 |
if (txg == spa->spa_syncing_txg && |
|
427 |
spa->spa_sync_pass > zio_sync_pass.zp_defer_free) { |
|
428 |
bplist_enqueue_deferred(&spa->spa_sync_bplist, bp); |
|
429 |
return (zio_null(pio, spa, NULL, NULL, 0)); |
|
430 |
} |
|
431 |
||
432 |
/* XXBP - We need to re-evaluate when to insert pipeline stages */ |
|
433 |
zio = zio_create(pio, spa, txg, bp, NULL, 0, done, private, |
|
434 |
ZIO_TYPE_FREE, ZIO_PRIORITY_FREE, 0, |
|
435 |
ZIO_STAGE_OPEN, ZIO_FREE_PIPELINE); |
|
436 |
||
437 |
zio->io_bp = &zio->io_bp_copy; |
|
438 |
||
439 |
return (zio); |
|
440 |
} |
|
441 |
||
442 |
zio_t * |
|
443 |
zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, |
|
444 |
zio_done_func_t *done, void *private) |
|
445 |
{ |
|
446 |
zio_t *zio; |
|
447 |
||
448 |
/* |
|
449 |
* A claim is an allocation of a specific block. Claims are needed |
|
450 |
* to support immediate writes in the intent log. The issue is that |
|
451 |
* immediate writes contain committed data, but in a txg that was |
|
452 |
* *not* committed. Upon opening the pool after an unclean shutdown, |
|
453 |
* the intent log claims all blocks that contain immediate write data |
|
454 |
* so that the SPA knows they're in use. |
|
455 |
* |
|
456 |
* All claims *must* be resolved in the first txg -- before the SPA |
|
457 |
* starts allocating blocks -- so that nothing is allocated twice. |
|
458 |
*/ |
|
459 |
ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa)); |
|
460 |
ASSERT3U(spa_first_txg(spa), <=, txg); |
|
461 |
||
462 |
/* XXBP - We need to re-evaluate when to insert pipeline stages */ |
|
463 |
zio = zio_create(pio, spa, txg, bp, NULL, 0, done, private, |
|
464 |
ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, 0, |
|
465 |
ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE); |
|
466 |
||
467 |
zio->io_bp = &zio->io_bp_copy; |
|
468 |
||
469 |
return (zio); |
|
470 |
} |
|
471 |
||
472 |
zio_t * |
|
473 |
zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, |
|
474 |
zio_done_func_t *done, void *private, int priority, int flags) |
|
475 |
{ |
|
476 |
zio_t *zio; |
|
477 |
int c; |
|
478 |
||
479 |
if (vd->vdev_children == 0) { |
|
480 |
zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, |
|
481 |
ZIO_TYPE_IOCTL, priority, flags, |
|
482 |
ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); |
|
483 |
||
484 |
zio->io_vd = vd; |
|
485 |
zio->io_cmd = cmd; |
|
486 |
} else { |
|
487 |
zio = zio_null(pio, spa, NULL, NULL, flags); |
|
488 |
||
489 |
for (c = 0; c < vd->vdev_children; c++) |
|
490 |
zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, |
|
491 |
done, private, priority, flags)); |
|
492 |
} |
|
493 |
||
494 |
return (zio); |
|
495 |
} |
|
496 |
||
497 |
static void |
|
498 |
zio_phys_bp_init(vdev_t *vd, blkptr_t *bp, uint64_t offset, uint64_t size, |
|
499 |
int checksum) |
|
500 |
{ |
|
501 |
ASSERT(vd->vdev_children == 0); |
|
502 |
||
503 |
ASSERT(size <= SPA_MAXBLOCKSIZE); |
|
504 |
ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); |
|
505 |
ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); |
|
506 |
||
507 |
ASSERT(offset + size <= VDEV_LABEL_START_SIZE || |
|
508 |
offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); |
|
509 |
ASSERT3U(offset + size, <=, vd->vdev_psize); |
|
510 |
||
511 |
BP_ZERO(bp); |
|
512 |
||
513 |
BP_SET_LSIZE(bp, size); |
|
514 |
BP_SET_PSIZE(bp, size); |
|
515 |
||
516 |
BP_SET_CHECKSUM(bp, checksum); |
|
517 |
BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); |
|
518 |
BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); |
|
519 |
||
520 |
if (checksum != ZIO_CHECKSUM_OFF) |
|
521 |
ZIO_SET_CHECKSUM(&bp->blk_cksum, offset, 0, 0, 0); |
|
522 |
} |
|
523 |
||
524 |
zio_t * |
|
525 |
zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, |
|
526 |
void *data, int checksum, zio_done_func_t *done, void *private, |
|
527 |
int priority, int flags) |
|
528 |
{ |
|
529 |
zio_t *zio; |
|
530 |
blkptr_t blk; |
|
531 |
||
532 |
zio_phys_bp_init(vd, &blk, offset, size, checksum); |
|
533 |
||
534 |
zio = zio_create(pio, vd->vdev_spa, 0, &blk, data, size, done, private, |
|
535 |
ZIO_TYPE_READ, priority, flags | ZIO_FLAG_PHYSICAL, |
|
536 |
ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); |
|
537 |
||
538 |
zio->io_vd = vd; |
|
539 |
zio->io_offset = offset; |
|
540 |
||
541 |
/* |
|
542 |
* Work off our copy of the bp so the caller can free it. |
|
543 |
*/ |
|
544 |
zio->io_bp = &zio->io_bp_copy; |
|
545 |
||
546 |
return (zio); |
|
547 |
} |
|
548 |
||
549 |
zio_t * |
|
550 |
zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, |
|
551 |
void *data, int checksum, zio_done_func_t *done, void *private, |
|
552 |
int priority, int flags) |
|
553 |
{ |
|
554 |
zio_block_tail_t *zbt; |
|
555 |
void *wbuf; |
|
556 |
zio_t *zio; |
|
557 |
blkptr_t blk; |
|
558 |
||
559 |
zio_phys_bp_init(vd, &blk, offset, size, checksum); |
|
560 |
||
561 |
zio = zio_create(pio, vd->vdev_spa, 0, &blk, data, size, done, private, |
|
562 |
ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_PHYSICAL, |
|
563 |
ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); |
|
564 |
||
565 |
zio->io_vd = vd; |
|
566 |
zio->io_offset = offset; |
|
567 |
||
568 |
zio->io_bp = &zio->io_bp_copy; |
|
569 |
zio->io_checksum = checksum; |
|
570 |
||
571 |
if (zio_checksum_table[checksum].ci_zbt) { |
|
572 |
/* |
|
573 |
* zbt checksums are necessarily destructive -- they modify |
|
574 |
* one word of the write buffer to hold the verifier/checksum. |
|
575 |
* Therefore, we must make a local copy in case the data is |
|
576 |
* being written to multiple places. |
|
577 |
*/ |
|
578 |
wbuf = zio_buf_alloc(size); |
|
579 |
bcopy(data, wbuf, size); |
|
580 |
zio_push_transform(zio, wbuf, size, size); |
|
581 |
||
582 |
zbt = (zio_block_tail_t *)((char *)wbuf + size) - 1; |
|
583 |
zbt->zbt_cksum = blk.blk_cksum; |
|
584 |
} |
|
585 |
||
586 |
return (zio); |
|
587 |
} |
|
588 |
||
589 |
/* |
|
590 |
* Create a child I/O to do some work for us. It has no associated bp. |
|
591 |
*/ |
|
592 |
zio_t * |
|
593 |
zio_vdev_child_io(zio_t *zio, blkptr_t *bp, vdev_t *vd, uint64_t offset, |
|
594 |
void *data, uint64_t size, int type, int priority, int flags, |
|
595 |
zio_done_func_t *done, void *private) |
|
596 |
{ |
|
597 |
uint32_t pipeline = ZIO_VDEV_CHILD_PIPELINE; |
|
598 |
zio_t *cio; |
|
599 |
||
600 |
if (type == ZIO_TYPE_READ && bp != NULL) { |
|
601 |
/* |
|
602 |
* If we have the bp, then the child should perform the |
|
603 |
* checksum and the parent need not. This pushes error |
|
604 |
* detection as close to the leaves as possible and |
|
605 |
* eliminates redundant checksums in the interior nodes. |
|
606 |
*/ |
|
607 |
pipeline |= 1U << ZIO_STAGE_CHECKSUM_VERIFY; |
|
608 |
zio->io_pipeline &= ~(1U << ZIO_STAGE_CHECKSUM_VERIFY); |
|
609 |
} |
|
610 |
||
611 |
cio = zio_create(zio, zio->io_spa, zio->io_txg, bp, data, size, |
|
612 |
done, private, type, priority, |
|
613 |
(zio->io_flags & ZIO_FLAG_VDEV_INHERIT) | ZIO_FLAG_CANFAIL | flags, |
|
614 |
ZIO_STAGE_VDEV_IO_SETUP - 1, pipeline); |
|
615 |
||
616 |
cio->io_vd = vd; |
|
617 |
cio->io_offset = offset; |
|
618 |
||
619 |
return (cio); |
|
620 |
} |
|
621 |
||
622 |
/* |
|
623 |
* ========================================================================== |
|
624 |
* Initiate I/O, either sync or async |
|
625 |
* ========================================================================== |
|
626 |
*/ |
|
627 |
int |
|
628 |
zio_wait(zio_t *zio) |
|
629 |
{ |
|
630 |
int error; |
|
631 |
||
632 |
ASSERT(zio->io_stage == ZIO_STAGE_OPEN); |
|
633 |
||
634 |
zio->io_waiter = curthread; |
|
635 |
||
636 |
zio_next_stage_async(zio); |
|
637 |
||
638 |
mutex_enter(&zio->io_lock); |
|
639 |
while (zio->io_stalled != ZIO_STAGE_DONE) |
|
640 |
cv_wait(&zio->io_cv, &zio->io_lock); |
|
641 |
mutex_exit(&zio->io_lock); |
|
642 |
||
643 |
error = zio->io_error; |
|
644 |
||
645 |
kmem_free(zio, sizeof (zio_t)); |
|
646 |
||
647 |
return (error); |
|
648 |
} |
|
649 |
||
650 |
void |
|
651 |
zio_nowait(zio_t *zio) |
|
652 |
{ |
|
653 |
zio_next_stage_async(zio); |
|
654 |
} |
|
655 |
||
656 |
/* |
|
657 |
* ========================================================================== |
|
658 |
* I/O pipeline interlocks: parent/child dependency scoreboarding |
|
659 |
* ========================================================================== |
|
660 |
*/ |
|
661 |
static void |
|
662 |
zio_wait_for_children(zio_t *zio, uint32_t stage, uint64_t *countp) |
|
663 |
{ |
|
664 |
mutex_enter(&zio->io_lock); |
|
665 |
if (*countp == 0) { |
|
666 |
ASSERT(zio->io_stalled == 0); |
|
667 |
mutex_exit(&zio->io_lock); |
|
668 |
zio_next_stage(zio); |
|
669 |
} else { |
|
670 |
if (zio->io_stage == ZIO_STAGE_VDEV_IO_START) |
|
671 |
zio_vdev_io_exit(zio); |
|
672 |
zio->io_stalled = stage; |
|
673 |
mutex_exit(&zio->io_lock); |
|
674 |
} |
|
675 |
} |
|
676 |
||
677 |
static void |
|
678 |
zio_notify_parent(zio_t *zio, uint32_t stage, uint64_t *countp) |
|
679 |
{ |
|
680 |
zio_t *pio = zio->io_parent; |
|
681 |
||
682 |
mutex_enter(&pio->io_lock); |
|
683 |
if (pio->io_error == 0 && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) |
|
684 |
pio->io_error = zio->io_error; |
|
685 |
if (--*countp == 0 && pio->io_stalled == stage) { |
|
686 |
if (pio->io_stage == ZIO_STAGE_VDEV_IO_START) |
|
687 |
zio_vdev_io_enter(pio); |
|
688 |
pio->io_stalled = 0; |
|
689 |
mutex_exit(&pio->io_lock); |
|
690 |
zio_next_stage_async(pio); |
|
691 |
} else { |
|
692 |
mutex_exit(&pio->io_lock); |
|
693 |
} |
|
694 |
} |
|
695 |
||
696 |
static void |
|
697 |
zio_wait_children_ready(zio_t *zio) |
|
698 |
{ |
|
699 |
zio_wait_for_children(zio, ZIO_STAGE_WAIT_CHILDREN_READY, |
|
700 |
&zio->io_children_notready); |
|
701 |
} |
|
702 |
||
703 |
void |
|
704 |
zio_wait_children_done(zio_t *zio) |
|
705 |
{ |
|
706 |
zio_wait_for_children(zio, ZIO_STAGE_WAIT_CHILDREN_DONE, |
|
707 |
&zio->io_children_notdone); |
|
708 |
} |
|
709 |
||
710 |
static void |
|
711 |
zio_ready(zio_t *zio) |
|
712 |
{ |
|
713 |
zio_t *pio = zio->io_parent; |
|
714 |
||
715 |
if (pio != NULL) |
|
716 |
zio_notify_parent(zio, ZIO_STAGE_WAIT_CHILDREN_READY, |
|
717 |
&pio->io_children_notready); |
|
718 |
||
719 |
if (zio->io_bp) |
|
720 |
zio->io_bp_copy = *zio->io_bp; |
|
721 |
||
722 |
zio_next_stage(zio); |
|
723 |
} |
|
724 |
||
725 |
static void |
|
726 |
zio_done(zio_t *zio) |
|
727 |
{ |
|
728 |
zio_t *pio = zio->io_parent; |
|
729 |
spa_t *spa = zio->io_spa; |
|
730 |
blkptr_t *bp = zio->io_bp; |
|
731 |
vdev_t *vd = zio->io_vd; |
|
896 | 732 |
char blkbuf[BP_SPRINTF_LEN]; |
789 | 733 |
|
734 |
ASSERT(zio->io_children_notready == 0); |
|
735 |
ASSERT(zio->io_children_notdone == 0); |
|
736 |
||
737 |
if (bp != NULL) { |
|
738 |
ASSERT(bp->blk_pad[0] == 0); |
|
739 |
ASSERT(bp->blk_pad[1] == 0); |
|
740 |
ASSERT(bp->blk_pad[2] == 0); |
|
741 |
ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0); |
|
742 |
if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) && |
|
743 |
!(zio->io_flags & ZIO_FLAG_IO_REPAIR)) |
|
744 |
ASSERT(!BP_SHOULD_BYTESWAP(bp)); |
|
745 |
} |
|
746 |
||
747 |
if (vd != NULL) |
|
748 |
vdev_stat_update(zio); |
|
749 |
||
750 |
if (zio->io_error) { |
|
896 | 751 |
sprintf_blkptr(blkbuf, BP_SPRINTF_LEN, |
752 |
bp ? bp : &zio->io_bp_copy); |
|
789 | 753 |
dprintf("ZFS: %s (%s on %s off %llx: zio %p %s): error %d\n", |
754 |
zio->io_error == ECKSUM ? "bad checksum" : "I/O failure", |
|
755 |
zio_type_name[zio->io_type], |
|
756 |
vdev_description(vd), |
|
757 |
(u_longlong_t)zio->io_offset, |
|
758 |
zio, blkbuf, zio->io_error); |
|
759 |
} |
|
760 |
||
761 |
if (zio->io_numerrors != 0 && zio->io_type == ZIO_TYPE_WRITE) { |
|
896 | 762 |
sprintf_blkptr(blkbuf, BP_SPRINTF_LEN, |
763 |
bp ? bp : &zio->io_bp_copy); |
|
789 | 764 |
dprintf("ZFS: %s (%s on %s off %llx: zio %p %s): %d errors\n", |
765 |
"partial write", |
|
766 |
zio_type_name[zio->io_type], |
|
767 |
vdev_description(vd), |
|
768 |
(u_longlong_t)zio->io_offset, |
|
769 |
zio, blkbuf, zio->io_numerrors); |
|
770 |
} |
|
771 |
||
772 |
if (zio->io_error && !(zio->io_flags & ZIO_FLAG_CANFAIL)) { |
|
896 | 773 |
sprintf_blkptr(blkbuf, BP_SPRINTF_LEN, |
774 |
bp ? bp : &zio->io_bp_copy); |
|
789 | 775 |
panic("ZFS: %s (%s on %s off %llx: zio %p %s): error %d", |
776 |
zio->io_error == ECKSUM ? "bad checksum" : "I/O failure", |
|
777 |
zio_type_name[zio->io_type], |
|
778 |
vdev_description(vd), |
|
779 |
(u_longlong_t)zio->io_offset, |
|
780 |
zio, blkbuf, zio->io_error); |
|
781 |
} |
|
782 |
||
783 |
zio_clear_transform_stack(zio); |
|
784 |
||
785 |
if (zio->io_done) |
|
786 |
zio->io_done(zio); |
|
787 |
||
788 |
ASSERT(zio->io_delegate_list == NULL); |
|
789 |
ASSERT(zio->io_delegate_next == NULL); |
|
790 |
||
791 |
if (pio != NULL) { |
|
792 |
zio_t *next, *prev; |
|
793 |
||
794 |
mutex_enter(&pio->io_lock); |
|
795 |
next = zio->io_sibling_next; |
|
796 |
prev = zio->io_sibling_prev; |
|
797 |
if (next != NULL) |
|
798 |
next->io_sibling_prev = prev; |
|
799 |
if (prev != NULL) |
|
800 |
prev->io_sibling_next = next; |
|
801 |
if (pio->io_child == zio) |
|
802 |
pio->io_child = next; |
|
803 |
mutex_exit(&pio->io_lock); |
|
804 |
||
805 |
zio_notify_parent(zio, ZIO_STAGE_WAIT_CHILDREN_DONE, |
|
806 |
&pio->io_children_notdone); |
|
807 |
} |
|
808 |
||
809 |
if (pio == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_HELD)) |
|
810 |
spa_config_exit(spa); |
|
811 |
||
812 |
if (zio->io_waiter != NULL) { |
|
813 |
mutex_enter(&zio->io_lock); |
|
814 |
ASSERT(zio->io_stage == ZIO_STAGE_DONE); |
|
815 |
zio->io_stalled = zio->io_stage; |
|
816 |
cv_broadcast(&zio->io_cv); |
|
817 |
mutex_exit(&zio->io_lock); |
|
818 |
} else { |
|
819 |
kmem_free(zio, sizeof (zio_t)); |
|
820 |
} |
|
821 |
} |
|
822 |
||
823 |
/* |
|
824 |
* ========================================================================== |
|
825 |
* Compression support |
|
826 |
* ========================================================================== |
|
827 |
*/ |
|
828 |
static void |
|
829 |
zio_write_compress(zio_t *zio) |
|
830 |
{ |
|
831 |
int compress = zio->io_compress; |
|
832 |
blkptr_t *bp = zio->io_bp; |
|
833 |
void *cbuf; |
|
834 |
uint64_t lsize = zio->io_size; |
|
835 |
uint64_t csize = lsize; |
|
836 |
uint64_t cbufsize = 0; |
|
837 |
int pass; |
|
838 |
||
839 |
if (bp->blk_birth == zio->io_txg) { |
|
840 |
/* |
|
841 |
* We're rewriting an existing block, which means we're |
|
842 |
* working on behalf of spa_sync(). For spa_sync() to |
|
843 |
* converge, it must eventually be the case that we don't |
|
844 |
* have to allocate new blocks. But compression changes |
|
845 |
* the blocksize, which forces a reallocate, and makes |
|
846 |
* convergence take longer. Therefore, after the first |
|
847 |
* few passes, stop compressing to ensure convergence. |
|
848 |
*/ |
|
849 |
pass = spa_sync_pass(zio->io_spa); |
|
850 |
if (pass > zio_sync_pass.zp_dontcompress) |
|
851 |
compress = ZIO_COMPRESS_OFF; |
|
852 |
} else { |
|
853 |
ASSERT(BP_IS_HOLE(bp)); |
|
854 |
pass = 1; |
|
855 |
} |
|
856 |
||
857 |
if (compress != ZIO_COMPRESS_OFF) |
|
858 |
if (!zio_compress_data(compress, zio->io_data, zio->io_size, |
|
859 |
&cbuf, &csize, &cbufsize)) |
|
860 |
compress = ZIO_COMPRESS_OFF; |
|
861 |
||
862 |
if (compress != ZIO_COMPRESS_OFF && csize != 0) |
|
863 |
zio_push_transform(zio, cbuf, csize, cbufsize); |
|
864 |
||
865 |
/* |
|
866 |
* The final pass of spa_sync() must be all rewrites, but the first |
|
867 |
* few passes offer a trade-off: allocating blocks defers convergence, |
|
868 |
* but newly allocated blocks are sequential, so they can be written |
|
869 |
* to disk faster. Therefore, we allow the first few passes of |
|
870 |
* spa_sync() to reallocate new blocks, but force rewrites after that. |
|
871 |
* There should only be a handful of blocks after pass 1 in any case. |
|
872 |
*/ |
|
873 |
if (bp->blk_birth == zio->io_txg && BP_GET_PSIZE(bp) == csize && |
|
874 |
pass > zio_sync_pass.zp_rewrite) { |
|
875 |
ASSERT(csize != 0); |
|
876 |
ASSERT3U(BP_GET_COMPRESS(bp), ==, compress); |
|
877 |
ASSERT3U(BP_GET_LSIZE(bp), ==, lsize); |
|
878 |
||
879 |
zio->io_pipeline = ZIO_REWRITE_PIPELINE; |
|
880 |
} else { |
|
881 |
if (bp->blk_birth == zio->io_txg) { |
|
882 |
ASSERT3U(BP_GET_LSIZE(bp), ==, lsize); |
|
883 |
bzero(bp, sizeof (blkptr_t)); |
|
884 |
} |
|
885 |
if (csize == 0) { |
|
886 |
BP_ZERO(bp); |
|
887 |
zio->io_pipeline = ZIO_WAIT_FOR_CHILDREN_PIPELINE; |
|
888 |
} else { |
|
889 |
BP_SET_LSIZE(bp, lsize); |
|
890 |
BP_SET_PSIZE(bp, csize); |
|
891 |
BP_SET_COMPRESS(bp, compress); |
|
892 |
zio->io_pipeline = ZIO_WRITE_ALLOCATE_PIPELINE; |
|
893 |
} |
|
894 |
} |
|
895 |
||
896 |
zio_next_stage(zio); |
|
897 |
} |
|
898 |
||
899 |
static void |
|
900 |
zio_read_decompress(zio_t *zio) |
|
901 |
{ |
|
902 |
blkptr_t *bp = zio->io_bp; |
|
903 |
void *data; |
|
904 |
uint64_t size; |
|
905 |
uint64_t bufsize; |
|
906 |
int compress = BP_GET_COMPRESS(bp); |
|
907 |
||
908 |
ASSERT(compress != ZIO_COMPRESS_OFF); |
|
909 |
||
910 |
zio_pop_transform(zio, &data, &size, &bufsize); |
|
911 |
||
912 |
if (zio_decompress_data(compress, data, size, |
|
913 |
zio->io_data, zio->io_size)) |
|
914 |
zio->io_error = EIO; |
|
915 |
||
916 |
zio_buf_free(data, bufsize); |
|
917 |
||
918 |
zio_next_stage(zio); |
|
919 |
} |
|
920 |
||
921 |
/* |
|
922 |
* ========================================================================== |
|
923 |
* Gang block support |
|
924 |
* ========================================================================== |
|
925 |
*/ |
|
926 |
static void |
|
927 |
zio_gang_pipeline(zio_t *zio) |
|
928 |
{ |
|
929 |
/* |
|
930 |
* By default, the pipeline assumes that we're dealing with a gang |
|
931 |
* block. If we're not, strip out any gang-specific stages. |
|
932 |
*/ |
|
933 |
if (!DVA_GET_GANG(ZIO_GET_DVA(zio))) |
|
934 |
zio->io_pipeline &= ~ZIO_GANG_STAGES; |
|
935 |
||
936 |
zio_next_stage(zio); |
|
937 |
} |
|
938 |
||
939 |
static void |
|
940 |
zio_gang_byteswap(zio_t *zio) |
|
941 |
{ |
|
942 |
ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); |
|
943 |
||
944 |
if (BP_SHOULD_BYTESWAP(zio->io_bp)) |
|
945 |
byteswap_uint64_array(zio->io_data, zio->io_size); |
|
946 |
} |
|
947 |
||
948 |
static void |
|
949 |
zio_get_gang_header(zio_t *zio) |
|
950 |
{ |
|
951 |
blkptr_t *bp = zio->io_bp; |
|
952 |
uint64_t gsize = SPA_GANGBLOCKSIZE; |
|
953 |
void *gbuf = zio_buf_alloc(gsize); |
|
954 |
||
955 |
ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); |
|
956 |
||
957 |
zio_push_transform(zio, gbuf, gsize, gsize); |
|
958 |
||
959 |
zio_nowait(zio_create(zio, zio->io_spa, bp->blk_birth, bp, gbuf, gsize, |
|
960 |
NULL, NULL, ZIO_TYPE_READ, zio->io_priority, |
|
961 |
zio->io_flags & ZIO_FLAG_GANG_INHERIT, |
|
962 |
ZIO_STAGE_OPEN, ZIO_READ_PIPELINE)); |
|
963 |
||
964 |
zio_wait_children_done(zio); |
|
965 |
} |
|
966 |
||
967 |
static void |
|
968 |
zio_read_gang_members(zio_t *zio) |
|
969 |
{ |
|
970 |
zio_gbh_phys_t *gbh; |
|
971 |
uint64_t gsize, gbufsize, loff, lsize; |
|
972 |
int i; |
|
973 |
||
974 |
ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); |
|
975 |
||
976 |
zio_gang_byteswap(zio); |
|
977 |
zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); |
|
978 |
||
979 |
for (loff = 0, i = 0; loff != zio->io_size; loff += lsize, i++) { |
|
980 |
blkptr_t *gbp = &gbh->zg_blkptr[i]; |
|
981 |
lsize = BP_GET_PSIZE(gbp); |
|
982 |
||
983 |
ASSERT(BP_GET_COMPRESS(gbp) == ZIO_COMPRESS_OFF); |
|
984 |
ASSERT3U(lsize, ==, BP_GET_LSIZE(gbp)); |
|
985 |
ASSERT3U(loff + lsize, <=, zio->io_size); |
|
986 |
ASSERT(i < SPA_GBH_NBLKPTRS); |
|
987 |
ASSERT(!BP_IS_HOLE(gbp)); |
|
988 |
||
989 |
zio_nowait(zio_read(zio, zio->io_spa, gbp, |
|
990 |
(char *)zio->io_data + loff, lsize, NULL, NULL, |
|
991 |
zio->io_priority, zio->io_flags & ZIO_FLAG_GANG_INHERIT)); |
|
992 |
} |
|
993 |
||
994 |
zio_buf_free(gbh, gbufsize); |
|
995 |
zio_wait_children_done(zio); |
|
996 |
} |
|
997 |
||
998 |
static void |
|
999 |
zio_rewrite_gang_members(zio_t *zio) |
|
1000 |
{ |
|
1001 |
zio_gbh_phys_t *gbh; |
|
1002 |
uint64_t gsize, gbufsize, loff, lsize; |
|
1003 |
int i; |
|
1004 |
||
1005 |
ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); |
|
1006 |
ASSERT3U(zio->io_size, ==, SPA_GANGBLOCKSIZE); |
|
1007 |
||
1008 |
zio_gang_byteswap(zio); |
|
1009 |
zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); |
|
1010 |
||
1011 |
ASSERT(gsize == gbufsize); |
|
1012 |
||
1013 |
for (loff = 0, i = 0; loff != zio->io_size; loff += lsize, i++) { |
|
1014 |
blkptr_t *gbp = &gbh->zg_blkptr[i]; |
|
1015 |
lsize = BP_GET_PSIZE(gbp); |
|
1016 |
||
1017 |
ASSERT(BP_GET_COMPRESS(gbp) == ZIO_COMPRESS_OFF); |
|
1018 |
ASSERT3U(lsize, ==, BP_GET_LSIZE(gbp)); |
|
1019 |
ASSERT3U(loff + lsize, <=, zio->io_size); |
|
1020 |
ASSERT(i < SPA_GBH_NBLKPTRS); |
|
1021 |
ASSERT(!BP_IS_HOLE(gbp)); |
|
1022 |
||
1023 |
zio_nowait(zio_rewrite(zio, zio->io_spa, zio->io_checksum, |
|
1024 |
zio->io_txg, gbp, (char *)zio->io_data + loff, lsize, |
|
1025 |
NULL, NULL, zio->io_priority, zio->io_flags)); |
|
1026 |
} |
|
1027 |
||
1028 |
zio_push_transform(zio, gbh, gsize, gbufsize); |
|
1029 |
zio_wait_children_ready(zio); |
|
1030 |
} |
|
1031 |
||
1032 |
static void |
|
1033 |
zio_free_gang_members(zio_t *zio) |
|
1034 |
{ |
|
1035 |
zio_gbh_phys_t *gbh; |
|
1036 |
uint64_t gsize, gbufsize; |
|
1037 |
int i; |
|
1038 |
||
1039 |
ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); |
|
1040 |
||
1041 |
zio_gang_byteswap(zio); |
|
1042 |
zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); |
|
1043 |
||
1044 |
for (i = 0; i < SPA_GBH_NBLKPTRS; i++) { |
|
1045 |
blkptr_t *gbp = &gbh->zg_blkptr[i]; |
|
1046 |
||
1047 |
if (BP_IS_HOLE(gbp)) |
|
1048 |
continue; |
|
1049 |
zio_nowait(zio_free(zio, zio->io_spa, zio->io_txg, |
|
1050 |
gbp, NULL, NULL)); |
|
1051 |
} |
|
1052 |
||
1053 |
zio_buf_free(gbh, gbufsize); |
|
1054 |
zio_next_stage(zio); |
|
1055 |
} |
|
1056 |
||
1057 |
static void |
|
1058 |
zio_claim_gang_members(zio_t *zio) |
|
1059 |
{ |
|
1060 |
zio_gbh_phys_t *gbh; |
|
1061 |
uint64_t gsize, gbufsize; |
|
1062 |
int i; |
|
1063 |
||
1064 |
ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); |
|
1065 |
||
1066 |
zio_gang_byteswap(zio); |
|
1067 |
zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); |
|
1068 |
||
1069 |
for (i = 0; i < SPA_GBH_NBLKPTRS; i++) { |
|
1070 |
blkptr_t *gbp = &gbh->zg_blkptr[i]; |
|
1071 |
if (BP_IS_HOLE(gbp)) |
|
1072 |
continue; |
|
1073 |
zio_nowait(zio_claim(zio, zio->io_spa, zio->io_txg, |
|
1074 |
gbp, NULL, NULL)); |
|
1075 |
} |
|
1076 |
||
1077 |
zio_buf_free(gbh, gbufsize); |
|
1078 |
zio_next_stage(zio); |
|
1079 |
} |
|
1080 |
||
1081 |
static void |
|
1082 |
zio_write_allocate_gang_member_done(zio_t *zio) |
|
1083 |
{ |
|
1084 |
zio_t *pio = zio->io_parent; |
|
1085 |
dva_t *cdva = ZIO_GET_DVA(zio); |
|
1086 |
dva_t *pdva = ZIO_GET_DVA(pio); |
|
1087 |
uint64_t asize; |
|
1088 |
||
1089 |
ASSERT(DVA_GET_GANG(pdva)); |
|
1090 |
||
1091 |
/* XXBP - Need to be careful here with multiple DVAs */ |
|
1092 |
mutex_enter(&pio->io_lock); |
|
1093 |
asize = DVA_GET_ASIZE(pdva); |
|
1094 |
asize += DVA_GET_ASIZE(cdva); |
|
1095 |
DVA_SET_ASIZE(pdva, asize); |
|
1096 |
mutex_exit(&pio->io_lock); |
|
1097 |
} |
|
1098 |
||
1099 |
static void |
|
1100 |
zio_write_allocate_gang_members(zio_t *zio) |
|
1101 |
{ |
|
1102 |
blkptr_t *bp = zio->io_bp; |
|
1103 |
dva_t *dva = ZIO_GET_DVA(zio); |
|
1104 |
zio_gbh_phys_t *gbh; |
|
1105 |
uint64_t resid = zio->io_size; |
|
1106 |
uint64_t maxalloc = P2ROUNDUP(zio->io_size >> 1, SPA_MINBLOCKSIZE); |
|
1107 |
uint64_t gsize, loff, lsize; |
|
1108 |
uint32_t gbps_left; |
|
1109 |
int error; |
|
1110 |
int i; |
|
1111 |
||
1112 |
gsize = SPA_GANGBLOCKSIZE; |
|
1113 |
gbps_left = SPA_GBH_NBLKPTRS; |
|
1114 |
||
1115 |
error = metaslab_alloc(zio->io_spa, gsize, dva, zio->io_txg); |
|
1116 |
if (error == ENOSPC) |
|
1117 |
panic("can't allocate gang block header"); |
|
1118 |
ASSERT(error == 0); |
|
1119 |
||
1120 |
DVA_SET_GANG(dva, 1); |
|
1121 |
||
1122 |
bp->blk_birth = zio->io_txg; |
|
1123 |
||
1124 |
gbh = zio_buf_alloc(gsize); |
|
1125 |
bzero(gbh, gsize); |
|
1126 |
||
1127 |
for (loff = 0, i = 0; loff != zio->io_size; |
|
1128 |
loff += lsize, resid -= lsize, gbps_left--, i++) { |
|
1129 |
blkptr_t *gbp = &gbh->zg_blkptr[i]; |
|
1130 |
dva = &gbp->blk_dva[0]; |
|
1131 |
||
1132 |
ASSERT(gbps_left != 0); |
|
1133 |
maxalloc = MIN(maxalloc, resid); |
|
1134 |
||
1135 |
while (resid <= maxalloc * gbps_left) { |
|
1136 |
error = metaslab_alloc(zio->io_spa, maxalloc, dva, |
|
1137 |
zio->io_txg); |
|
1138 |
if (error == 0) |
|
1139 |
break; |
|
1140 |
ASSERT3U(error, ==, ENOSPC); |
|
1141 |
if (maxalloc == SPA_MINBLOCKSIZE) |
|
1142 |
panic("really out of space"); |
|
1143 |
maxalloc = P2ROUNDUP(maxalloc >> 1, SPA_MINBLOCKSIZE); |
|
1144 |
} |
|
1145 |
||
1146 |
if (resid <= maxalloc * gbps_left) { |
|
1147 |
lsize = maxalloc; |
|
1148 |
BP_SET_LSIZE(gbp, lsize); |
|
1149 |
BP_SET_PSIZE(gbp, lsize); |
|
1150 |
BP_SET_COMPRESS(gbp, ZIO_COMPRESS_OFF); |
|
1151 |
gbp->blk_birth = zio->io_txg; |
|
1152 |
zio_nowait(zio_rewrite(zio, zio->io_spa, |
|
1153 |
zio->io_checksum, zio->io_txg, gbp, |
|
1154 |
(char *)zio->io_data + loff, lsize, |
|
1155 |
zio_write_allocate_gang_member_done, NULL, |
|
1156 |
zio->io_priority, zio->io_flags)); |
|
1157 |
} else { |
|
1158 |
lsize = P2ROUNDUP(resid / gbps_left, SPA_MINBLOCKSIZE); |
|
1159 |
ASSERT(lsize != SPA_MINBLOCKSIZE); |
|
1160 |
zio_nowait(zio_write_allocate(zio, zio->io_spa, |
|
1161 |
zio->io_checksum, zio->io_txg, gbp, |
|
1162 |
(char *)zio->io_data + loff, lsize, |
|
1163 |
zio_write_allocate_gang_member_done, NULL, |
|
1164 |
zio->io_priority, zio->io_flags)); |
|
1165 |
} |
|
1166 |
} |
|
1167 |
||
1168 |
ASSERT(resid == 0 && loff == zio->io_size); |
|
1169 |
||
1170 |
zio->io_pipeline |= 1U << ZIO_STAGE_GANG_CHECKSUM_GENERATE; |
|
1171 |
||
1172 |
zio_push_transform(zio, gbh, gsize, gsize); |
|
1173 |
zio_wait_children_done(zio); |
|
1174 |
} |
|
1175 |
||
1176 |
/* |
|
1177 |
* ========================================================================== |
|
1178 |
* Allocate and free blocks |
|
1179 |
* ========================================================================== |
|
1180 |
*/ |
|
1181 |
static void |
|
1182 |
zio_dva_allocate(zio_t *zio) |
|
1183 |
{ |
|
1184 |
blkptr_t *bp = zio->io_bp; |
|
1185 |
dva_t *dva = ZIO_GET_DVA(zio); |
|
1186 |
int error; |
|
1187 |
||
1188 |
ASSERT(BP_IS_HOLE(bp)); |
|
1189 |
||
1190 |
/* For testing, make some blocks above a certain size be gang blocks */ |
|
1191 |
if (zio->io_size >= zio_gang_bang && (lbolt & 0x3) == 0) { |
|
1192 |
zio_write_allocate_gang_members(zio); |
|
1193 |
return; |
|
1194 |
} |
|
1195 |
||
1196 |
ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); |
|
1197 |
||
1198 |
error = metaslab_alloc(zio->io_spa, zio->io_size, dva, zio->io_txg); |
|
1199 |
||
1200 |
if (error == 0) { |
|
1201 |
bp->blk_birth = zio->io_txg; |
|
1202 |
} else if (error == ENOSPC) { |
|
1203 |
if (zio->io_size == SPA_MINBLOCKSIZE) |
|
1204 |
panic("really, truly out of space"); |
|
1205 |
zio_write_allocate_gang_members(zio); |
|
1206 |
return; |
|
1207 |
} else { |
|
1208 |
zio->io_error = error; |
|
1209 |
} |
|
1210 |
zio_next_stage(zio); |
|
1211 |
} |
|
1212 |
||
1213 |
static void |
|
1214 |
zio_dva_free(zio_t *zio) |
|
1215 |
{ |
|
1216 |
blkptr_t *bp = zio->io_bp; |
|
1217 |
dva_t *dva = ZIO_GET_DVA(zio); |
|
1218 |
||
1219 |
ASSERT(!BP_IS_HOLE(bp)); |
|
1220 |
||
1221 |
metaslab_free(zio->io_spa, dva, zio->io_txg); |
|
1222 |
||
1223 |
BP_ZERO(bp); |
|
1224 |
||
1225 |
zio_next_stage(zio); |
|
1226 |
} |
|
1227 |
||
1228 |
static void |
|
1229 |
zio_dva_claim(zio_t *zio) |
|
1230 |
{ |
|
1231 |
blkptr_t *bp = zio->io_bp; |
|
1232 |
dva_t *dva = ZIO_GET_DVA(zio); |
|
1233 |
||
1234 |
ASSERT(!BP_IS_HOLE(bp)); |
|
1235 |
||
1236 |
zio->io_error = metaslab_claim(zio->io_spa, dva, zio->io_txg); |
|
1237 |
||
1238 |
zio_next_stage(zio); |
|
1239 |
} |
|
1240 |
||
1241 |
static void |
|
1242 |
zio_dva_translate(zio_t *zio) |
|
1243 |
{ |
|
1244 |
spa_t *spa = zio->io_spa; |
|
1245 |
dva_t *dva = ZIO_GET_DVA(zio); |
|
1246 |
uint64_t vdev = DVA_GET_VDEV(dva); |
|
1247 |
uint64_t offset = DVA_GET_OFFSET(dva); |
|
1248 |
||
1249 |
ASSERT3U(zio->io_size, ==, ZIO_GET_IOSIZE(zio)); |
|
1250 |
||
1251 |
zio->io_offset = offset; |
|
1252 |
||
1253 |
if ((zio->io_vd = vdev_lookup_top(spa, vdev)) == NULL) |
|
1254 |
zio->io_error = ENXIO; |
|
1255 |
else if (offset + zio->io_size > zio->io_vd->vdev_asize) |
|
1256 |
zio->io_error = EOVERFLOW; |
|
1257 |
||
1258 |
zio_next_stage(zio); |
|
1259 |
} |
|
1260 |
||
1261 |
/* |
|
1262 |
* ========================================================================== |
|
1263 |
* Read and write to physical devices |
|
1264 |
* ========================================================================== |
|
1265 |
*/ |
|
1266 |
static void |
|
1267 |
zio_vdev_io_enter(zio_t *zio) |
|
1268 |
{ |
|
1269 |
vdev_t *tvd = zio->io_vd->vdev_top; |
|
1270 |
||
1271 |
mutex_enter(&tvd->vdev_io_lock); |
|
1272 |
ASSERT(zio->io_pending.list_next == NULL); |
|
1273 |
list_insert_tail(&tvd->vdev_io_pending, zio); |
|
1274 |
mutex_exit(&tvd->vdev_io_lock); |
|
1275 |
} |
|
1276 |
||
1277 |
static void |
|
1278 |
zio_vdev_io_exit(zio_t *zio) |
|
1279 |
{ |
|
1280 |
vdev_t *tvd = zio->io_vd->vdev_top; |
|
1281 |
||
1282 |
mutex_enter(&tvd->vdev_io_lock); |
|
1283 |
ASSERT(zio->io_pending.list_next != NULL); |
|
1284 |
list_remove(&tvd->vdev_io_pending, zio); |
|
1285 |
if (list_head(&tvd->vdev_io_pending) == NULL) |
|
1286 |
cv_broadcast(&tvd->vdev_io_cv); |
|
1287 |
mutex_exit(&tvd->vdev_io_lock); |
|
1288 |
} |
|
1289 |
||
1290 |
static void |
|
1291 |
zio_vdev_io_retry(void *vdarg) |
|
1292 |
{ |
|
1293 |
vdev_t *vd = vdarg; |
|
1294 |
zio_t *zio, *zq; |
|
1295 |
||
1296 |
ASSERT(vd == vd->vdev_top); |
|
1297 |
||
1298 |
/* XXPOLICY */ |
|
1299 |
delay(hz); |
|
1300 |
||
1301 |
vdev_reopen(vd, &zq); |
|
1302 |
||
1303 |
while ((zio = zq) != NULL) { |
|
1304 |
zq = zio->io_retry_next; |
|
1305 |
zio->io_retry_next = NULL; |
|
1306 |
dprintf("async retry #%d for I/O to %s offset %llx\n", |
|
1307 |
zio->io_retries, vdev_description(vd), zio->io_offset); |
|
1308 |
zio_next_stage_async(zio); |
|
1309 |
} |
|
1310 |
} |
|
1311 |
||
1312 |
static void |
|
1313 |
zio_vdev_io_setup(zio_t *zio) |
|
1314 |
{ |
|
1315 |
vdev_t *vd = zio->io_vd; |
|
1316 |
||
1317 |
/* XXPOLICY */ |
|
1318 |
if (zio->io_retries == 0 && vd == vd->vdev_top) |
|
1319 |
zio->io_flags |= ZIO_FLAG_FAILFAST; |
|
1320 |
||
1321 |
if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) && vd->vdev_children == 0) { |
|
1322 |
zio->io_flags |= ZIO_FLAG_PHYSICAL; |
|
1323 |
zio->io_offset += VDEV_LABEL_START_SIZE; |
|
1324 |
} |
|
1325 |
||
1326 |
zio_vdev_io_enter(zio); |
|
1327 |
||
1328 |
zio_next_stage(zio); |
|
1329 |
} |
|
1330 |
||
1331 |
static void |
|
1332 |
zio_vdev_io_start(zio_t *zio) |
|
1333 |
{ |
|
1334 |
blkptr_t *bp = zio->io_bp; |
|
1335 |
||
1336 |
ASSERT(P2PHASE(zio->io_offset, 1ULL << zio->io_vd->vdev_ashift) == 0); |
|
1337 |
ASSERT(P2PHASE(zio->io_size, 1ULL << zio->io_vd->vdev_ashift) == 0); |
|
1338 |
ASSERT(bp == NULL || ZIO_GET_IOSIZE(zio) == zio->io_size); |
|
1339 |
ASSERT(zio->io_type != ZIO_TYPE_WRITE || (spa_mode & FWRITE)); |
|
1340 |
||
1341 |
vdev_io_start(zio); |
|
1342 |
||
1343 |
/* zio_next_stage_async() gets called from io completion interrupt */ |
|
1344 |
} |
|
1345 |
||
1346 |
static void |
|
1347 |
zio_vdev_io_done(zio_t *zio) |
|
1348 |
{ |
|
1349 |
vdev_io_done(zio); |
|
1350 |
} |
|
1351 |
||
1352 |
/* XXPOLICY */ |
|
1353 |
static boolean_t |
|
1354 |
zio_should_retry(zio_t *zio) |
|
1355 |
{ |
|
1356 |
vdev_t *vd = zio->io_vd; |
|
1357 |
||
1358 |
if (zio->io_error == 0) |
|
1359 |
return (B_FALSE); |
|
1360 |
if (zio->io_delegate_list != NULL) |
|
1361 |
return (B_FALSE); |
|
1362 |
if (vd != vd->vdev_top) |
|
1363 |
return (B_FALSE); |
|
1364 |
if (zio->io_flags & ZIO_FLAG_DONT_RETRY) |
|
1365 |
return (B_FALSE); |
|
1366 |
if (zio->io_retries > 300 && |
|
1367 |
(zio->io_flags & (ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL))) |
|
1368 |
return (B_FALSE); |
|
1369 |
if (zio->io_retries > 1 && |
|
1370 |
(zio->io_error == ECKSUM || zio->io_error == ENXIO)) |
|
1371 |
return (B_FALSE); |
|
1372 |
||
1373 |
return (B_TRUE); |
|
1374 |
} |
|
1375 |
||
1376 |
static void |
|
1377 |
zio_vdev_io_assess(zio_t *zio) |
|
1378 |
{ |
|
1379 |
vdev_t *vd = zio->io_vd; |
|
1380 |
vdev_t *tvd = vd->vdev_top; |
|
1381 |
||
1382 |
zio_vdev_io_exit(zio); |
|
1383 |
||
1384 |
ASSERT(zio->io_vsd == NULL); |
|
1385 |
||
1386 |
/* |
|
1387 |
* If the I/O failed, determine whether we should attempt to retry it. |
|
1388 |
*/ |
|
1389 |
/* XXPOLICY */ |
|
1390 |
if (zio_should_retry(zio)) { |
|
1391 |
zio_t *zq; |
|
1392 |
||
1393 |
ASSERT(tvd == vd); |
|
1394 |
ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)); |
|
1395 |
||
1396 |
zio->io_retries++; |
|
1397 |
zio->io_error = 0; |
|
1398 |
zio->io_flags &= ZIO_FLAG_VDEV_INHERIT; |
|
1399 |
/* XXPOLICY */ |
|
1400 |
zio->io_flags &= ~ZIO_FLAG_FAILFAST; |
|
1401 |
zio->io_flags |= ZIO_FLAG_DONT_CACHE; |
|
1402 |
zio->io_stage = ZIO_STAGE_VDEV_IO_SETUP - 1; |
|
1403 |
||
1404 |
dprintf("retry #%d for %s to %s offset %llx\n", |
|
1405 |
zio->io_retries, zio_type_name[zio->io_type], |
|
1406 |
vdev_description(vd), zio->io_offset); |
|
1407 |
||
1408 |
/* |
|
1409 |
* If this is the first retry, do it immediately. |
|
1410 |
*/ |
|
1411 |
/* XXPOLICY */ |
|
1412 |
if (zio->io_retries == 1) { |
|
1413 |
zio_next_stage_async(zio); |
|
1414 |
return; |
|
1415 |
} |
|
1416 |
||
1417 |
/* |
|
1418 |
* This was not the first retry, so go through the |
|
1419 |
* longer enqueue/delay/vdev_reopen() process. |
|
1420 |
*/ |
|
1421 |
mutex_enter(&tvd->vdev_io_lock); |
|
1422 |
ASSERT(zio->io_retry_next == NULL); |
|
1423 |
zio->io_retry_next = zq = tvd->vdev_io_retry; |
|
1424 |
tvd->vdev_io_retry = zio; |
|
1425 |
mutex_exit(&tvd->vdev_io_lock); |
|
1426 |
if (zq == NULL) |
|
1427 |
(void) taskq_dispatch( |
|
1428 |
tvd->vdev_spa->spa_vdev_retry_taskq, |
|
1429 |
zio_vdev_io_retry, tvd, TQ_SLEEP); |
|
1430 |
return; |
|
1431 |
} |
|
1432 |
||
1433 |
zio_next_stage(zio); |
|
1434 |
} |
|
1435 |
||
1436 |
void |
|
1437 |
zio_vdev_io_reissue(zio_t *zio) |
|
1438 |
{ |
|
1439 |
ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); |
|
1440 |
ASSERT(zio->io_error == 0); |
|
1441 |
||
1442 |
zio->io_stage--; |
|
1443 |
} |
|
1444 |
||
1445 |
void |
|
1446 |
zio_vdev_io_redone(zio_t *zio) |
|
1447 |
{ |
|
1448 |
ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); |
|
1449 |
||
1450 |
zio->io_stage--; |
|
1451 |
} |
|
1452 |
||
1453 |
void |
|
1454 |
zio_vdev_io_bypass(zio_t *zio) |
|
1455 |
{ |
|
1456 |
ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); |
|
1457 |
ASSERT(zio->io_error == 0); |
|
1458 |
||
1459 |
zio->io_flags |= ZIO_FLAG_IO_BYPASS; |
|
1460 |
zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS - 1; |
|
1461 |
} |
|
1462 |
||
1463 |
/* |
|
1464 |
* ========================================================================== |
|
1465 |
* Generate and verify checksums |
|
1466 |
* ========================================================================== |
|
1467 |
*/ |
|
1468 |
static void |
|
1469 |
zio_checksum_generate(zio_t *zio) |
|
1470 |
{ |
|
1471 |
int checksum = zio->io_checksum; |
|
1472 |
blkptr_t *bp = zio->io_bp; |
|
1473 |
||
1474 |
ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); |
|
1475 |
||
1476 |
BP_SET_CHECKSUM(bp, checksum); |
|
1477 |
BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); |
|
1478 |
||
1479 |
zio_checksum(checksum, &bp->blk_cksum, zio->io_data, zio->io_size); |
|
1480 |
||
1481 |
zio_next_stage(zio); |
|
1482 |
} |
|
1483 |
||
1484 |
static void |
|
1485 |
zio_gang_checksum_generate(zio_t *zio) |
|
1486 |
{ |
|
1487 |
zio_cksum_t zc; |
|
1488 |
zio_gbh_phys_t *gbh = zio->io_data; |
|
1489 |
||
1490 |
ASSERT3U(zio->io_size, ==, SPA_GANGBLOCKSIZE); |
|
1491 |
ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); |
|
1492 |
||
1493 |
zio_set_gang_verifier(zio, &gbh->zg_tail.zbt_cksum); |
|
1494 |
||
1495 |
zio_checksum(ZIO_CHECKSUM_GANG_HEADER, &zc, zio->io_data, zio->io_size); |
|
1496 |
||
1497 |
zio_next_stage(zio); |
|
1498 |
} |
|
1499 |
||
1500 |
static void |
|
1501 |
zio_checksum_verify(zio_t *zio) |
|
1502 |
{ |
|
1503 |
if (zio->io_bp != NULL) { |
|
1504 |
zio->io_error = zio_checksum_error(zio); |
|
1505 |
if (zio->io_error) { |
|
1506 |
dprintf("bad checksum on vdev %s\n", |
|
1507 |
vdev_description(zio->io_vd)); |
|
1508 |
} |
|
1509 |
} |
|
1510 |
||
1511 |
zio_next_stage(zio); |
|
1512 |
} |
|
1513 |
||
1514 |
/* |
|
1515 |
* Called by RAID-Z to ensure we don't compute the checksum twice. |
|
1516 |
*/ |
|
1517 |
void |
|
1518 |
zio_checksum_verified(zio_t *zio) |
|
1519 |
{ |
|
1520 |
zio->io_pipeline &= ~(1U << ZIO_STAGE_CHECKSUM_VERIFY); |
|
1521 |
} |
|
1522 |
||
1523 |
/* |
|
1524 |
* Set the external verifier for a gang block based on stuff in the bp |
|
1525 |
*/ |
|
1526 |
void |
|
1527 |
zio_set_gang_verifier(zio_t *zio, zio_cksum_t *zcp) |
|
1528 |
{ |
|
1529 |
zcp->zc_word[0] = DVA_GET_VDEV(ZIO_GET_DVA(zio)); |
|
1530 |
zcp->zc_word[1] = DVA_GET_OFFSET(ZIO_GET_DVA(zio)); |
|
1531 |
zcp->zc_word[2] = zio->io_bp->blk_birth; |
|
1532 |
zcp->zc_word[3] = 0; |
|
1533 |
} |
|
1534 |
||
1535 |
/* |
|
1536 |
* ========================================================================== |
|
1537 |
* Define the pipeline |
|
1538 |
* ========================================================================== |
|
1539 |
*/ |
|
1540 |
typedef void zio_pipe_stage_t(zio_t *zio); |
|
1541 |
||
1542 |
static void |
|
1543 |
zio_badop(zio_t *zio) |
|
1544 |
{ |
|
1545 |
panic("Invalid I/O pipeline stage %u for zio %p", zio->io_stage, zio); |
|
1546 |
} |
|
1547 |
||
1548 |
zio_pipe_stage_t *zio_pipeline[ZIO_STAGE_DONE + 2] = { |
|
1549 |
zio_badop, |
|
1550 |
zio_wait_children_ready, |
|
1551 |
zio_write_compress, |
|
1552 |
zio_checksum_generate, |
|
1553 |
zio_gang_pipeline, |
|
1554 |
zio_get_gang_header, |
|
1555 |
zio_rewrite_gang_members, |
|
1556 |
zio_free_gang_members, |
|
1557 |
zio_claim_gang_members, |
|
1558 |
zio_dva_allocate, |
|
1559 |
zio_dva_free, |
|
1560 |
zio_dva_claim, |
|
1561 |
zio_gang_checksum_generate, |
|
1562 |
zio_ready, |
|
1563 |
zio_dva_translate, |
|
1564 |
zio_vdev_io_setup, |
|
1565 |
zio_vdev_io_start, |
|
1566 |
zio_vdev_io_done, |
|
1567 |
zio_vdev_io_assess, |
|
1568 |
zio_wait_children_done, |
|
1569 |
zio_checksum_verify, |
|
1570 |
zio_read_gang_members, |
|
1571 |
zio_read_decompress, |
|
1572 |
zio_done, |
|
1573 |
zio_badop |
|
1574 |
}; |
|
1575 |
||
1576 |
/* |
|
1577 |
* Move an I/O to the next stage of the pipeline and execute that stage. |
|
1578 |
* There's no locking on io_stage because there's no legitimate way for |
|
1579 |
* multiple threads to be attempting to process the same I/O. |
|
1580 |
*/ |
|
1581 |
void |
|
1582 |
zio_next_stage(zio_t *zio) |
|
1583 |
{ |
|
1584 |
uint32_t pipeline = zio->io_pipeline; |
|
1585 |
||
1586 |
ASSERT(!MUTEX_HELD(&zio->io_lock)); |
|
1587 |
||
1588 |
if (zio->io_error) { |
|
1589 |
dprintf("zio %p vdev %s offset %llx stage %d error %d\n", |
|
1590 |
zio, vdev_description(zio->io_vd), |
|
1591 |
zio->io_offset, zio->io_stage, zio->io_error); |
|
1592 |
if (((1U << zio->io_stage) & ZIO_VDEV_IO_PIPELINE) == 0) |
|
1593 |
pipeline &= ZIO_ERROR_PIPELINE_MASK; |
|
1594 |
} |
|
1595 |
||
1596 |
while (((1U << ++zio->io_stage) & pipeline) == 0) |
|
1597 |
continue; |
|
1598 |
||
1599 |
ASSERT(zio->io_stage <= ZIO_STAGE_DONE); |
|
1600 |
ASSERT(zio->io_stalled == 0); |
|
1601 |
||
1602 |
zio_pipeline[zio->io_stage](zio); |
|
1603 |
} |
|
1604 |
||
1605 |
void |
|
1606 |
zio_next_stage_async(zio_t *zio) |
|
1607 |
{ |
|
1608 |
taskq_t *tq; |
|
1609 |
uint32_t pipeline = zio->io_pipeline; |
|
1610 |
||
1611 |
ASSERT(!MUTEX_HELD(&zio->io_lock)); |
|
1612 |
||
1613 |
if (zio->io_error) { |
|
1614 |
dprintf("zio %p vdev %s offset %llx stage %d error %d\n", |
|
1615 |
zio, vdev_description(zio->io_vd), |
|
1616 |
zio->io_offset, zio->io_stage, zio->io_error); |
|
1617 |
if (((1U << zio->io_stage) & ZIO_VDEV_IO_PIPELINE) == 0) |
|
1618 |
pipeline &= ZIO_ERROR_PIPELINE_MASK; |
|
1619 |
} |
|
1620 |
||
1621 |
while (((1U << ++zio->io_stage) & pipeline) == 0) |
|
1622 |
continue; |
|
1623 |
||
1624 |
ASSERT(zio->io_stage <= ZIO_STAGE_DONE); |
|
1625 |
ASSERT(zio->io_stalled == 0); |
|
1626 |
||
1627 |
/* |
|
1628 |
* For performance, we'll probably want two sets of task queues: |
|
1629 |
* per-CPU issue taskqs and per-CPU completion taskqs. The per-CPU |
|
1630 |
* part is for read performance: since we have to make a pass over |
|
1631 |
* the data to checksum it anyway, we want to do this on the same CPU |
|
1632 |
* that issued the read, because (assuming CPU scheduling affinity) |
|
1633 |
* that thread is probably still there. Getting this optimization |
|
1634 |
* right avoids performance-hostile cache-to-cache transfers. |
|
1635 |
* |
|
1636 |
* Note that having two sets of task queues is also necessary for |
|
1637 |
* correctness: if all of the issue threads get bogged down waiting |
|
1638 |
* for dependent reads (e.g. metaslab freelist) to complete, then |
|
1639 |
* there won't be any threads available to service I/O completion |
|
1640 |
* interrupts. |
|
1641 |
*/ |
|
1642 |
if ((1U << zio->io_stage) & zio->io_async_stages) { |
|
1643 |
if (zio->io_stage < ZIO_STAGE_VDEV_IO_DONE) |
|
1644 |
tq = zio->io_spa->spa_zio_issue_taskq[zio->io_type]; |
|
1645 |
else |
|
1646 |
tq = zio->io_spa->spa_zio_intr_taskq[zio->io_type]; |
|
1647 |
(void) taskq_dispatch(tq, |
|
1648 |
(task_func_t *)zio_pipeline[zio->io_stage], zio, TQ_SLEEP); |
|
1649 |
} else { |
|
1650 |
zio_pipeline[zio->io_stage](zio); |
|
1651 |
} |
|
1652 |
} |
|
1653 |
||
1654 |
/* |
|
1655 |
* Try to allocate an intent log block. Return 0 on success, errno on failure. |
|
1656 |
*/ |
|
1657 |
int |
|
1658 |
zio_alloc_blk(spa_t *spa, int checksum, uint64_t size, blkptr_t *bp, |
|
1659 |
uint64_t txg) |
|
1660 |
{ |
|
1661 |
int error; |
|
1662 |
||
1663 |
spa_config_enter(spa, RW_READER); |
|
1664 |
||
1665 |
BP_ZERO(bp); |
|
1666 |
||
1667 |
error = metaslab_alloc(spa, size, BP_IDENTITY(bp), txg); |
|
1668 |
||
1669 |
if (error == 0) { |
|
1670 |
BP_SET_CHECKSUM(bp, checksum); |
|
1671 |
BP_SET_LSIZE(bp, size); |
|
1672 |
BP_SET_PSIZE(bp, size); |
|
1673 |
BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); |
|
1674 |
BP_SET_TYPE(bp, DMU_OT_INTENT_LOG); |
|
1675 |
BP_SET_LEVEL(bp, 0); |
|
1676 |
BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); |
|
1677 |
bp->blk_birth = txg; |
|
1678 |
} |
|
1679 |
||
1680 |
spa_config_exit(spa); |
|
1681 |
||
1682 |
return (error); |
|
1683 |
} |
|
1684 |
||
1685 |
/* |
|
1686 |
* Free an intent log block. We know it can't be a gang block, so there's |
|
1687 |
* nothing to do except metaslab_free() it. |
|
1688 |
*/ |
|
1689 |
void |
|
1690 |
zio_free_blk(spa_t *spa, blkptr_t *bp, uint64_t txg) |
|
1691 |
{ |
|
1692 |
ASSERT(DVA_GET_GANG(BP_IDENTITY(bp)) == 0); |
|
1693 |
||
1694 |
dprintf_bp(bp, "txg %llu: ", txg); |
|
1695 |
||
1696 |
spa_config_enter(spa, RW_READER); |
|
1697 |
||
1698 |
metaslab_free(spa, BP_IDENTITY(bp), txg); |
|
1699 |
||
1700 |
spa_config_exit(spa); |
|
1701 |
} |