author | Darren Moffat <Darren.Moffat@Sun.COM> |
Mon, 08 Sep 2008 10:50:56 +0100 | |
changeset 13369 | 32a041998ab1 |
parent 13368 | 2661054b6aaa |
child 13371 | 45507bab8d65 |
permissions | -rw-r--r-- |
789 | 1 |
/* |
2 |
* CDDL HEADER START |
|
3 |
* |
|
4 |
* The contents of this file are subject to the terms of the |
|
1544 | 5 |
* Common Development and Distribution License (the "License"). |
6 |
* You may not use this file except in compliance with the License. |
|
789 | 7 |
* |
8 |
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE |
|
9 |
* or http://www.opensolaris.org/os/licensing. |
|
10 |
* See the License for the specific language governing permissions |
|
11 |
* and limitations under the License. |
|
12 |
* |
|
13 |
* When distributing Covered Code, include this CDDL HEADER in each |
|
14 |
* file and include the License file at usr/src/OPENSOLARIS.LICENSE. |
|
15 |
* If applicable, add the following below this CDDL HEADER, with the |
|
16 |
* fields enclosed by brackets "[]" replaced with your own identifying |
|
17 |
* information: Portions Copyright [yyyy] [name of copyright owner] |
|
18 |
* |
|
19 |
* CDDL HEADER END |
|
20 |
*/ |
|
21 |
/* |
|
6245
1a2a7cfb9f26
6429205 each zpool needs to monitor its throughput and throttle heavy writers
maybee
parents:
5688
diff
changeset
|
22 |
* Copyright 2008 Sun Microsystems, Inc. All rights reserved. |
789 | 23 |
* Use is subject to license terms. |
24 |
*/ |
|
25 |
||
26 |
#pragma ident "%Z%%M% %I% %E% SMI" |
|
27 |
||
28 |
#include <sys/zfs_context.h> |
|
1544 | 29 |
#include <sys/fm/fs/zfs.h> |
789 | 30 |
#include <sys/spa.h> |
31 |
#include <sys/txg.h> |
|
32 |
#include <sys/spa_impl.h> |
|
33 |
#include <sys/vdev_impl.h> |
|
34 |
#include <sys/zio_impl.h> |
|
35 |
#include <sys/zio_compress.h> |
|
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
36 |
#include <sys/zio_crypt.h> |
789 | 37 |
#include <sys/zio_checksum.h> |
38 |
||
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
39 |
#include <sys/sdt.h> |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
40 |
|
789 | 41 |
/* |
42 |
* ========================================================================== |
|
43 |
* I/O priority table |
|
44 |
* ========================================================================== |
|
45 |
*/ |
|
46 |
uint8_t zio_priority_table[ZIO_PRIORITY_TABLE_SIZE] = { |
|
47 |
0, /* ZIO_PRIORITY_NOW */ |
|
48 |
0, /* ZIO_PRIORITY_SYNC_READ */ |
|
49 |
0, /* ZIO_PRIORITY_SYNC_WRITE */ |
|
50 |
6, /* ZIO_PRIORITY_ASYNC_READ */ |
|
51 |
4, /* ZIO_PRIORITY_ASYNC_WRITE */ |
|
52 |
4, /* ZIO_PRIORITY_FREE */ |
|
53 |
0, /* ZIO_PRIORITY_CACHE_FILL */ |
|
54 |
0, /* ZIO_PRIORITY_LOG_WRITE */ |
|
55 |
10, /* ZIO_PRIORITY_RESILVER */ |
|
56 |
20, /* ZIO_PRIORITY_SCRUB */ |
|
57 |
}; |
|
58 |
||
59 |
/* |
|
60 |
* ========================================================================== |
|
61 |
* I/O type descriptions |
|
62 |
* ========================================================================== |
|
63 |
*/ |
|
64 |
char *zio_type_name[ZIO_TYPES] = { |
|
65 |
"null", "read", "write", "free", "claim", "ioctl" }; |
|
66 |
||
3668 | 67 |
/* Force an allocation failure when non-zero */ |
68 |
uint16_t zio_zil_fail_shift = 0; |
|
5329 | 69 |
uint16_t zio_io_fail_shift = 0; |
70 |
||
71 |
/* Enable/disable the write-retry logic */ |
|
72 |
int zio_write_retry = 1; |
|
73 |
||
74 |
/* Taskq to handle reissuing of I/Os */ |
|
75 |
taskq_t *zio_taskq; |
|
76 |
int zio_resume_threads = 4; |
|
3668 | 77 |
|
789 | 78 |
typedef struct zio_sync_pass { |
79 |
int zp_defer_free; /* defer frees after this pass */ |
|
80 |
int zp_dontcompress; /* don't compress after this pass */ |
|
81 |
int zp_rewrite; /* rewrite new bps after this pass */ |
|
82 |
} zio_sync_pass_t; |
|
83 |
||
84 |
zio_sync_pass_t zio_sync_pass = { |
|
85 |
1, /* zp_defer_free */ |
|
86 |
4, /* zp_dontcompress */ |
|
87 |
1, /* zp_rewrite */ |
|
88 |
}; |
|
89 |
||
5329 | 90 |
static boolean_t zio_io_should_fail(uint16_t); |
91 |
||
789 | 92 |
/* |
93 |
* ========================================================================== |
|
94 |
* I/O kmem caches |
|
95 |
* ========================================================================== |
|
96 |
*/ |
|
4055 | 97 |
kmem_cache_t *zio_cache; |
789 | 98 |
kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; |
3290 | 99 |
kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; |
100 |
||
101 |
#ifdef _KERNEL |
|
102 |
extern vmem_t *zio_alloc_arena; |
|
103 |
#endif |
|
789 | 104 |
|
5329 | 105 |
/* |
106 |
* Determine if we are allowed to issue the IO based on the |
|
107 |
* pool state. If we must wait then block until we are told |
|
108 |
* that we may continue. |
|
109 |
*/ |
|
110 |
#define ZIO_ENTER(spa) { \ |
|
111 |
if (spa->spa_state == POOL_STATE_IO_FAILURE) { \ |
|
112 |
mutex_enter(&spa->spa_zio_lock); \ |
|
113 |
while (spa->spa_state == POOL_STATE_IO_FAILURE) \ |
|
114 |
cv_wait(&spa->spa_zio_cv, &spa->spa_zio_lock); \ |
|
115 |
mutex_exit(&spa->spa_zio_lock); \ |
|
116 |
} \ |
|
117 |
} |
|
118 |
||
119 |
/* |
|
120 |
* An allocation zio is one that either currently has the DVA allocate |
|
121 |
* stage set or will have it later in it's lifetime. |
|
122 |
*/ |
|
123 |
#define IO_IS_ALLOCATING(zio) \ |
|
5688
c0b02c8fd2c0
6640580 spa_get_random() is insanely slow in userland
bonwick
parents:
5530
diff
changeset
|
124 |
((zio)->io_orig_pipeline & (1U << ZIO_STAGE_DVA_ALLOCATE)) |
5329 | 125 |
|
789 | 126 |
void |
127 |
zio_init(void) |
|
128 |
{ |
|
129 |
size_t c; |
|
3290 | 130 |
vmem_t *data_alloc_arena = NULL; |
131 |
||
132 |
#ifdef _KERNEL |
|
133 |
data_alloc_arena = zio_alloc_arena; |
|
134 |
#endif |
|
789 | 135 |
|
4055 | 136 |
zio_cache = kmem_cache_create("zio_cache", sizeof (zio_t), 0, |
137 |
NULL, NULL, NULL, NULL, NULL, 0); |
|
138 |
||
789 | 139 |
/* |
140 |
* For small buffers, we want a cache for each multiple of |
|
141 |
* SPA_MINBLOCKSIZE. For medium-size buffers, we want a cache |
|
142 |
* for each quarter-power of 2. For large buffers, we want |
|
143 |
* a cache for each multiple of PAGESIZE. |
|
144 |
*/ |
|
145 |
for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { |
|
146 |
size_t size = (c + 1) << SPA_MINBLOCKSHIFT; |
|
147 |
size_t p2 = size; |
|
148 |
size_t align = 0; |
|
149 |
||
150 |
while (p2 & (p2 - 1)) |
|
151 |
p2 &= p2 - 1; |
|
152 |
||
153 |
if (size <= 4 * SPA_MINBLOCKSIZE) { |
|
154 |
align = SPA_MINBLOCKSIZE; |
|
155 |
} else if (P2PHASE(size, PAGESIZE) == 0) { |
|
156 |
align = PAGESIZE; |
|
157 |
} else if (P2PHASE(size, p2 >> 2) == 0) { |
|
158 |
align = p2 >> 2; |
|
159 |
} |
|
160 |
||
161 |
if (align != 0) { |
|
3290 | 162 |
char name[36]; |
2856 | 163 |
(void) sprintf(name, "zio_buf_%lu", (ulong_t)size); |
789 | 164 |
zio_buf_cache[c] = kmem_cache_create(name, size, |
849
8d799fd81a9b
6345023 /dev/zfs fails to open once ZFS module is unloaded
bonwick
parents:
789
diff
changeset
|
165 |
align, NULL, NULL, NULL, NULL, NULL, KMC_NODEBUG); |
3290 | 166 |
|
167 |
(void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size); |
|
168 |
zio_data_buf_cache[c] = kmem_cache_create(name, size, |
|
169 |
align, NULL, NULL, NULL, NULL, data_alloc_arena, |
|
170 |
KMC_NODEBUG); |
|
171 |
||
789 | 172 |
} |
173 |
} |
|
174 |
||
175 |
while (--c != 0) { |
|
176 |
ASSERT(zio_buf_cache[c] != NULL); |
|
177 |
if (zio_buf_cache[c - 1] == NULL) |
|
178 |
zio_buf_cache[c - 1] = zio_buf_cache[c]; |
|
3290 | 179 |
|
180 |
ASSERT(zio_data_buf_cache[c] != NULL); |
|
181 |
if (zio_data_buf_cache[c - 1] == NULL) |
|
182 |
zio_data_buf_cache[c - 1] = zio_data_buf_cache[c]; |
|
789 | 183 |
} |
1544 | 184 |
|
5329 | 185 |
zio_taskq = taskq_create("zio_taskq", zio_resume_threads, |
186 |
maxclsyspri, 50, INT_MAX, TASKQ_PREPOPULATE); |
|
187 |
||
1544 | 188 |
zio_inject_init(); |
789 | 189 |
} |
190 |
||
191 |
void |
|
192 |
zio_fini(void) |
|
193 |
{ |
|
194 |
size_t c; |
|
195 |
kmem_cache_t *last_cache = NULL; |
|
3290 | 196 |
kmem_cache_t *last_data_cache = NULL; |
789 | 197 |
|
198 |
for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { |
|
199 |
if (zio_buf_cache[c] != last_cache) { |
|
200 |
last_cache = zio_buf_cache[c]; |
|
201 |
kmem_cache_destroy(zio_buf_cache[c]); |
|
202 |
} |
|
203 |
zio_buf_cache[c] = NULL; |
|
3290 | 204 |
|
205 |
if (zio_data_buf_cache[c] != last_data_cache) { |
|
206 |
last_data_cache = zio_data_buf_cache[c]; |
|
207 |
kmem_cache_destroy(zio_data_buf_cache[c]); |
|
208 |
} |
|
209 |
zio_data_buf_cache[c] = NULL; |
|
789 | 210 |
} |
1544 | 211 |
|
5329 | 212 |
taskq_destroy(zio_taskq); |
213 |
||
4055 | 214 |
kmem_cache_destroy(zio_cache); |
215 |
||
1544 | 216 |
zio_inject_fini(); |
789 | 217 |
} |
218 |
||
219 |
/* |
|
220 |
* ========================================================================== |
|
221 |
* Allocate and free I/O buffers |
|
222 |
* ========================================================================== |
|
223 |
*/ |
|
3290 | 224 |
|
225 |
/* |
|
226 |
* Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a |
|
227 |
* crashdump if the kernel panics, so use it judiciously. Obviously, it's |
|
228 |
* useful to inspect ZFS metadata, but if possible, we should avoid keeping |
|
229 |
* excess / transient data in-core during a crashdump. |
|
230 |
*/ |
|
789 | 231 |
void * |
232 |
zio_buf_alloc(size_t size) |
|
233 |
{ |
|
234 |
size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; |
|
235 |
||
236 |
ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); |
|
237 |
||
6245
1a2a7cfb9f26
6429205 each zpool needs to monitor its throughput and throttle heavy writers
maybee
parents:
5688
diff
changeset
|
238 |
return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE)); |
789 | 239 |
} |
240 |
||
3290 | 241 |
/* |
242 |
* Use zio_data_buf_alloc to allocate data. The data will not appear in a |
|
243 |
* crashdump if the kernel panics. This exists so that we will limit the amount |
|
244 |
* of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount |
|
245 |
* of kernel heap dumped to disk when the kernel panics) |
|
246 |
*/ |
|
247 |
void * |
|
248 |
zio_data_buf_alloc(size_t size) |
|
249 |
{ |
|
250 |
size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; |
|
251 |
||
252 |
ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); |
|
253 |
||
6245
1a2a7cfb9f26
6429205 each zpool needs to monitor its throughput and throttle heavy writers
maybee
parents:
5688
diff
changeset
|
254 |
return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE)); |
3290 | 255 |
} |
256 |
||
789 | 257 |
void |
258 |
zio_buf_free(void *buf, size_t size) |
|
259 |
{ |
|
260 |
size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; |
|
261 |
||
262 |
ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); |
|
263 |
||
264 |
kmem_cache_free(zio_buf_cache[c], buf); |
|
265 |
} |
|
266 |
||
3290 | 267 |
void |
268 |
zio_data_buf_free(void *buf, size_t size) |
|
269 |
{ |
|
270 |
size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; |
|
271 |
||
272 |
ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); |
|
273 |
||
274 |
kmem_cache_free(zio_data_buf_cache[c], buf); |
|
275 |
} |
|
3463 | 276 |
|
789 | 277 |
/* |
278 |
* ========================================================================== |
|
279 |
* Push and pop I/O transform buffers |
|
280 |
* ========================================================================== |
|
281 |
*/ |
|
282 |
static void |
|
283 |
zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize) |
|
284 |
{ |
|
285 |
zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); |
|
286 |
||
287 |
zt->zt_data = data; |
|
288 |
zt->zt_size = size; |
|
289 |
zt->zt_bufsize = bufsize; |
|
290 |
||
291 |
zt->zt_next = zio->io_transform_stack; |
|
292 |
zio->io_transform_stack = zt; |
|
293 |
||
294 |
zio->io_data = data; |
|
295 |
zio->io_size = size; |
|
296 |
} |
|
297 |
||
298 |
static void |
|
299 |
zio_pop_transform(zio_t *zio, void **data, uint64_t *size, uint64_t *bufsize) |
|
300 |
{ |
|
301 |
zio_transform_t *zt = zio->io_transform_stack; |
|
302 |
||
303 |
*data = zt->zt_data; |
|
304 |
*size = zt->zt_size; |
|
305 |
*bufsize = zt->zt_bufsize; |
|
306 |
||
307 |
zio->io_transform_stack = zt->zt_next; |
|
308 |
kmem_free(zt, sizeof (zio_transform_t)); |
|
309 |
||
310 |
if ((zt = zio->io_transform_stack) != NULL) { |
|
311 |
zio->io_data = zt->zt_data; |
|
312 |
zio->io_size = zt->zt_size; |
|
313 |
} |
|
314 |
} |
|
315 |
||
316 |
static void |
|
317 |
zio_clear_transform_stack(zio_t *zio) |
|
318 |
{ |
|
319 |
void *data; |
|
320 |
uint64_t size, bufsize; |
|
321 |
||
322 |
ASSERT(zio->io_transform_stack != NULL); |
|
323 |
||
324 |
zio_pop_transform(zio, &data, &size, &bufsize); |
|
325 |
while (zio->io_transform_stack != NULL) { |
|
326 |
zio_buf_free(data, bufsize); |
|
327 |
zio_pop_transform(zio, &data, &size, &bufsize); |
|
328 |
} |
|
329 |
} |
|
330 |
||
331 |
/* |
|
332 |
* ========================================================================== |
|
333 |
* Create the various types of I/O (read, write, free) |
|
334 |
* ========================================================================== |
|
335 |
*/ |
|
336 |
static zio_t * |
|
337 |
zio_create(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, |
|
338 |
void *data, uint64_t size, zio_done_func_t *done, void *private, |
|
339 |
zio_type_t type, int priority, int flags, uint8_t stage, uint32_t pipeline) |
|
340 |
{ |
|
341 |
zio_t *zio; |
|
342 |
||
343 |
ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); |
|
344 |
ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); |
|
345 |
||
7046
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
346 |
/* Only we should set CONFIG_GRABBED */ |
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
347 |
ASSERT(!(flags & ZIO_FLAG_CONFIG_GRABBED)); |
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
348 |
|
4055 | 349 |
zio = kmem_cache_alloc(zio_cache, KM_SLEEP); |
350 |
bzero(zio, sizeof (zio_t)); |
|
789 | 351 |
zio->io_parent = pio; |
352 |
zio->io_spa = spa; |
|
353 |
zio->io_txg = txg; |
|
4634
39bfb9e90d34
6437054 vdev_cache wises up: increase DB performance by 16%
ek110237
parents:
4527
diff
changeset
|
354 |
zio->io_flags = flags; |
789 | 355 |
if (bp != NULL) { |
356 |
zio->io_bp = bp; |
|
357 |
zio->io_bp_copy = *bp; |
|
358 |
zio->io_bp_orig = *bp; |
|
359 |
} |
|
360 |
zio->io_done = done; |
|
361 |
zio->io_private = private; |
|
362 |
zio->io_type = type; |
|
363 |
zio->io_priority = priority; |
|
364 |
zio->io_stage = stage; |
|
365 |
zio->io_pipeline = pipeline; |
|
366 |
zio->io_timestamp = lbolt64; |
|
2856 | 367 |
mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL); |
4831
41ec732c6d9f
6584470 zdb needs to initialize the bpl_lock mutex
gw25295
parents:
4634
diff
changeset
|
368 |
cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL); |
789 | 369 |
zio_push_transform(zio, data, size, size); |
370 |
||
3463 | 371 |
/* |
372 |
* Note on config lock: |
|
373 |
* |
|
374 |
* If CONFIG_HELD is set, then the caller already has the config |
|
375 |
* lock, so we don't need it for this io. |
|
376 |
* |
|
377 |
* We set CONFIG_GRABBED to indicate that we have grabbed the |
|
378 |
* config lock on behalf of this io, so it should be released |
|
379 |
* in zio_done. |
|
380 |
* |
|
381 |
* Unless CONFIG_HELD is set, we will grab the config lock for |
|
382 |
* any top-level (parent-less) io, *except* NULL top-level ios. |
|
383 |
* The NULL top-level ios rarely have any children, so we delay |
|
384 |
* grabbing the lock until the first child is added (but it is |
|
385 |
* still grabbed on behalf of the top-level i/o, so additional |
|
386 |
* children don't need to also grab it). This greatly reduces |
|
387 |
* contention on the config lock. |
|
388 |
*/ |
|
789 | 389 |
if (pio == NULL) { |
3463 | 390 |
if (type != ZIO_TYPE_NULL && |
391 |
!(flags & ZIO_FLAG_CONFIG_HELD)) { |
|
5530 | 392 |
spa_config_enter(spa, RW_READER, zio); |
3463 | 393 |
zio->io_flags |= ZIO_FLAG_CONFIG_GRABBED; |
394 |
} |
|
789 | 395 |
zio->io_root = zio; |
396 |
} else { |
|
397 |
zio->io_root = pio->io_root; |
|
1544 | 398 |
if (!(flags & ZIO_FLAG_NOBOOKMARK)) |
399 |
zio->io_logical = pio->io_logical; |
|
789 | 400 |
mutex_enter(&pio->io_lock); |
3463 | 401 |
if (pio->io_parent == NULL && |
402 |
pio->io_type == ZIO_TYPE_NULL && |
|
403 |
!(pio->io_flags & ZIO_FLAG_CONFIG_GRABBED) && |
|
404 |
!(pio->io_flags & ZIO_FLAG_CONFIG_HELD)) { |
|
405 |
pio->io_flags |= ZIO_FLAG_CONFIG_GRABBED; |
|
5530 | 406 |
spa_config_enter(spa, RW_READER, pio); |
3463 | 407 |
} |
789 | 408 |
if (stage < ZIO_STAGE_READY) |
409 |
pio->io_children_notready++; |
|
410 |
pio->io_children_notdone++; |
|
411 |
zio->io_sibling_next = pio->io_child; |
|
412 |
zio->io_sibling_prev = NULL; |
|
413 |
if (pio->io_child != NULL) |
|
414 |
pio->io_child->io_sibling_prev = zio; |
|
415 |
pio->io_child = zio; |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
416 |
zio->io_ndvas = pio->io_ndvas; |
789 | 417 |
mutex_exit(&pio->io_lock); |
418 |
} |
|
419 |
||
5329 | 420 |
/* |
421 |
* Save off the original state incase we need to retry later. |
|
422 |
*/ |
|
423 |
zio->io_orig_stage = zio->io_stage; |
|
424 |
zio->io_orig_pipeline = zio->io_pipeline; |
|
425 |
zio->io_orig_flags = zio->io_flags; |
|
426 |
||
7046
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
427 |
/* |
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
428 |
* If this is not a null zio, and config is not already held, |
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
429 |
* then the root zio should have grabbed the config lock. |
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
430 |
* If this is not a root zio, it should not have grabbed the |
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
431 |
* config lock. |
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
432 |
*/ |
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
433 |
ASSERT((zio->io_root->io_flags & ZIO_FLAG_CONFIG_HELD) || |
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
434 |
zio->io_type == ZIO_TYPE_NULL || |
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
435 |
(zio->io_root->io_flags & ZIO_FLAG_CONFIG_GRABBED)); |
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
436 |
ASSERT(zio->io_root == zio || |
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
437 |
!(zio->io_flags & ZIO_FLAG_CONFIG_GRABBED)); |
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
438 |
|
789 | 439 |
return (zio); |
440 |
} |
|
441 |
||
5329 | 442 |
static void |
443 |
zio_reset(zio_t *zio) |
|
444 |
{ |
|
445 |
zio_clear_transform_stack(zio); |
|
446 |
||
447 |
zio->io_flags = zio->io_orig_flags; |
|
448 |
zio->io_stage = zio->io_orig_stage; |
|
449 |
zio->io_pipeline = zio->io_orig_pipeline; |
|
450 |
zio_push_transform(zio, zio->io_data, zio->io_size, zio->io_size); |
|
451 |
} |
|
452 |
||
789 | 453 |
zio_t * |
454 |
zio_null(zio_t *pio, spa_t *spa, zio_done_func_t *done, void *private, |
|
455 |
int flags) |
|
456 |
{ |
|
457 |
zio_t *zio; |
|
458 |
||
459 |
zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, |
|
460 |
ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, ZIO_STAGE_OPEN, |
|
461 |
ZIO_WAIT_FOR_CHILDREN_PIPELINE); |
|
462 |
||
463 |
return (zio); |
|
464 |
} |
|
465 |
||
466 |
zio_t * |
|
467 |
zio_root(spa_t *spa, zio_done_func_t *done, void *private, int flags) |
|
468 |
{ |
|
469 |
return (zio_null(NULL, spa, done, private, flags)); |
|
470 |
} |
|
471 |
||
472 |
zio_t * |
|
7046
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
473 |
zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, void *data, |
789 | 474 |
uint64_t size, zio_done_func_t *done, void *private, |
7046
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
475 |
int priority, int flags, const zbookmark_t *zb) |
789 | 476 |
{ |
477 |
zio_t *zio; |
|
478 |
||
479 |
ASSERT3U(size, ==, BP_GET_LSIZE(bp)); |
|
480 |
||
5329 | 481 |
/* |
482 |
* If the user has specified that we allow I/Os to continue |
|
483 |
* then attempt to satisfy the read. |
|
484 |
*/ |
|
485 |
if (spa_get_failmode(spa) != ZIO_FAILURE_MODE_CONTINUE) |
|
486 |
ZIO_ENTER(spa); |
|
487 |
||
7046
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
488 |
zio = zio_create(pio, spa, bp->blk_birth, (blkptr_t *)bp, |
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
489 |
data, size, done, private, |
2981 | 490 |
ZIO_TYPE_READ, priority, flags | ZIO_FLAG_USER, |
491 |
ZIO_STAGE_OPEN, ZIO_READ_PIPELINE); |
|
1544 | 492 |
zio->io_bookmark = *zb; |
493 |
||
494 |
zio->io_logical = zio; |
|
789 | 495 |
|
496 |
/* |
|
497 |
* Work off our copy of the bp so the caller can free it. |
|
498 |
*/ |
|
499 |
zio->io_bp = &zio->io_bp_copy; |
|
500 |
||
501 |
return (zio); |
|
502 |
} |
|
503 |
||
504 |
zio_t * |
|
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
505 |
zio_write(zio_t *pio, spa_t *spa, int checksum, int compress, int crypt, |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
506 |
int ncopies, uint64_t txg, blkptr_t *bp, void *data, uint64_t size, |
3547
e396e0a440b1
6512391 DMU should leverage ZIO dependencies to achieve greater parallelism
maybee
parents:
3463
diff
changeset
|
507 |
zio_done_func_t *ready, zio_done_func_t *done, void *private, int priority, |
7046
361307ae060d
6343667 scrub/resilver has to start over when a snapshot is taken
ahrens
parents:
7030
diff
changeset
|
508 |
int flags, const zbookmark_t *zb) |
789 | 509 |
{ |
510 |
zio_t *zio; |
|
511 |
||
512 |
ASSERT(checksum >= ZIO_CHECKSUM_OFF && |
|
513 |
checksum < ZIO_CHECKSUM_FUNCTIONS); |
|
514 |
||
515 |
ASSERT(compress >= ZIO_COMPRESS_OFF && |
|
516 |
compress < ZIO_COMPRESS_FUNCTIONS); |
|
517 |
||
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
518 |
ASSERT(crypt >= ZIO_CRYPT_OFF && |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
519 |
crypt < ZIO_CRYPT_FUNCTIONS); |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
520 |
|
5329 | 521 |
ZIO_ENTER(spa); |
522 |
||
789 | 523 |
zio = zio_create(pio, spa, txg, bp, data, size, done, private, |
2981 | 524 |
ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_USER, |
789 | 525 |
ZIO_STAGE_OPEN, ZIO_WRITE_PIPELINE); |
526 |
||
3547
e396e0a440b1
6512391 DMU should leverage ZIO dependencies to achieve greater parallelism
maybee
parents:
3463
diff
changeset
|
527 |
zio->io_ready = ready; |
e396e0a440b1
6512391 DMU should leverage ZIO dependencies to achieve greater parallelism
maybee
parents:
3463
diff
changeset
|
528 |
|
1544 | 529 |
zio->io_bookmark = *zb; |
530 |
||
531 |
zio->io_logical = zio; |
|
532 |
||
789 | 533 |
zio->io_checksum = checksum; |
534 |
zio->io_compress = compress; |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
535 |
zio->io_ndvas = ncopies; |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
536 |
zio->io_crypt = crypt; |
789 | 537 |
|
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
538 |
if (crypt != ZIO_CRYPT_OFF && crypt != ZIO_CRYPT_INHERIT) { |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
539 |
zio->io_ckey = zio_crypt_key_lookup(spa, zb->zb_objset, crypt); |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
540 |
if (zio->io_ckey != NULL) { |
13361
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
541 |
zio_crypt_key_hold(zio->io_ckey, FTAG); |
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
542 |
} |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
543 |
} |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
544 |
|
789 | 545 |
if (bp->blk_birth != txg) { |
546 |
/* XXX the bp usually (always?) gets re-zeroed later */ |
|
547 |
BP_ZERO(bp); |
|
548 |
BP_SET_LSIZE(bp, size); |
|
549 |
BP_SET_PSIZE(bp, size); |
|
13251
f037017b1f9b
1701 zio_write_encrypt ASSERT trips on old spa version
Darren Moffat <darrenm@opensolaris.org>
parents:
13230
diff
changeset
|
550 |
BP_SET_CRYPT(bp, ZIO_CRYPT_OFF); |
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
551 |
} else { |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
552 |
/* Make sure someone doesn't change their mind on overwrites */ |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
553 |
ASSERT(MIN(zio->io_ndvas + BP_IS_GANG(bp), |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
554 |
spa_max_replication(spa)) == BP_GET_NDVAS(bp)); |
789 | 555 |
} |
556 |
||
557 |
return (zio); |
|
558 |
} |
|
559 |
||
560 |
zio_t * |
|
13320 | 561 |
zio_rewrite(zio_t *pio, spa_t *spa, int checksum, int crypt, |
13338 | 562 |
uint64_t txg, blkptr_t *bp, void *data, uint64_t size, |
13320 | 563 |
zio_done_func_t *done, void *private, int priority, |
7030
b81089449d53
6721869 zil needs to pass block birth txg to zil_rewrite()
perrin
parents:
6976
diff
changeset
|
564 |
int flags, zbookmark_t *zb) |
789 | 565 |
{ |
566 |
zio_t *zio; |
|
567 |
||
7181
8d299641aa23
6719934 assertion failed: ((&zh->zh_log)->blk_birth == 0), file: ../../common/fs/zfs/zil.c, line: 1336
perrin
parents:
7046
diff
changeset
|
568 |
zio = zio_create(pio, spa, txg, bp, data, size, done, private, |
2981 | 569 |
ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_USER, |
5530 | 570 |
ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE(bp)); |
789 | 571 |
|
1544 | 572 |
zio->io_bookmark = *zb; |
789 | 573 |
zio->io_checksum = checksum; |
574 |
zio->io_compress = ZIO_COMPRESS_OFF; |
|
13190
a75f21839b8a
ZIL encryption preparation work
Darren Moffat <darrenm@opensolaris.org>
parents:
13186
diff
changeset
|
575 |
zio->io_crypt = crypt; |
789 | 576 |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
577 |
if (pio != NULL) |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
578 |
ASSERT3U(zio->io_ndvas, <=, BP_GET_NDVAS(bp)); |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
579 |
|
789 | 580 |
return (zio); |
581 |
} |
|
582 |
||
5329 | 583 |
static void |
584 |
zio_write_allocate_ready(zio_t *zio) |
|
585 |
{ |
|
586 |
/* Free up the previous block */ |
|
587 |
if (!BP_IS_HOLE(&zio->io_bp_orig)) { |
|
588 |
zio_nowait(zio_free(zio, zio->io_spa, zio->io_txg, |
|
589 |
&zio->io_bp_orig, NULL, NULL)); |
|
590 |
} |
|
591 |
} |
|
592 |
||
789 | 593 |
static zio_t * |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
594 |
zio_write_allocate(zio_t *pio, spa_t *spa, int checksum, uint64_t txg, |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
595 |
blkptr_t *bp, void *data, uint64_t size, |
789 | 596 |
zio_done_func_t *done, void *private, int priority, int flags) |
597 |
{ |
|
598 |
zio_t *zio; |
|
599 |
||
600 |
BP_ZERO(bp); |
|
601 |
BP_SET_LSIZE(bp, size); |
|
602 |
BP_SET_PSIZE(bp, size); |
|
603 |
BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); |
|
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
604 |
BP_SET_CRYPT(bp, ZIO_CRYPT_OFF); |
789 | 605 |
|
606 |
zio = zio_create(pio, spa, txg, bp, data, size, done, private, |
|
607 |
ZIO_TYPE_WRITE, priority, flags, |
|
608 |
ZIO_STAGE_OPEN, ZIO_WRITE_ALLOCATE_PIPELINE); |
|
609 |
||
610 |
zio->io_checksum = checksum; |
|
611 |
zio->io_compress = ZIO_COMPRESS_OFF; |
|
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
612 |
zio->io_crypt = ZIO_CRYPT_OFF; |
5329 | 613 |
zio->io_ready = zio_write_allocate_ready; |
789 | 614 |
|
615 |
return (zio); |
|
616 |
} |
|
617 |
||
618 |
zio_t * |
|
619 |
zio_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, |
|
620 |
zio_done_func_t *done, void *private) |
|
621 |
{ |
|
622 |
zio_t *zio; |
|
623 |
||
624 |
ASSERT(!BP_IS_HOLE(bp)); |
|
625 |
||
626 |
if (txg == spa->spa_syncing_txg && |
|
627 |
spa->spa_sync_pass > zio_sync_pass.zp_defer_free) { |
|
628 |
bplist_enqueue_deferred(&spa->spa_sync_bplist, bp); |
|
629 |
return (zio_null(pio, spa, NULL, NULL, 0)); |
|
630 |
} |
|
631 |
||
632 |
zio = zio_create(pio, spa, txg, bp, NULL, 0, done, private, |
|
2981 | 633 |
ZIO_TYPE_FREE, ZIO_PRIORITY_FREE, ZIO_FLAG_USER, |
5530 | 634 |
ZIO_STAGE_OPEN, ZIO_FREE_PIPELINE(bp)); |
789 | 635 |
|
636 |
zio->io_bp = &zio->io_bp_copy; |
|
637 |
||
638 |
return (zio); |
|
639 |
} |
|
640 |
||
641 |
zio_t * |
|
642 |
zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, |
|
643 |
zio_done_func_t *done, void *private) |
|
644 |
{ |
|
645 |
zio_t *zio; |
|
646 |
||
647 |
/* |
|
648 |
* A claim is an allocation of a specific block. Claims are needed |
|
649 |
* to support immediate writes in the intent log. The issue is that |
|
650 |
* immediate writes contain committed data, but in a txg that was |
|
651 |
* *not* committed. Upon opening the pool after an unclean shutdown, |
|
652 |
* the intent log claims all blocks that contain immediate write data |
|
653 |
* so that the SPA knows they're in use. |
|
654 |
* |
|
655 |
* All claims *must* be resolved in the first txg -- before the SPA |
|
656 |
* starts allocating blocks -- so that nothing is allocated twice. |
|
657 |
*/ |
|
658 |
ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa)); |
|
659 |
ASSERT3U(spa_first_txg(spa), <=, txg); |
|
660 |
||
661 |
zio = zio_create(pio, spa, txg, bp, NULL, 0, done, private, |
|
662 |
ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, 0, |
|
5530 | 663 |
ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE(bp)); |
789 | 664 |
|
665 |
zio->io_bp = &zio->io_bp_copy; |
|
666 |
||
667 |
return (zio); |
|
668 |
} |
|
669 |
||
670 |
zio_t * |
|
671 |
zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, |
|
672 |
zio_done_func_t *done, void *private, int priority, int flags) |
|
673 |
{ |
|
674 |
zio_t *zio; |
|
675 |
int c; |
|
676 |
||
677 |
if (vd->vdev_children == 0) { |
|
678 |
zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, |
|
679 |
ZIO_TYPE_IOCTL, priority, flags, |
|
680 |
ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); |
|
681 |
||
682 |
zio->io_vd = vd; |
|
683 |
zio->io_cmd = cmd; |
|
684 |
} else { |
|
685 |
zio = zio_null(pio, spa, NULL, NULL, flags); |
|
686 |
||
687 |
for (c = 0; c < vd->vdev_children; c++) |
|
688 |
zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, |
|
689 |
done, private, priority, flags)); |
|
690 |
} |
|
691 |
||
692 |
return (zio); |
|
693 |
} |
|
694 |
||
695 |
static void |
|
696 |
zio_phys_bp_init(vdev_t *vd, blkptr_t *bp, uint64_t offset, uint64_t size, |
|
5450 | 697 |
int checksum, boolean_t labels) |
789 | 698 |
{ |
699 |
ASSERT(vd->vdev_children == 0); |
|
700 |
||
701 |
ASSERT(size <= SPA_MAXBLOCKSIZE); |
|
702 |
ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); |
|
703 |
ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); |
|
704 |
||
5450 | 705 |
#ifdef ZFS_DEBUG |
706 |
if (labels) { |
|
707 |
ASSERT(offset + size <= VDEV_LABEL_START_SIZE || |
|
708 |
offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); |
|
709 |
} |
|
710 |
#endif |
|
789 | 711 |
ASSERT3U(offset + size, <=, vd->vdev_psize); |
712 |
||
713 |
BP_ZERO(bp); |
|
714 |
||
715 |
BP_SET_LSIZE(bp, size); |
|
716 |
BP_SET_PSIZE(bp, size); |
|
717 |
||
718 |
BP_SET_CHECKSUM(bp, checksum); |
|
719 |
BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); |
|
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
720 |
BP_SET_CRYPT(bp, ZIO_CRYPT_OFF); |
789 | 721 |
BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); |
722 |
||
723 |
if (checksum != ZIO_CHECKSUM_OFF) |
|
724 |
ZIO_SET_CHECKSUM(&bp->blk_cksum, offset, 0, 0, 0); |
|
725 |
} |
|
726 |
||
727 |
zio_t * |
|
728 |
zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, |
|
729 |
void *data, int checksum, zio_done_func_t *done, void *private, |
|
5450 | 730 |
int priority, int flags, boolean_t labels) |
789 | 731 |
{ |
732 |
zio_t *zio; |
|
733 |
blkptr_t blk; |
|
734 |
||
5329 | 735 |
ZIO_ENTER(vd->vdev_spa); |
736 |
||
5450 | 737 |
zio_phys_bp_init(vd, &blk, offset, size, checksum, labels); |
789 | 738 |
|
739 |
zio = zio_create(pio, vd->vdev_spa, 0, &blk, data, size, done, private, |
|
740 |
ZIO_TYPE_READ, priority, flags | ZIO_FLAG_PHYSICAL, |
|
741 |
ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); |
|
742 |
||
743 |
zio->io_vd = vd; |
|
744 |
zio->io_offset = offset; |
|
745 |
||
746 |
/* |
|
747 |
* Work off our copy of the bp so the caller can free it. |
|
748 |
*/ |
|
749 |
zio->io_bp = &zio->io_bp_copy; |
|
750 |
||
751 |
return (zio); |
|
752 |
} |
|
753 |
||
754 |
zio_t * |
|
755 |
zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, |
|
756 |
void *data, int checksum, zio_done_func_t *done, void *private, |
|
5450 | 757 |
int priority, int flags, boolean_t labels) |
789 | 758 |
{ |
759 |
zio_block_tail_t *zbt; |
|
760 |
void *wbuf; |
|
761 |
zio_t *zio; |
|
762 |
blkptr_t blk; |
|
763 |
||
5329 | 764 |
ZIO_ENTER(vd->vdev_spa); |
765 |
||
5450 | 766 |
zio_phys_bp_init(vd, &blk, offset, size, checksum, labels); |
789 | 767 |
|
768 |
zio = zio_create(pio, vd->vdev_spa, 0, &blk, data, size, done, private, |
|
769 |
ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_PHYSICAL, |
|
770 |
ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); |
|
771 |
||
772 |
zio->io_vd = vd; |
|
773 |
zio->io_offset = offset; |
|
774 |
||
775 |
zio->io_bp = &zio->io_bp_copy; |
|
776 |
zio->io_checksum = checksum; |
|
777 |
||
778 |
if (zio_checksum_table[checksum].ci_zbt) { |
|
779 |
/* |
|
780 |
* zbt checksums are necessarily destructive -- they modify |
|
781 |
* one word of the write buffer to hold the verifier/checksum. |
|
782 |
* Therefore, we must make a local copy in case the data is |
|
783 |
* being written to multiple places. |
|
784 |
*/ |
|
785 |
wbuf = zio_buf_alloc(size); |
|
786 |
bcopy(data, wbuf, size); |
|
787 |
zio_push_transform(zio, wbuf, size, size); |
|
788 |
||
789 |
zbt = (zio_block_tail_t *)((char *)wbuf + size) - 1; |
|
790 |
zbt->zbt_cksum = blk.blk_cksum; |
|
791 |
} |
|
792 |
||
793 |
return (zio); |
|
794 |
} |
|
795 |
||
796 |
/* |
|
797 |
* Create a child I/O to do some work for us. It has no associated bp. |
|
798 |
*/ |
|
799 |
zio_t * |
|
800 |
zio_vdev_child_io(zio_t *zio, blkptr_t *bp, vdev_t *vd, uint64_t offset, |
|
801 |
void *data, uint64_t size, int type, int priority, int flags, |
|
802 |
zio_done_func_t *done, void *private) |
|
803 |
{ |
|
804 |
uint32_t pipeline = ZIO_VDEV_CHILD_PIPELINE; |
|
805 |
zio_t *cio; |
|
806 |
||
807 |
if (type == ZIO_TYPE_READ && bp != NULL) { |
|
808 |
/* |
|
809 |
* If we have the bp, then the child should perform the |
|
810 |
* checksum and the parent need not. This pushes error |
|
811 |
* detection as close to the leaves as possible and |
|
812 |
* eliminates redundant checksums in the interior nodes. |
|
813 |
*/ |
|
814 |
pipeline |= 1U << ZIO_STAGE_CHECKSUM_VERIFY; |
|
815 |
zio->io_pipeline &= ~(1U << ZIO_STAGE_CHECKSUM_VERIFY); |
|
816 |
} |
|
817 |
||
818 |
cio = zio_create(zio, zio->io_spa, zio->io_txg, bp, data, size, |
|
819 |
done, private, type, priority, |
|
820 |
(zio->io_flags & ZIO_FLAG_VDEV_INHERIT) | ZIO_FLAG_CANFAIL | flags, |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
821 |
ZIO_STAGE_VDEV_IO_START - 1, pipeline); |
789 | 822 |
|
823 |
cio->io_vd = vd; |
|
824 |
cio->io_offset = offset; |
|
825 |
||
826 |
return (cio); |
|
827 |
} |
|
828 |
||
829 |
/* |
|
830 |
* ========================================================================== |
|
831 |
* Initiate I/O, either sync or async |
|
832 |
* ========================================================================== |
|
833 |
*/ |
|
6523
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
834 |
static void |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
835 |
zio_destroy(zio_t *zio) |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
836 |
{ |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
837 |
mutex_destroy(&zio->io_lock); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
838 |
cv_destroy(&zio->io_cv); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
839 |
if (zio->io_failed_vds != NULL) { |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
840 |
kmem_free(zio->io_failed_vds, |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
841 |
zio->io_failed_vds_count * sizeof (vdev_t *)); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
842 |
zio->io_failed_vds = NULL; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
843 |
zio->io_failed_vds_count = 0; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
844 |
} |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
845 |
kmem_cache_free(zio_cache, zio); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
846 |
} |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
847 |
|
789 | 848 |
int |
849 |
zio_wait(zio_t *zio) |
|
850 |
{ |
|
851 |
int error; |
|
852 |
||
853 |
ASSERT(zio->io_stage == ZIO_STAGE_OPEN); |
|
854 |
||
855 |
zio->io_waiter = curthread; |
|
856 |
||
5530 | 857 |
zio_execute(zio); |
789 | 858 |
|
859 |
mutex_enter(&zio->io_lock); |
|
860 |
while (zio->io_stalled != ZIO_STAGE_DONE) |
|
861 |
cv_wait(&zio->io_cv, &zio->io_lock); |
|
862 |
mutex_exit(&zio->io_lock); |
|
863 |
||
864 |
error = zio->io_error; |
|
6523
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
865 |
zio_destroy(zio); |
789 | 866 |
|
867 |
return (error); |
|
868 |
} |
|
869 |
||
870 |
void |
|
871 |
zio_nowait(zio_t *zio) |
|
872 |
{ |
|
5530 | 873 |
zio_execute(zio); |
874 |
} |
|
875 |
||
876 |
void |
|
877 |
zio_interrupt(zio_t *zio) |
|
878 |
{ |
|
879 |
(void) taskq_dispatch(zio->io_spa->spa_zio_intr_taskq[zio->io_type], |
|
880 |
(task_func_t *)zio_execute, zio, TQ_SLEEP); |
|
881 |
} |
|
882 |
||
883 |
static int |
|
884 |
zio_issue_async(zio_t *zio) |
|
885 |
{ |
|
886 |
(void) taskq_dispatch(zio->io_spa->spa_zio_issue_taskq[zio->io_type], |
|
887 |
(task_func_t *)zio_execute, zio, TQ_SLEEP); |
|
888 |
||
889 |
return (ZIO_PIPELINE_STOP); |
|
789 | 890 |
} |
891 |
||
892 |
/* |
|
893 |
* ========================================================================== |
|
894 |
* I/O pipeline interlocks: parent/child dependency scoreboarding |
|
895 |
* ========================================================================== |
|
896 |
*/ |
|
5530 | 897 |
static int |
789 | 898 |
zio_wait_for_children(zio_t *zio, uint32_t stage, uint64_t *countp) |
899 |
{ |
|
5530 | 900 |
int rv = ZIO_PIPELINE_CONTINUE; |
901 |
||
789 | 902 |
mutex_enter(&zio->io_lock); |
5530 | 903 |
ASSERT(zio->io_stalled == 0); |
904 |
if (*countp != 0) { |
|
789 | 905 |
zio->io_stalled = stage; |
5530 | 906 |
rv = ZIO_PIPELINE_STOP; |
789 | 907 |
} |
5530 | 908 |
mutex_exit(&zio->io_lock); |
909 |
||
910 |
return (rv); |
|
789 | 911 |
} |
912 |
||
913 |
static void |
|
6523
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
914 |
zio_add_failed_vdev(zio_t *pio, zio_t *zio) |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
915 |
{ |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
916 |
uint64_t oldcount = pio->io_failed_vds_count; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
917 |
vdev_t **new_vds; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
918 |
int i; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
919 |
|
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
920 |
ASSERT(MUTEX_HELD(&pio->io_lock)); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
921 |
|
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
922 |
if (zio->io_vd == NULL) |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
923 |
return; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
924 |
|
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
925 |
for (i = 0; i < oldcount; i++) { |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
926 |
if (pio->io_failed_vds[i] == zio->io_vd) |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
927 |
return; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
928 |
} |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
929 |
|
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
930 |
new_vds = kmem_zalloc((oldcount + 1) * sizeof (vdev_t *), KM_SLEEP); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
931 |
if (pio->io_failed_vds != NULL) { |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
932 |
bcopy(pio->io_failed_vds, new_vds, |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
933 |
oldcount * sizeof (vdev_t *)); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
934 |
kmem_free(pio->io_failed_vds, oldcount * sizeof (vdev_t *)); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
935 |
} |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
936 |
pio->io_failed_vds = new_vds; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
937 |
pio->io_failed_vds[oldcount] = zio->io_vd; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
938 |
pio->io_failed_vds_count++; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
939 |
} |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
940 |
|
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
941 |
static void |
789 | 942 |
zio_notify_parent(zio_t *zio, uint32_t stage, uint64_t *countp) |
943 |
{ |
|
944 |
zio_t *pio = zio->io_parent; |
|
945 |
||
946 |
mutex_enter(&pio->io_lock); |
|
6523
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
947 |
if (pio->io_error == 0 && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) { |
789 | 948 |
pio->io_error = zio->io_error; |
6523
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
949 |
if (zio->io_error && zio->io_error != ENOTSUP) |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
950 |
zio_add_failed_vdev(pio, zio); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
951 |
} |
5329 | 952 |
ASSERT3U(*countp, >, 0); |
789 | 953 |
if (--*countp == 0 && pio->io_stalled == stage) { |
954 |
pio->io_stalled = 0; |
|
955 |
mutex_exit(&pio->io_lock); |
|
5530 | 956 |
zio_execute(pio); |
789 | 957 |
} else { |
958 |
mutex_exit(&pio->io_lock); |
|
959 |
} |
|
960 |
} |
|
961 |
||
5530 | 962 |
int |
963 |
zio_wait_for_children_ready(zio_t *zio) |
|
789 | 964 |
{ |
5530 | 965 |
return (zio_wait_for_children(zio, ZIO_STAGE_WAIT_FOR_CHILDREN_READY, |
966 |
&zio->io_children_notready)); |
|
789 | 967 |
} |
968 |
||
5530 | 969 |
int |
970 |
zio_wait_for_children_done(zio_t *zio) |
|
789 | 971 |
{ |
5530 | 972 |
return (zio_wait_for_children(zio, ZIO_STAGE_WAIT_FOR_CHILDREN_DONE, |
973 |
&zio->io_children_notdone)); |
|
789 | 974 |
} |
975 |
||
5530 | 976 |
static int |
5329 | 977 |
zio_read_init(zio_t *zio) |
978 |
{ |
|
5530 | 979 |
blkptr_t *bp = zio->io_bp; |
13361
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
980 |
int crypt = BP_GET_CRYPT(bp); |
5530 | 981 |
|
13308
b11402c5f64b
2207 scrub/resilver says pools have errors when key not present
Darren Moffat <darrenm@opensolaris.org>
parents:
13307
diff
changeset
|
982 |
/* |
b11402c5f64b
2207 scrub/resilver says pools have errors when key not present
Darren Moffat <darrenm@opensolaris.org>
parents:
13307
diff
changeset
|
983 |
* Don't add crypto to the pipeline if this is a scrub/resilver |
b11402c5f64b
2207 scrub/resilver says pools have errors when key not present
Darren Moffat <darrenm@opensolaris.org>
parents:
13307
diff
changeset
|
984 |
* because we don't need the actual data in that case and the |
b11402c5f64b
2207 scrub/resilver says pools have errors when key not present
Darren Moffat <darrenm@opensolaris.org>
parents:
13307
diff
changeset
|
985 |
* decrypt could fail due to a lack of a key (in which case the |
b11402c5f64b
2207 scrub/resilver says pools have errors when key not present
Darren Moffat <darrenm@opensolaris.org>
parents:
13307
diff
changeset
|
986 |
* datasets wouldn't be mounted but still need to be scrubbed). |
b11402c5f64b
2207 scrub/resilver says pools have errors when key not present
Darren Moffat <darrenm@opensolaris.org>
parents:
13307
diff
changeset
|
987 |
*/ |
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
988 |
|
13361
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
989 |
if ((spa_version(zio->io_spa) >= SPA_VERSION_CRYPTO) && |
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
990 |
(crypt != ZIO_CRYPT_OFF && crypt != ZIO_CRYPT_INHERIT)) { |
13368
2661054b6aaa
6738244 zpool scrub shows errors when key present
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13361
diff
changeset
|
991 |
uint64_t csize; |
2661054b6aaa
6738244 zpool scrub shows errors when key present
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13361
diff
changeset
|
992 |
void *cbuf; |
2661054b6aaa
6738244 zpool scrub shows errors when key present
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13361
diff
changeset
|
993 |
if (!(zio->io_flags & ZIO_FLAG_SCRUB || |
2661054b6aaa
6738244 zpool scrub shows errors when key present
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13361
diff
changeset
|
994 |
zio->io_flags & ZIO_FLAG_RESILVER)) { |
2661054b6aaa
6738244 zpool scrub shows errors when key present
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13361
diff
changeset
|
995 |
csize = BP_GET_PSIZE(bp); |
2661054b6aaa
6738244 zpool scrub shows errors when key present
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13361
diff
changeset
|
996 |
cbuf = zio_buf_alloc(csize); |
2661054b6aaa
6738244 zpool scrub shows errors when key present
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13361
diff
changeset
|
997 |
zio_push_transform(zio, cbuf, csize, csize); |
2661054b6aaa
6738244 zpool scrub shows errors when key present
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13361
diff
changeset
|
998 |
zio->io_pipeline |= 1U << ZIO_STAGE_READ_DECRYPT; |
2661054b6aaa
6738244 zpool scrub shows errors when key present
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13361
diff
changeset
|
999 |
} |
13361
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1000 |
} |
13155
0d4586a98ec6
encryption disappeared out of the pipeline on last resync with onnv-gate
Darren Moffat <darrenm@opensolaris.org>
parents:
13152
diff
changeset
|
1001 |
|
5530 | 1002 |
if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) { |
1003 |
uint64_t csize = BP_GET_PSIZE(bp); |
|
5329 | 1004 |
void *cbuf = zio_buf_alloc(csize); |
1005 |
||
1006 |
zio_push_transform(zio, cbuf, csize, csize); |
|
1007 |
zio->io_pipeline |= 1U << ZIO_STAGE_READ_DECOMPRESS; |
|
1008 |
} |
|
1009 |
||
5530 | 1010 |
if (BP_IS_GANG(bp)) { |
5329 | 1011 |
uint64_t gsize = SPA_GANGBLOCKSIZE; |
1012 |
void *gbuf = zio_buf_alloc(gsize); |
|
1013 |
||
1014 |
zio_push_transform(zio, gbuf, gsize, gsize); |
|
1015 |
zio->io_pipeline |= 1U << ZIO_STAGE_READ_GANG_MEMBERS; |
|
1016 |
} |
|
5530 | 1017 |
|
1018 |
if (!dmu_ot[BP_GET_TYPE(bp)].ot_metadata && BP_GET_LEVEL(bp) == 0) |
|
1019 |
zio->io_flags |= ZIO_FLAG_DONT_CACHE; |
|
1020 |
||
1021 |
return (ZIO_PIPELINE_CONTINUE); |
|
5329 | 1022 |
} |
1023 |
||
5530 | 1024 |
static int |
789 | 1025 |
zio_ready(zio_t *zio) |
1026 |
{ |
|
1027 |
zio_t *pio = zio->io_parent; |
|
1028 |
||
3547
e396e0a440b1
6512391 DMU should leverage ZIO dependencies to achieve greater parallelism
maybee
parents:
3463
diff
changeset
|
1029 |
if (zio->io_ready) |
e396e0a440b1
6512391 DMU should leverage ZIO dependencies to achieve greater parallelism
maybee
parents:
3463
diff
changeset
|
1030 |
zio->io_ready(zio); |
e396e0a440b1
6512391 DMU should leverage ZIO dependencies to achieve greater parallelism
maybee
parents:
3463
diff
changeset
|
1031 |
|
789 | 1032 |
if (pio != NULL) |
5530 | 1033 |
zio_notify_parent(zio, ZIO_STAGE_WAIT_FOR_CHILDREN_READY, |
789 | 1034 |
&pio->io_children_notready); |
1035 |
||
1036 |
if (zio->io_bp) |
|
1037 |
zio->io_bp_copy = *zio->io_bp; |
|
1038 |
||
5530 | 1039 |
return (ZIO_PIPELINE_CONTINUE); |
789 | 1040 |
} |
1041 |
||
5530 | 1042 |
static int |
5329 | 1043 |
zio_vdev_retry_io(zio_t *zio) |
789 | 1044 |
{ |
1045 |
zio_t *pio = zio->io_parent; |
|
5329 | 1046 |
|
1047 |
/* |
|
1048 |
* Preserve the failed bp so that the io_ready() callback can |
|
1049 |
* update the accounting accordingly. The callback will also be |
|
1050 |
* responsible for freeing the previously allocated block, if one |
|
1051 |
* exists. |
|
1052 |
*/ |
|
1053 |
zio->io_bp_orig = *zio->io_bp; |
|
1054 |
||
1055 |
/* |
|
1056 |
* We must zero out the old DVA and blk_birth before reallocating |
|
5403 | 1057 |
* the bp. |
5329 | 1058 |
*/ |
5403 | 1059 |
BP_ZERO_DVAS(zio->io_bp); |
5329 | 1060 |
zio_reset(zio); |
1061 |
||
1062 |
if (pio) { |
|
1063 |
/* |
|
1064 |
* Let the parent know that we will |
|
1065 |
* re-alloc the write (=> new bp info). |
|
1066 |
*/ |
|
1067 |
mutex_enter(&pio->io_lock); |
|
1068 |
pio->io_children_notready++; |
|
1069 |
||
1070 |
/* |
|
1071 |
* If the parent I/O is still in the open stage, then |
|
1072 |
* don't bother telling it to retry since it hasn't |
|
1073 |
* progressed far enough for it to care. |
|
1074 |
*/ |
|
1075 |
if (pio->io_stage > ZIO_STAGE_OPEN && IO_IS_ALLOCATING(pio)) |
|
1076 |
pio->io_flags |= ZIO_FLAG_WRITE_RETRY; |
|
1077 |
||
5530 | 1078 |
ASSERT(pio->io_stage <= ZIO_STAGE_WAIT_FOR_CHILDREN_DONE); |
5329 | 1079 |
mutex_exit(&pio->io_lock); |
1080 |
} |
|
1081 |
||
1082 |
/* |
|
1083 |
* We are getting ready to process the retry request so clear |
|
1084 |
* the flag and the zio's current error status. |
|
1085 |
*/ |
|
1086 |
zio->io_flags &= ~ZIO_FLAG_WRITE_RETRY; |
|
1087 |
zio->io_error = 0; |
|
5530 | 1088 |
|
1089 |
return (ZIO_PIPELINE_CONTINUE); |
|
5329 | 1090 |
} |
1091 |
||
1092 |
int |
|
1093 |
zio_vdev_resume_io(spa_t *spa) |
|
1094 |
{ |
|
1095 |
zio_t *zio; |
|
1096 |
||
1097 |
mutex_enter(&spa->spa_zio_lock); |
|
1098 |
||
1099 |
/* |
|
1100 |
* Probe all of vdevs that have experienced an I/O error. |
|
1101 |
* If we are still unable to verify the integrity of the vdev |
|
1102 |
* then we prevent the resume from proceeeding. |
|
1103 |
*/ |
|
1104 |
for (zio = list_head(&spa->spa_zio_list); zio != NULL; |
|
1105 |
zio = list_next(&spa->spa_zio_list, zio)) { |
|
1106 |
int error = 0; |
|
1107 |
||
1108 |
/* We only care about I/Os that must succeed */ |
|
1109 |
if (zio->io_vd == NULL || zio->io_flags & ZIO_FLAG_CANFAIL) |
|
1110 |
continue; |
|
1111 |
error = vdev_probe(zio->io_vd); |
|
1112 |
if (error) { |
|
1113 |
mutex_exit(&spa->spa_zio_lock); |
|
1114 |
return (error); |
|
1115 |
} |
|
1116 |
} |
|
1117 |
||
1118 |
/* |
|
1119 |
* Clear the vdev stats so that I/O can flow. |
|
1120 |
*/ |
|
1121 |
vdev_clear(spa, NULL, B_FALSE); |
|
1122 |
||
1123 |
spa->spa_state = POOL_STATE_ACTIVE; |
|
1124 |
while ((zio = list_head(&spa->spa_zio_list)) != NULL) { |
|
1125 |
list_remove(&spa->spa_zio_list, zio); |
|
1126 |
zio->io_error = 0; |
|
1127 |
||
1128 |
/* |
|
1129 |
* If we are resuming an allocating I/O then we force it |
|
1130 |
* to retry and let it resume operation where it left off. |
|
1131 |
* Otherwise, go back to the ready stage and pick up from |
|
1132 |
* there. |
|
1133 |
*/ |
|
1134 |
if (zio_write_retry && IO_IS_ALLOCATING(zio)) { |
|
1135 |
zio->io_flags |= ZIO_FLAG_WRITE_RETRY; |
|
1136 |
zio->io_stage--; |
|
1137 |
} else { |
|
1138 |
zio->io_stage = ZIO_STAGE_READY; |
|
1139 |
} |
|
1140 |
||
5530 | 1141 |
(void) taskq_dispatch(zio_taskq, (task_func_t *)zio_execute, |
5329 | 1142 |
zio, TQ_SLEEP); |
1143 |
} |
|
1144 |
mutex_exit(&spa->spa_zio_lock); |
|
1145 |
||
1146 |
/* |
|
1147 |
* Wait for the taskqs to finish and recheck the pool state since |
|
1148 |
* it's possible that a resumed I/O has failed again. |
|
1149 |
*/ |
|
1150 |
taskq_wait(zio_taskq); |
|
1151 |
if (spa_state(spa) == POOL_STATE_IO_FAILURE) |
|
1152 |
return (EIO); |
|
1153 |
||
1154 |
mutex_enter(&spa->spa_zio_lock); |
|
1155 |
cv_broadcast(&spa->spa_zio_cv); |
|
1156 |
mutex_exit(&spa->spa_zio_lock); |
|
1157 |
||
1158 |
return (0); |
|
1159 |
} |
|
1160 |
||
5530 | 1161 |
static int |
5329 | 1162 |
zio_vdev_suspend_io(zio_t *zio) |
1163 |
{ |
|
1164 |
spa_t *spa = zio->io_spa; |
|
1165 |
||
1166 |
/* |
|
1167 |
* We've experienced an unrecoverable failure so |
|
1168 |
* set the pool state accordingly and queue all |
|
1169 |
* failed IOs. |
|
1170 |
*/ |
|
1171 |
spa->spa_state = POOL_STATE_IO_FAILURE; |
|
1172 |
||
1173 |
mutex_enter(&spa->spa_zio_lock); |
|
1174 |
list_insert_tail(&spa->spa_zio_list, zio); |
|
1175 |
||
1176 |
#ifndef _KERNEL |
|
1177 |
/* Used to notify ztest that the pool has suspended */ |
|
1178 |
cv_broadcast(&spa->spa_zio_cv); |
|
1179 |
#endif |
|
1180 |
mutex_exit(&spa->spa_zio_lock); |
|
5530 | 1181 |
|
1182 |
return (ZIO_PIPELINE_STOP); |
|
5329 | 1183 |
} |
1184 |
||
6523
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1185 |
static void |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1186 |
zio_handle_io_failure(zio_t *zio, vdev_t *vd) |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1187 |
{ |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1188 |
spa_t *spa = zio->io_spa; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1189 |
blkptr_t *bp = zio->io_bp; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1190 |
char *blkbuf; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1191 |
|
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1192 |
#ifdef ZFS_DEBUG |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1193 |
blkbuf = kmem_alloc(BP_SPRINTF_LEN, KM_NOSLEEP); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1194 |
if (blkbuf) { |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1195 |
sprintf_blkptr(blkbuf, BP_SPRINTF_LEN, |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1196 |
bp ? bp : &zio->io_bp_copy); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1197 |
} |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1198 |
cmn_err(CE_WARN, "ZFS: %s (%s on %s off %llx: zio %p %s): error %d", |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1199 |
zio->io_error == ECKSUM ? "bad checksum" : "I/O failure", |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1200 |
zio_type_name[zio->io_type], vdev_description(vd), |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1201 |
(u_longlong_t)zio->io_offset, (void *)zio, |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1202 |
blkbuf ? blkbuf : "", zio->io_error); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1203 |
if (blkbuf) |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1204 |
kmem_free(blkbuf, BP_SPRINTF_LEN); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1205 |
#endif |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1206 |
|
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1207 |
if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC) { |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1208 |
fm_panic("Pool '%s' has encountered an uncorrectable I/O " |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1209 |
"failure and the failure mode property for this pool " |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1210 |
"is set to panic.", spa_name(spa)); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1211 |
} |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1212 |
zfs_ereport_post(FM_EREPORT_ZFS_IO_FAILURE, spa, NULL, NULL, 0, 0); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1213 |
vdev_set_state(vd, vd == spa->spa_root_vdev ? B_TRUE : B_FALSE, |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1214 |
VDEV_STATE_FAULTED, VDEV_AUX_IO_FAILURE); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1215 |
} |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1216 |
|
5530 | 1217 |
static int |
5329 | 1218 |
zio_assess(zio_t *zio) |
1219 |
{ |
|
789 | 1220 |
spa_t *spa = zio->io_spa; |
1221 |
blkptr_t *bp = zio->io_bp; |
|
1222 |
vdev_t *vd = zio->io_vd; |
|
1223 |
||
1224 |
ASSERT(zio->io_children_notready == 0); |
|
1225 |
ASSERT(zio->io_children_notdone == 0); |
|
1226 |
||
1227 |
if (bp != NULL) { |
|
1228 |
ASSERT(bp->blk_pad[0] == 0); |
|
1229 |
ASSERT(bp->blk_pad[1] == 0); |
|
1230 |
ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0); |
|
1231 |
if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) && |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1232 |
!(zio->io_flags & ZIO_FLAG_IO_REPAIR)) { |
789 | 1233 |
ASSERT(!BP_SHOULD_BYTESWAP(bp)); |
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1234 |
if (zio->io_ndvas != 0) |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1235 |
ASSERT3U(zio->io_ndvas, <=, BP_GET_NDVAS(bp)); |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1236 |
ASSERT(BP_COUNT_GANG(bp) == 0 || |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1237 |
(BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp))); |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1238 |
} |
789 | 1239 |
} |
1240 |
||
5329 | 1241 |
/* |
1242 |
* Some child I/O has indicated that a retry is necessary, so |
|
1243 |
* we set an error on the I/O and let the logic below do the |
|
1244 |
* rest. |
|
1245 |
*/ |
|
1246 |
if (zio->io_flags & ZIO_FLAG_WRITE_RETRY) |
|
1247 |
zio->io_error = ERESTART; |
|
1248 |
||
789 | 1249 |
if (vd != NULL) |
1250 |
vdev_stat_update(zio); |
|
1251 |
||
1252 |
if (zio->io_error) { |
|
1544 | 1253 |
/* |
1254 |
* If this I/O is attached to a particular vdev, |
|
1255 |
* generate an error message describing the I/O failure |
|
1256 |
* at the block level. We ignore these errors if the |
|
1257 |
* device is currently unavailable. |
|
1258 |
*/ |
|
1732 | 1259 |
if (zio->io_error != ECKSUM && vd != NULL && !vdev_is_dead(vd)) |
5329 | 1260 |
zfs_ereport_post(FM_EREPORT_ZFS_IO, spa, vd, zio, 0, 0); |
789 | 1261 |
|
1544 | 1262 |
if ((zio->io_error == EIO || |
1263 |
!(zio->io_flags & ZIO_FLAG_SPECULATIVE)) && |
|
1264 |
zio->io_logical == zio) { |
|
1265 |
/* |
|
1266 |
* For root I/O requests, tell the SPA to log the error |
|
1267 |
* appropriately. Also, generate a logical data |
|
1268 |
* ereport. |
|
1269 |
*/ |
|
5329 | 1270 |
spa_log_error(spa, zio); |
1544 | 1271 |
|
5329 | 1272 |
zfs_ereport_post(FM_EREPORT_ZFS_DATA, spa, NULL, zio, |
1273 |
0, 0); |
|
1544 | 1274 |
} |
789 | 1275 |
|
1544 | 1276 |
/* |
5403 | 1277 |
* If we are an allocating I/O then we attempt to reissue |
1278 |
* the I/O on another vdev unless the pool is out of space. |
|
1279 |
* We handle this condition based on the spa's failmode |
|
1280 |
* property. |
|
5329 | 1281 |
*/ |
1282 |
if (zio_write_retry && zio->io_error != ENOSPC && |
|
5530 | 1283 |
IO_IS_ALLOCATING(zio)) |
1284 |
return (zio_vdev_retry_io(zio)); |
|
1285 |
||
5329 | 1286 |
ASSERT(!(zio->io_flags & ZIO_FLAG_WRITE_RETRY)); |
1287 |
||
1288 |
/* |
|
1289 |
* For I/O requests that cannot fail, we carry out |
|
1290 |
* the requested behavior based on the failmode pool |
|
1291 |
* property. |
|
1292 |
* |
|
1293 |
* XXX - Need to differentiate between an ENOSPC as |
|
1294 |
* a result of vdev failures vs. a full pool. |
|
1544 | 1295 |
*/ |
1296 |
if (!(zio->io_flags & ZIO_FLAG_CANFAIL)) { |
|
6523
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1297 |
int i; |
3459
358b6bdb15b6
6512311 zio_done() is chunky and needs to go on a diet
ek110237
parents:
3290
diff
changeset
|
1298 |
|
6523
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1299 |
for (i = 0; i < zio->io_failed_vds_count; i++) { |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1300 |
zio_handle_io_failure(zio, |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1301 |
zio->io_failed_vds[i]); |
3459
358b6bdb15b6
6512311 zio_done() is chunky and needs to go on a diet
ek110237
parents:
3290
diff
changeset
|
1302 |
} |
6523
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1303 |
if (zio->io_failed_vds_count == 0) { |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1304 |
zio_handle_io_failure(zio, |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1305 |
vd ? vd : spa->spa_root_vdev); |
5329 | 1306 |
} |
6523
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1307 |
if (zio->io_failed_vds != NULL) { |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1308 |
kmem_free(zio->io_failed_vds, |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1309 |
zio->io_failed_vds_count * |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1310 |
sizeof (vdev_t *)); |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1311 |
zio->io_failed_vds = NULL; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1312 |
zio->io_failed_vds_count = 0; |
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1313 |
} |
5530 | 1314 |
return (zio_vdev_suspend_io(zio)); |
1544 | 1315 |
} |
789 | 1316 |
} |
5329 | 1317 |
ASSERT(!(zio->io_flags & ZIO_FLAG_WRITE_RETRY)); |
1318 |
ASSERT(zio->io_children_notready == 0); |
|
5530 | 1319 |
|
1320 |
return (ZIO_PIPELINE_CONTINUE); |
|
5329 | 1321 |
} |
1322 |
||
5530 | 1323 |
static int |
5329 | 1324 |
zio_done(zio_t *zio) |
1325 |
{ |
|
1326 |
zio_t *pio = zio->io_parent; |
|
1327 |
spa_t *spa = zio->io_spa; |
|
1328 |
||
1329 |
ASSERT(zio->io_children_notready == 0); |
|
1330 |
ASSERT(zio->io_children_notdone == 0); |
|
1331 |
||
789 | 1332 |
zio_clear_transform_stack(zio); |
1333 |
||
1334 |
if (zio->io_done) |
|
1335 |
zio->io_done(zio); |
|
1336 |
||
1337 |
ASSERT(zio->io_delegate_list == NULL); |
|
1338 |
ASSERT(zio->io_delegate_next == NULL); |
|
1339 |
||
1340 |
if (pio != NULL) { |
|
1341 |
zio_t *next, *prev; |
|
1342 |
||
1343 |
mutex_enter(&pio->io_lock); |
|
1344 |
next = zio->io_sibling_next; |
|
1345 |
prev = zio->io_sibling_prev; |
|
1346 |
if (next != NULL) |
|
1347 |
next->io_sibling_prev = prev; |
|
1348 |
if (prev != NULL) |
|
1349 |
prev->io_sibling_next = next; |
|
1350 |
if (pio->io_child == zio) |
|
1351 |
pio->io_child = next; |
|
1352 |
mutex_exit(&pio->io_lock); |
|
1353 |
||
5530 | 1354 |
zio_notify_parent(zio, ZIO_STAGE_WAIT_FOR_CHILDREN_DONE, |
789 | 1355 |
&pio->io_children_notdone); |
1356 |
} |
|
1357 |
||
3463 | 1358 |
/* |
4055 | 1359 |
* Note: this I/O is now done, and will shortly be freed, so there is no |
1360 |
* need to clear this (or any other) flag. |
|
3463 | 1361 |
*/ |
1362 |
if (zio->io_flags & ZIO_FLAG_CONFIG_GRABBED) |
|
1544 | 1363 |
spa_config_exit(spa, zio); |
789 | 1364 |
|
1365 |
if (zio->io_waiter != NULL) { |
|
1366 |
mutex_enter(&zio->io_lock); |
|
1367 |
ASSERT(zio->io_stage == ZIO_STAGE_DONE); |
|
1368 |
zio->io_stalled = zio->io_stage; |
|
1369 |
cv_broadcast(&zio->io_cv); |
|
1370 |
mutex_exit(&zio->io_lock); |
|
1371 |
} else { |
|
6523
c1d2a7f04573
6616739 panic message ZFS: I/O failure (write on <unknown> is not very helpful
ek110237
parents:
6245
diff
changeset
|
1372 |
zio_destroy(zio); |
789 | 1373 |
} |
5530 | 1374 |
|
1375 |
return (ZIO_PIPELINE_STOP); |
|
789 | 1376 |
} |
1377 |
||
1378 |
/* |
|
1379 |
* ========================================================================== |
|
1380 |
* Compression support |
|
1381 |
* ========================================================================== |
|
1382 |
*/ |
|
5530 | 1383 |
static int |
789 | 1384 |
zio_write_compress(zio_t *zio) |
1385 |
{ |
|
1386 |
int compress = zio->io_compress; |
|
1387 |
blkptr_t *bp = zio->io_bp; |
|
1388 |
void *cbuf; |
|
1389 |
uint64_t lsize = zio->io_size; |
|
1390 |
uint64_t csize = lsize; |
|
1391 |
uint64_t cbufsize = 0; |
|
1392 |
int pass; |
|
1393 |
||
1394 |
if (bp->blk_birth == zio->io_txg) { |
|
1395 |
/* |
|
1396 |
* We're rewriting an existing block, which means we're |
|
1397 |
* working on behalf of spa_sync(). For spa_sync() to |
|
1398 |
* converge, it must eventually be the case that we don't |
|
1399 |
* have to allocate new blocks. But compression changes |
|
1400 |
* the blocksize, which forces a reallocate, and makes |
|
1401 |
* convergence take longer. Therefore, after the first |
|
1402 |
* few passes, stop compressing to ensure convergence. |
|
1403 |
*/ |
|
1404 |
pass = spa_sync_pass(zio->io_spa); |
|
1405 |
if (pass > zio_sync_pass.zp_dontcompress) |
|
1406 |
compress = ZIO_COMPRESS_OFF; |
|
1407 |
} else { |
|
1408 |
ASSERT(BP_IS_HOLE(bp)); |
|
1409 |
pass = 1; |
|
1410 |
} |
|
1411 |
||
1412 |
if (compress != ZIO_COMPRESS_OFF) |
|
1413 |
if (!zio_compress_data(compress, zio->io_data, zio->io_size, |
|
1414 |
&cbuf, &csize, &cbufsize)) |
|
1415 |
compress = ZIO_COMPRESS_OFF; |
|
1416 |
||
1417 |
if (compress != ZIO_COMPRESS_OFF && csize != 0) |
|
1418 |
zio_push_transform(zio, cbuf, csize, cbufsize); |
|
1419 |
||
1420 |
/* |
|
1421 |
* The final pass of spa_sync() must be all rewrites, but the first |
|
1422 |
* few passes offer a trade-off: allocating blocks defers convergence, |
|
1423 |
* but newly allocated blocks are sequential, so they can be written |
|
1424 |
* to disk faster. Therefore, we allow the first few passes of |
|
1425 |
* spa_sync() to reallocate new blocks, but force rewrites after that. |
|
1426 |
* There should only be a handful of blocks after pass 1 in any case. |
|
1427 |
*/ |
|
1428 |
if (bp->blk_birth == zio->io_txg && BP_GET_PSIZE(bp) == csize && |
|
1429 |
pass > zio_sync_pass.zp_rewrite) { |
|
1430 |
ASSERT(csize != 0); |
|
2885 | 1431 |
BP_SET_LSIZE(bp, lsize); |
1432 |
BP_SET_COMPRESS(bp, compress); |
|
5530 | 1433 |
zio->io_pipeline = ZIO_REWRITE_PIPELINE(bp); |
789 | 1434 |
} else { |
3882 | 1435 |
if (bp->blk_birth == zio->io_txg) |
1436 |
BP_ZERO(bp); |
|
789 | 1437 |
if (csize == 0) { |
1438 |
BP_ZERO(bp); |
|
1439 |
zio->io_pipeline = ZIO_WAIT_FOR_CHILDREN_PIPELINE; |
|
1440 |
} else { |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1441 |
ASSERT3U(BP_GET_NDVAS(bp), ==, 0); |
789 | 1442 |
BP_SET_LSIZE(bp, lsize); |
1443 |
BP_SET_PSIZE(bp, csize); |
|
1444 |
BP_SET_COMPRESS(bp, compress); |
|
1445 |
} |
|
1446 |
} |
|
1447 |
||
5530 | 1448 |
return (ZIO_PIPELINE_CONTINUE); |
789 | 1449 |
} |
1450 |
||
5530 | 1451 |
static int |
789 | 1452 |
zio_read_decompress(zio_t *zio) |
1453 |
{ |
|
1454 |
blkptr_t *bp = zio->io_bp; |
|
1455 |
void *data; |
|
1456 |
uint64_t size; |
|
1457 |
uint64_t bufsize; |
|
1458 |
int compress = BP_GET_COMPRESS(bp); |
|
1459 |
||
1460 |
ASSERT(compress != ZIO_COMPRESS_OFF); |
|
1461 |
||
1462 |
zio_pop_transform(zio, &data, &size, &bufsize); |
|
1463 |
||
1464 |
if (zio_decompress_data(compress, data, size, |
|
1465 |
zio->io_data, zio->io_size)) |
|
1466 |
zio->io_error = EIO; |
|
1467 |
||
1468 |
zio_buf_free(data, bufsize); |
|
1469 |
||
5530 | 1470 |
return (ZIO_PIPELINE_CONTINUE); |
789 | 1471 |
} |
1472 |
||
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1473 |
|
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1474 |
/* |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1475 |
* ========================================================================== |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1476 |
* Cryptographic support |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1477 |
* ========================================================================== |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1478 |
*/ |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1479 |
|
13152 | 1480 |
static int |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1481 |
zio_write_encrypt(zio_t *zio) |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1482 |
{ |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1483 |
int crypt = zio->io_crypt; |
13361
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1484 |
int crypt_error = EAGAIN; |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1485 |
blkptr_t *bp = zio->io_bp; |
13347
90a127dff2c5
6719749 dnode dn_bonus needs to be encrypted
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13338
diff
changeset
|
1486 |
zbookmark_t *bookmark = &zio->io_bookmark; |
13307
3d90d41b6103
1551 data decryption cross architecture fails in AES_CCM
Darren Moffat <darrenm@opensolaris.org>
parents:
13301
diff
changeset
|
1487 |
zio_cksum_t *mac = NULL; |
13186
a6948d45b93f
AES_CCM bug fixes
Darren Moffat <darrenm@opensolaris.org>
parents:
13177
diff
changeset
|
1488 |
size_t maclen; |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1489 |
void *cbuf; |
13160
e8b02045519a
295 mac+checksum in zio_checksum_t (initial draft)
Darren Moffat <darrenm@opensolaris.org>
parents:
13158
diff
changeset
|
1490 |
uint64_t cbufsize; |
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1491 |
boolean_t isdnode = B_FALSE; |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1492 |
|
13326
00ca5372f202
6719684 zfs destroy on clear text can cause panic
Darren Moffat <darrenm@opensolaris.org>
parents:
13323
diff
changeset
|
1493 |
if (spa_version(zio->io_spa) < SPA_VERSION_CRYPTO || |
00ca5372f202
6719684 zfs destroy on clear text can cause panic
Darren Moffat <darrenm@opensolaris.org>
parents:
13323
diff
changeset
|
1494 |
crypt == ZIO_CRYPT_OFF) |
13251
f037017b1f9b
1701 zio_write_encrypt ASSERT trips on old spa version
Darren Moffat <darrenm@opensolaris.org>
parents:
13230
diff
changeset
|
1495 |
return (ZIO_PIPELINE_CONTINUE); |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1496 |
|
13326
00ca5372f202
6719684 zfs destroy on clear text can cause panic
Darren Moffat <darrenm@opensolaris.org>
parents:
13323
diff
changeset
|
1497 |
ASSERT3U(crypt, !=, ZIO_CRYPT_INHERIT); |
13369
32a041998ab1
6745678 zio->io_checksum == ZIO_CHECKSUM_SHA256_CCM_MAC (0x5 == 0x9), file: zio.c, line: 1498
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13368
diff
changeset
|
1498 |
/* |
32a041998ab1
6745678 zio->io_checksum == ZIO_CHECKSUM_SHA256_CCM_MAC (0x5 == 0x9), file: zio.c, line: 1498
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13368
diff
changeset
|
1499 |
* These asserts are a problem for ZVOLS, remove them for now |
32a041998ab1
6745678 zio->io_checksum == ZIO_CHECKSUM_SHA256_CCM_MAC (0x5 == 0x9), file: zio.c, line: 1498
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13368
diff
changeset
|
1500 |
* but we should be able to assert at least the checksum case, |
32a041998ab1
6745678 zio->io_checksum == ZIO_CHECKSUM_SHA256_CCM_MAC (0x5 == 0x9), file: zio.c, line: 1498
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13368
diff
changeset
|
1501 |
* even for ZVOL, is problem that they didn't go through |
32a041998ab1
6745678 zio->io_checksum == ZIO_CHECKSUM_SHA256_CCM_MAC (0x5 == 0x9), file: zio.c, line: 1498
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13368
diff
changeset
|
1502 |
* the write_policy() call ? |
32a041998ab1
6745678 zio->io_checksum == ZIO_CHECKSUM_SHA256_CCM_MAC (0x5 == 0x9), file: zio.c, line: 1498
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13368
diff
changeset
|
1503 |
* |
32a041998ab1
6745678 zio->io_checksum == ZIO_CHECKSUM_SHA256_CCM_MAC (0x5 == 0x9), file: zio.c, line: 1498
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13368
diff
changeset
|
1504 |
* ASSERT3U(zio->io_checksum, ==, ZIO_CHECKSUM_SHA256_CCM_MAC); |
32a041998ab1
6745678 zio->io_checksum == ZIO_CHECKSUM_SHA256_CCM_MAC (0x5 == 0x9), file: zio.c, line: 1498
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13368
diff
changeset
|
1505 |
* ASSERT3U(bookmark->zb_level, ==, 0); |
32a041998ab1
6745678 zio->io_checksum == ZIO_CHECKSUM_SHA256_CCM_MAC (0x5 == 0x9), file: zio.c, line: 1498
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13368
diff
changeset
|
1506 |
*/ |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1507 |
|
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1508 |
/* |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1509 |
* We pass in the zbookmark_t so we can lookup which key |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1510 |
* is needed. The io_txg is used by some mechansims as input |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1511 |
* into the IV generation routine. |
13186
a6948d45b93f
AES_CCM bug fixes
Darren Moffat <darrenm@opensolaris.org>
parents:
13177
diff
changeset
|
1512 |
* |
13307
3d90d41b6103
1551 data decryption cross architecture fails in AES_CCM
Darren Moffat <darrenm@opensolaris.org>
parents:
13301
diff
changeset
|
1513 |
* Note we don't need to pass in the length of the mac since |
3d90d41b6103
1551 data decryption cross architecture fails in AES_CCM
Darren Moffat <darrenm@opensolaris.org>
parents:
13301
diff
changeset
|
1514 |
* it is already checked for in zio_encrypt_data. |
13186
a6948d45b93f
AES_CCM bug fixes
Darren Moffat <darrenm@opensolaris.org>
parents:
13177
diff
changeset
|
1515 |
* Caller of zio_encrypt_data is responsible for freeing buffers |
a6948d45b93f
AES_CCM bug fixes
Darren Moffat <darrenm@opensolaris.org>
parents:
13177
diff
changeset
|
1516 |
* including the mac. |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1517 |
*/ |
13347
90a127dff2c5
6719749 dnode dn_bonus needs to be encrypted
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13338
diff
changeset
|
1518 |
isdnode = (bookmark->zb_object == 0) && (bookmark->zb_level == 0); |
90a127dff2c5
6719749 dnode dn_bonus needs to be encrypted
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13338
diff
changeset
|
1519 |
|
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1520 |
if (zio->io_ckey == NULL) { |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1521 |
zio->io_ckey = zio_crypt_key_lookup(zio->io_spa, |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1522 |
zio->io_bookmark.zb_objset, zio->io_crypt); |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1523 |
if (zio->io_ckey == NULL) { |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1524 |
crypt_error = EAGAIN; |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1525 |
} else { |
13361
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1526 |
zio_crypt_key_hold(zio->io_ckey, FTAG); |
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1527 |
} |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1528 |
} |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1529 |
|
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1530 |
if (zio->io_ckey != NULL) { |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1531 |
crypt_error = zio_encrypt_data(crypt, zio->io_ckey, |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1532 |
bookmark, zio->io_txg, isdnode, |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1533 |
zio->io_data, zio->io_size, |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1534 |
&cbuf, &cbufsize, (void **)&mac); |
13361
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1535 |
zio_crypt_key_release(zio->io_ckey, FTAG); |
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1536 |
} |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1537 |
|
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1538 |
/* |
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1539 |
* If we don't have access to the key material that the |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1540 |
* zboookmark_t says we needed, post an FMA event. |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1541 |
*/ |
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1542 |
if (crypt_error != 0) { |
13347
90a127dff2c5
6719749 dnode dn_bonus needs to be encrypted
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13338
diff
changeset
|
1543 |
zio->io_error = crypt_error; |
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1544 |
if (crypt_error == EAGAIN) { |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1545 |
zfs_ereport_post(FM_EREPORT_ZFS_CRYPTO_KEY_UNAVAIL, |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1546 |
zio->io_spa, zio->io_vd, zio, 0, 0); |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1547 |
} |
13361
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1548 |
} else { |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1549 |
|
13361
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1550 |
/* |
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1551 |
* The mac is stored in the blkptr |
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1552 |
* as the top two words of the checksum. |
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1553 |
*/ |
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1554 |
maclen = zio_crypt_table[crypt].ci_maclen; |
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1555 |
ASSERT3U(maclen, ==, 16); |
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1556 |
ASSERT3U((uintptr_t)mac, !=, NULL); |
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1557 |
bp->blk_cksum.zc_word[2] = BE_64(mac->zc_word[0]); |
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1558 |
bp->blk_cksum.zc_word[3] = BE_64(mac->zc_word[1]); |
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1559 |
kmem_free(mac, maclen); |
13186
a6948d45b93f
AES_CCM bug fixes
Darren Moffat <darrenm@opensolaris.org>
parents:
13177
diff
changeset
|
1560 |
|
13361
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1561 |
zio_push_transform(zio, cbuf, cbufsize, cbufsize); |
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1562 |
BP_SET_CRYPT(zio->io_bp, crypt); |
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1563 |
} |
13160
e8b02045519a
295 mac+checksum in zio_checksum_t (initial draft)
Darren Moffat <darrenm@opensolaris.org>
parents:
13158
diff
changeset
|
1564 |
|
13152 | 1565 |
return (ZIO_PIPELINE_CONTINUE); |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1566 |
} |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1567 |
|
13152 | 1568 |
static int |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1569 |
zio_read_decrypt(zio_t *zio) |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1570 |
{ |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1571 |
blkptr_t *bp = zio->io_bp; |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1572 |
int crypt = BP_GET_CRYPT(bp); |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1573 |
void *data; |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1574 |
uint64_t size; |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1575 |
uint64_t bufsize; |
13307
3d90d41b6103
1551 data decryption cross architecture fails in AES_CCM
Darren Moffat <darrenm@opensolaris.org>
parents:
13301
diff
changeset
|
1576 |
uint64_t mac[2]; |
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1577 |
int crypt_error = 0; |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1578 |
boolean_t isdnode = B_FALSE; |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1579 |
|
13251
f037017b1f9b
1701 zio_write_encrypt ASSERT trips on old spa version
Darren Moffat <darrenm@opensolaris.org>
parents:
13230
diff
changeset
|
1580 |
if (spa_version(zio->io_spa) < SPA_VERSION_CRYPTO) |
f037017b1f9b
1701 zio_write_encrypt ASSERT trips on old spa version
Darren Moffat <darrenm@opensolaris.org>
parents:
13230
diff
changeset
|
1581 |
return (ZIO_PIPELINE_CONTINUE); |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1582 |
|
13326
00ca5372f202
6719684 zfs destroy on clear text can cause panic
Darren Moffat <darrenm@opensolaris.org>
parents:
13323
diff
changeset
|
1583 |
ASSERT3U(crypt, !=, ZIO_CRYPT_OFF); |
00ca5372f202
6719684 zfs destroy on clear text can cause panic
Darren Moffat <darrenm@opensolaris.org>
parents:
13323
diff
changeset
|
1584 |
ASSERT3U(crypt, !=, ZIO_CRYPT_INHERIT); |
13368
2661054b6aaa
6738244 zpool scrub shows errors when key present
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13361
diff
changeset
|
1585 |
ASSERT3U(BP_GET_CHECKSUM(bp), ==, ZIO_CHECKSUM_SHA256_CCM_MAC); |
2661054b6aaa
6738244 zpool scrub shows errors when key present
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13361
diff
changeset
|
1586 |
ASSERT3U(BP_GET_LEVEL(bp), ==, 0); |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1587 |
|
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1588 |
zio_pop_transform(zio, &data, &size, &bufsize); |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1589 |
|
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1590 |
/* MAC is stored in the blkptr as the top two words of the checksum */ |
13308
b11402c5f64b
2207 scrub/resilver says pools have errors when key not present
Darren Moffat <darrenm@opensolaris.org>
parents:
13307
diff
changeset
|
1591 |
ASSERT3U(16, ==, zio_crypt_table[crypt].ci_maclen); |
13307
3d90d41b6103
1551 data decryption cross architecture fails in AES_CCM
Darren Moffat <darrenm@opensolaris.org>
parents:
13301
diff
changeset
|
1592 |
mac[0] = BE_64(bp->blk_cksum.zc_word[2]); |
3d90d41b6103
1551 data decryption cross architecture fails in AES_CCM
Darren Moffat <darrenm@opensolaris.org>
parents:
13301
diff
changeset
|
1593 |
mac[1] = BE_64(bp->blk_cksum.zc_word[3]); |
13160
e8b02045519a
295 mac+checksum in zio_checksum_t (initial draft)
Darren Moffat <darrenm@opensolaris.org>
parents:
13158
diff
changeset
|
1594 |
|
13347
90a127dff2c5
6719749 dnode dn_bonus needs to be encrypted
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13338
diff
changeset
|
1595 |
isdnode = (BP_GET_TYPE(bp) == DMU_OT_DNODE) && (BP_GET_LEVEL(bp) == 0); |
90a127dff2c5
6719749 dnode dn_bonus needs to be encrypted
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13338
diff
changeset
|
1596 |
|
13361
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1597 |
/* We may have got the key earlier but lets try again */ |
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1598 |
if (zio->io_ckey == NULL) { |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1599 |
zio->io_ckey = zio_crypt_key_lookup(zio->io_spa, |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1600 |
zio->io_bookmark.zb_objset, crypt); |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1601 |
if (zio->io_ckey == NULL) { |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1602 |
crypt_error = EAGAIN; |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1603 |
} else { |
13361
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1604 |
zio_crypt_key_hold(zio->io_ckey, FTAG); |
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1605 |
} |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1606 |
} |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1607 |
|
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1608 |
if (zio->io_ckey != NULL) { |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1609 |
crypt_error = zio_decrypt_data(crypt, zio->io_ckey, |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1610 |
&zio->io_bookmark, bp->blk_birth, isdnode, |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1611 |
data, size, &mac, zio->io_data, zio->io_size); |
13361
c95dbe71b9f7
6742059 zfs-crypto, backup_encrypt_001 TIMED_OUT and cause following zfs thread
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13355
diff
changeset
|
1612 |
zio_crypt_key_release(zio->io_ckey, FTAG); |
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1613 |
} |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1614 |
|
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1615 |
/* |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1616 |
* One possible failure is not having access to the |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1617 |
* key material that the zboookmark_t says we needed, |
13255
d3a0d9a40926
1444 keywrap algorithm should include integrity check
Darren Moffat <darrenm@opensolaris.org>
parents:
13251
diff
changeset
|
1618 |
* In that case we got ENOENT so post an FMA event, we do this |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1619 |
* here so that we don't need to pass the full zio downwards into |
13255
d3a0d9a40926
1444 keywrap algorithm should include integrity check
Darren Moffat <darrenm@opensolaris.org>
parents:
13251
diff
changeset
|
1620 |
* the crypto functions. Then change the error to EAGAIN so the |
d3a0d9a40926
1444 keywrap algorithm should include integrity check
Darren Moffat <darrenm@opensolaris.org>
parents:
13251
diff
changeset
|
1621 |
* read can be retried. |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1622 |
*/ |
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1623 |
if (crypt_error != 0) { |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1624 |
if (crypt_error == EAGAIN) { |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1625 |
zfs_ereport_post(FM_EREPORT_ZFS_CRYPTO_KEY_UNAVAIL, |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1626 |
zio->io_spa, zio->io_vd, zio, 0, 0); |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1627 |
/* |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1628 |
* If it is a dnode we were looking for we need the |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1629 |
* clear text parts to be passed on. |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1630 |
*/ |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1631 |
if (isdnode) { |
13368
2661054b6aaa
6738244 zpool scrub shows errors when key present
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13361
diff
changeset
|
1632 |
printf("zio_read_decrypt: isdnode crypt_error = %d", crypt_error); |
13355
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1633 |
bcopy(data, zio->io_data, size); |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1634 |
crypt_error = 0; |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1635 |
} |
03a8bd0e481a
pass keys to zio_encrypt_date/zio_decrypt_data not the spa
Darren Moffat <Darren.Moffat@Sun.COM>
parents:
13347
diff
changeset
|
1636 |
} |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1637 |
zio->io_error = crypt_error; |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1638 |
} |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1639 |
|
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1640 |
zio_buf_free(data, bufsize); |
13152 | 1641 |
return (ZIO_PIPELINE_CONTINUE); |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1642 |
} |
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
1643 |
|
789 | 1644 |
/* |
1645 |
* ========================================================================== |
|
1646 |
* Gang block support |
|
1647 |
* ========================================================================== |
|
1648 |
*/ |
|
1649 |
static void |
|
1650 |
zio_gang_byteswap(zio_t *zio) |
|
1651 |
{ |
|
1652 |
ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); |
|
1653 |
||
1654 |
if (BP_SHOULD_BYTESWAP(zio->io_bp)) |
|
1655 |
byteswap_uint64_array(zio->io_data, zio->io_size); |
|
1656 |
} |
|
1657 |
||
5530 | 1658 |
static int |
789 | 1659 |
zio_get_gang_header(zio_t *zio) |
1660 |
{ |
|
1661 |
blkptr_t *bp = zio->io_bp; |
|
1662 |
uint64_t gsize = SPA_GANGBLOCKSIZE; |
|
1663 |
void *gbuf = zio_buf_alloc(gsize); |
|
1664 |
||
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1665 |
ASSERT(BP_IS_GANG(bp)); |
789 | 1666 |
|
1667 |
zio_push_transform(zio, gbuf, gsize, gsize); |
|
1668 |
||
1669 |
zio_nowait(zio_create(zio, zio->io_spa, bp->blk_birth, bp, gbuf, gsize, |
|
1670 |
NULL, NULL, ZIO_TYPE_READ, zio->io_priority, |
|
1671 |
zio->io_flags & ZIO_FLAG_GANG_INHERIT, |
|
5329 | 1672 |
ZIO_STAGE_OPEN, ZIO_READ_GANG_PIPELINE)); |
789 | 1673 |
|
5530 | 1674 |
return (zio_wait_for_children_done(zio)); |
789 | 1675 |
} |
1676 |
||
5530 | 1677 |
static int |
789 | 1678 |
zio_read_gang_members(zio_t *zio) |
1679 |
{ |
|
1680 |
zio_gbh_phys_t *gbh; |
|
1681 |
uint64_t gsize, gbufsize, loff, lsize; |
|
1682 |
int i; |
|
1683 |
||
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1684 |
ASSERT(BP_IS_GANG(zio->io_bp)); |
789 | 1685 |
|
1686 |
zio_gang_byteswap(zio); |
|
1687 |
zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); |
|
1688 |
||
1689 |
for (loff = 0, i = 0; loff != zio->io_size; loff += lsize, i++) { |
|
1690 |
blkptr_t *gbp = &gbh->zg_blkptr[i]; |
|
1691 |
lsize = BP_GET_PSIZE(gbp); |
|
1692 |
||
1693 |
ASSERT(BP_GET_COMPRESS(gbp) == ZIO_COMPRESS_OFF); |
|
1694 |
ASSERT3U(lsize, ==, BP_GET_LSIZE(gbp)); |
|
1695 |
ASSERT3U(loff + lsize, <=, zio->io_size); |
|
1696 |
ASSERT(i < SPA_GBH_NBLKPTRS); |
|
1697 |
ASSERT(!BP_IS_HOLE(gbp)); |
|
1698 |
||
1699 |
zio_nowait(zio_read(zio, zio->io_spa, gbp, |
|
5530 | 1700 |
(char *)zio->io_data + loff, lsize, |
1701 |
NULL, NULL, zio->io_priority, |
|
1702 |
zio->io_flags & ZIO_FLAG_GANG_INHERIT, &zio->io_bookmark)); |
|
789 | 1703 |
} |
1704 |
||
1705 |
zio_buf_free(gbh, gbufsize); |
|
5530 | 1706 |
|
1707 |
return (zio_wait_for_children_done(zio)); |
|
789 | 1708 |
} |
1709 |
||
5530 | 1710 |
static int |
789 | 1711 |
zio_rewrite_gang_members(zio_t *zio) |
1712 |
{ |
|
1713 |
zio_gbh_phys_t *gbh; |
|
1714 |
uint64_t gsize, gbufsize, loff, lsize; |
|
1715 |
int i; |
|
1716 |
||
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1717 |
ASSERT(BP_IS_GANG(zio->io_bp)); |
789 | 1718 |
ASSERT3U(zio->io_size, ==, SPA_GANGBLOCKSIZE); |
1719 |
||
1720 |
zio_gang_byteswap(zio); |
|
1721 |
zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); |
|
1722 |
||
1723 |
ASSERT(gsize == gbufsize); |
|
1724 |
||
1725 |
for (loff = 0, i = 0; loff != zio->io_size; loff += lsize, i++) { |
|
1726 |
blkptr_t *gbp = &gbh->zg_blkptr[i]; |
|
1727 |
lsize = BP_GET_PSIZE(gbp); |
|
1728 |
||
1729 |
ASSERT(BP_GET_COMPRESS(gbp) == ZIO_COMPRESS_OFF); |
|
1730 |
ASSERT3U(lsize, ==, BP_GET_LSIZE(gbp)); |
|
1731 |
ASSERT3U(loff + lsize, <=, zio->io_size); |
|
1732 |
ASSERT(i < SPA_GBH_NBLKPTRS); |
|
1733 |
ASSERT(!BP_IS_HOLE(gbp)); |
|
1734 |
||
13320 | 1735 |
zio_nowait(zio_rewrite(zio, zio->io_spa, zio->io_checksum, |
13338 | 1736 |
ZIO_CRYPT_OFF, zio->io_txg, gbp, (char *)zio->io_data + loff, lsize, |
7181
8d299641aa23
6719934 assertion failed: ((&zh->zh_log)->blk_birth == 0), file: ../../common/fs/zfs/zil.c, line: 1336
perrin
parents:
7046
diff
changeset
|
1737 |
NULL, NULL, zio->io_priority, |
8d299641aa23
6719934 assertion failed: ((&zh->zh_log)->blk_birth == 0), file: ../../common/fs/zfs/zil.c, line: 1336
perrin
parents:
7046
diff
changeset
|
1738 |
zio->io_flags & ZIO_FLAG_GANG_INHERIT, &zio->io_bookmark)); |
789 | 1739 |
} |
1740 |
||
1741 |
zio_push_transform(zio, gbh, gsize, gbufsize); |
|
5530 | 1742 |
|
1743 |
return (zio_wait_for_children_ready(zio)); |
|
789 | 1744 |
} |
1745 |
||
5530 | 1746 |
static int |
789 | 1747 |
zio_free_gang_members(zio_t *zio) |
1748 |
{ |
|
1749 |
zio_gbh_phys_t *gbh; |
|
1750 |
uint64_t gsize, gbufsize; |
|
1751 |
int i; |
|
1752 |
||
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1753 |
ASSERT(BP_IS_GANG(zio->io_bp)); |
789 | 1754 |
|
1755 |
zio_gang_byteswap(zio); |
|
1756 |
zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); |
|
1757 |
||
1758 |
for (i = 0; i < SPA_GBH_NBLKPTRS; i++) { |
|
1759 |
blkptr_t *gbp = &gbh->zg_blkptr[i]; |
|
1760 |
||
1761 |
if (BP_IS_HOLE(gbp)) |
|
1762 |
continue; |
|
1763 |
zio_nowait(zio_free(zio, zio->io_spa, zio->io_txg, |
|
1764 |
gbp, NULL, NULL)); |
|
1765 |
} |
|
1766 |
||
1767 |
zio_buf_free(gbh, gbufsize); |
|
5530 | 1768 |
|
1769 |
return (ZIO_PIPELINE_CONTINUE); |
|
789 | 1770 |
} |
1771 |
||
5530 | 1772 |
static int |
789 | 1773 |
zio_claim_gang_members(zio_t *zio) |
1774 |
{ |
|
1775 |
zio_gbh_phys_t *gbh; |
|
1776 |
uint64_t gsize, gbufsize; |
|
1777 |
int i; |
|
1778 |
||
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1779 |
ASSERT(BP_IS_GANG(zio->io_bp)); |
789 | 1780 |
|
1781 |
zio_gang_byteswap(zio); |
|
1782 |
zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); |
|
1783 |
||
1784 |
for (i = 0; i < SPA_GBH_NBLKPTRS; i++) { |
|
1785 |
blkptr_t *gbp = &gbh->zg_blkptr[i]; |
|
1786 |
if (BP_IS_HOLE(gbp)) |
|
1787 |
continue; |
|
1788 |
zio_nowait(zio_claim(zio, zio->io_spa, zio->io_txg, |
|
1789 |
gbp, NULL, NULL)); |
|
1790 |
} |
|
1791 |
||
1792 |
zio_buf_free(gbh, gbufsize); |
|
5530 | 1793 |
|
1794 |
return (ZIO_PIPELINE_CONTINUE); |
|
789 | 1795 |
} |
1796 |
||
1797 |
static void |
|
1798 |
zio_write_allocate_gang_member_done(zio_t *zio) |
|
1799 |
{ |
|
1800 |
zio_t *pio = zio->io_parent; |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1801 |
dva_t *cdva = zio->io_bp->blk_dva; |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1802 |
dva_t *pdva = pio->io_bp->blk_dva; |
789 | 1803 |
uint64_t asize; |
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1804 |
int d; |
789 | 1805 |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1806 |
ASSERT3U(pio->io_ndvas, ==, zio->io_ndvas); |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1807 |
ASSERT3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp)); |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1808 |
ASSERT3U(zio->io_ndvas, <=, BP_GET_NDVAS(zio->io_bp)); |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1809 |
ASSERT3U(pio->io_ndvas, <=, BP_GET_NDVAS(pio->io_bp)); |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1810 |
|
789 | 1811 |
mutex_enter(&pio->io_lock); |
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1812 |
for (d = 0; d < BP_GET_NDVAS(pio->io_bp); d++) { |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1813 |
ASSERT(DVA_GET_GANG(&pdva[d])); |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1814 |
asize = DVA_GET_ASIZE(&pdva[d]); |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1815 |
asize += DVA_GET_ASIZE(&cdva[d]); |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1816 |
DVA_SET_ASIZE(&pdva[d], asize); |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1817 |
} |
789 | 1818 |
mutex_exit(&pio->io_lock); |
1819 |
} |
|
1820 |
||
5329 | 1821 |
static int |
4527 | 1822 |
zio_write_allocate_gang_members(zio_t *zio, metaslab_class_t *mc) |
789 | 1823 |
{ |
1824 |
blkptr_t *bp = zio->io_bp; |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1825 |
dva_t *dva = bp->blk_dva; |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1826 |
spa_t *spa = zio->io_spa; |
789 | 1827 |
zio_gbh_phys_t *gbh; |
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1828 |
uint64_t txg = zio->io_txg; |
789 | 1829 |
uint64_t resid = zio->io_size; |
1830 |
uint64_t maxalloc = P2ROUNDUP(zio->io_size >> 1, SPA_MINBLOCKSIZE); |
|
1831 |
uint64_t gsize, loff, lsize; |
|
1832 |
uint32_t gbps_left; |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1833 |
int ndvas = zio->io_ndvas; |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1834 |
int gbh_ndvas = MIN(ndvas + 1, spa_max_replication(spa)); |
789 | 1835 |
int error; |
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1836 |
int i, d; |
789 | 1837 |
|
1838 |
gsize = SPA_GANGBLOCKSIZE; |
|
1839 |
gbps_left = SPA_GBH_NBLKPTRS; |
|
1840 |
||
4527 | 1841 |
error = metaslab_alloc(spa, mc, gsize, bp, gbh_ndvas, txg, NULL, |
1842 |
B_FALSE); |
|
5530 | 1843 |
if (error) { |
1844 |
zio->io_error = error; |
|
1845 |
return (ZIO_PIPELINE_CONTINUE); |
|
1846 |
} |
|
789 | 1847 |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1848 |
for (d = 0; d < gbh_ndvas; d++) |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1849 |
DVA_SET_GANG(&dva[d], 1); |
789 | 1850 |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1851 |
bp->blk_birth = txg; |
789 | 1852 |
|
1853 |
gbh = zio_buf_alloc(gsize); |
|
1854 |
bzero(gbh, gsize); |
|
1855 |
||
1856 |
for (loff = 0, i = 0; loff != zio->io_size; |
|
1857 |
loff += lsize, resid -= lsize, gbps_left--, i++) { |
|
1858 |
blkptr_t *gbp = &gbh->zg_blkptr[i]; |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1859 |
dva = gbp->blk_dva; |
789 | 1860 |
|
1861 |
ASSERT(gbps_left != 0); |
|
1862 |
maxalloc = MIN(maxalloc, resid); |
|
1863 |
||
1864 |
while (resid <= maxalloc * gbps_left) { |
|
4527 | 1865 |
error = metaslab_alloc(spa, mc, maxalloc, gbp, ndvas, |
3063
b252896b372b
6341569 zio_alloc_blk() vdev distribution performs badly
perrin
parents:
2981
diff
changeset
|
1866 |
txg, bp, B_FALSE); |
789 | 1867 |
if (error == 0) |
1868 |
break; |
|
1869 |
ASSERT3U(error, ==, ENOSPC); |
|
5329 | 1870 |
/* XXX - free up previous allocations? */ |
5530 | 1871 |
if (maxalloc == SPA_MINBLOCKSIZE) { |
1872 |
zio->io_error = error; |
|
1873 |
return (ZIO_PIPELINE_CONTINUE); |
|
1874 |
} |
|
789 | 1875 |
maxalloc = P2ROUNDUP(maxalloc >> 1, SPA_MINBLOCKSIZE); |
1876 |
} |
|
1877 |
||
1878 |
if (resid <= maxalloc * gbps_left) { |
|
1879 |
lsize = maxalloc; |
|
1880 |
BP_SET_LSIZE(gbp, lsize); |
|
1881 |
BP_SET_PSIZE(gbp, lsize); |
|
1882 |
BP_SET_COMPRESS(gbp, ZIO_COMPRESS_OFF); |
|
13326
00ca5372f202
6719684 zfs destroy on clear text can cause panic
Darren Moffat <darrenm@opensolaris.org>
parents:
13323
diff
changeset
|
1883 |
BP_SET_CRYPT(gbp, ZIO_CRYPT_OFF); |
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1884 |
gbp->blk_birth = txg; |
13338 | 1885 |
zio_nowait(zio_rewrite(zio, spa, zio->io_checksum, |
1886 |
ZIO_CRYPT_OFF, txg, gbp, |
|
789 | 1887 |
(char *)zio->io_data + loff, lsize, |
1888 |
zio_write_allocate_gang_member_done, NULL, |
|
5403 | 1889 |
zio->io_priority, |
1890 |
zio->io_flags & ZIO_FLAG_GANG_INHERIT, |
|
1544 | 1891 |
&zio->io_bookmark)); |
789 | 1892 |
} else { |
1893 |
lsize = P2ROUNDUP(resid / gbps_left, SPA_MINBLOCKSIZE); |
|
1894 |
ASSERT(lsize != SPA_MINBLOCKSIZE); |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1895 |
zio_nowait(zio_write_allocate(zio, spa, |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1896 |
zio->io_checksum, txg, gbp, |
789 | 1897 |
(char *)zio->io_data + loff, lsize, |
1898 |
zio_write_allocate_gang_member_done, NULL, |
|
5403 | 1899 |
zio->io_priority, |
1900 |
zio->io_flags & ZIO_FLAG_GANG_INHERIT)); |
|
789 | 1901 |
} |
1902 |
} |
|
1903 |
||
1904 |
ASSERT(resid == 0 && loff == zio->io_size); |
|
1905 |
||
1906 |
zio->io_pipeline |= 1U << ZIO_STAGE_GANG_CHECKSUM_GENERATE; |
|
1907 |
||
1908 |
zio_push_transform(zio, gbh, gsize, gsize); |
|
5530 | 1909 |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1910 |
/* |
5530 | 1911 |
* As much as we'd like this to be 'ready' instead of 'done', |
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1912 |
* updating our ASIZE doesn't happen until the io_done callback, |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1913 |
* so we have to wait for that to finish in order for our BP |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1914 |
* to be stable. |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1915 |
*/ |
5530 | 1916 |
return (zio_wait_for_children_done(zio)); |
789 | 1917 |
} |
1918 |
||
1919 |
/* |
|
1920 |
* ========================================================================== |
|
1921 |
* Allocate and free blocks |
|
1922 |
* ========================================================================== |
|
1923 |
*/ |
|
5530 | 1924 |
static int |
789 | 1925 |
zio_dva_allocate(zio_t *zio) |
1926 |
{ |
|
4527 | 1927 |
spa_t *spa = zio->io_spa; |
1928 |
metaslab_class_t *mc = spa->spa_normal_class; |
|
789 | 1929 |
blkptr_t *bp = zio->io_bp; |
1930 |
int error; |
|
1931 |
||
1932 |
ASSERT(BP_IS_HOLE(bp)); |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1933 |
ASSERT3U(BP_GET_NDVAS(bp), ==, 0); |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1934 |
ASSERT3U(zio->io_ndvas, >, 0); |
4527 | 1935 |
ASSERT3U(zio->io_ndvas, <=, spa_max_replication(spa)); |
789 | 1936 |
|
5329 | 1937 |
/* |
1938 |
* For testing purposes, we force I/Os to retry. We don't allow |
|
1939 |
* retries beyond the first pass since those I/Os are non-allocating |
|
5403 | 1940 |
* writes. |
5329 | 1941 |
*/ |
1942 |
if (zio_io_fail_shift && |
|
1943 |
spa_sync_pass(zio->io_spa) <= zio_sync_pass.zp_rewrite && |
|
1944 |
zio_io_should_fail(zio_io_fail_shift)) |
|
1945 |
zio->io_flags |= ZIO_FLAG_WRITE_RETRY; |
|
1946 |
||
789 | 1947 |
ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); |
1948 |
||
4527 | 1949 |
error = metaslab_alloc(spa, mc, zio->io_size, bp, zio->io_ndvas, |
3063
b252896b372b
6341569 zio_alloc_blk() vdev distribution performs badly
perrin
parents:
2981
diff
changeset
|
1950 |
zio->io_txg, NULL, B_FALSE); |
789 | 1951 |
|
1952 |
if (error == 0) { |
|
1953 |
bp->blk_birth = zio->io_txg; |
|
5329 | 1954 |
} else if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE) { |
5530 | 1955 |
return (zio_write_allocate_gang_members(zio, mc)); |
789 | 1956 |
} else { |
1957 |
zio->io_error = error; |
|
1958 |
} |
|
5530 | 1959 |
|
1960 |
return (ZIO_PIPELINE_CONTINUE); |
|
789 | 1961 |
} |
1962 |
||
5530 | 1963 |
static int |
789 | 1964 |
zio_dva_free(zio_t *zio) |
1965 |
{ |
|
1966 |
blkptr_t *bp = zio->io_bp; |
|
1967 |
||
1807
35c8b566d7af
6410711 intent log blocks don't get invited to pool parties
bonwick
parents:
1775
diff
changeset
|
1968 |
metaslab_free(zio->io_spa, bp, zio->io_txg, B_FALSE); |
789 | 1969 |
|
1970 |
BP_ZERO(bp); |
|
1971 |
||
5530 | 1972 |
return (ZIO_PIPELINE_CONTINUE); |
789 | 1973 |
} |
1974 |
||
5530 | 1975 |
static int |
789 | 1976 |
zio_dva_claim(zio_t *zio) |
1977 |
{ |
|
1807
35c8b566d7af
6410711 intent log blocks don't get invited to pool parties
bonwick
parents:
1775
diff
changeset
|
1978 |
zio->io_error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg); |
789 | 1979 |
|
5530 | 1980 |
return (ZIO_PIPELINE_CONTINUE); |
789 | 1981 |
} |
1982 |
||
1983 |
/* |
|
1984 |
* ========================================================================== |
|
1985 |
* Read and write to physical devices |
|
1986 |
* ========================================================================== |
|
1987 |
*/ |
|
1988 |
||
5530 | 1989 |
static int |
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1990 |
zio_vdev_io_start(zio_t *zio) |
789 | 1991 |
{ |
1992 |
vdev_t *vd = zio->io_vd; |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1993 |
vdev_t *tvd = vd ? vd->vdev_top : NULL; |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1994 |
blkptr_t *bp = zio->io_bp; |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
1995 |
uint64_t align; |
5329 | 1996 |
spa_t *spa = zio->io_spa; |
1997 |
||
1998 |
/* |
|
1999 |
* If the pool is already in a failure state then just suspend |
|
2000 |
* this IO until the problem is resolved. We will reissue them |
|
2001 |
* at that time. |
|
2002 |
*/ |
|
2003 |
if (spa_state(spa) == POOL_STATE_IO_FAILURE && |
|
5530 | 2004 |
zio->io_type == ZIO_TYPE_WRITE) |
2005 |
return (zio_vdev_suspend_io(zio)); |
|
789 | 2006 |
|
5530 | 2007 |
/* |
2008 |
* The mirror_ops handle multiple DVAs in a single BP |
|
2009 |
*/ |
|
2010 |
if (vd == NULL) |
|
2011 |
return (vdev_mirror_ops.vdev_op_io_start(zio)); |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
2012 |
|
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
2013 |
align = 1ULL << tvd->vdev_ashift; |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
2014 |
|
1732 | 2015 |
if (zio->io_retries == 0 && vd == tvd) |
789 | 2016 |
zio->io_flags |= ZIO_FLAG_FAILFAST; |
2017 |
||
5530 | 2018 |
if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) && vd->vdev_children == 0) { |
789 | 2019 |
zio->io_flags |= ZIO_FLAG_PHYSICAL; |
2020 |
zio->io_offset += VDEV_LABEL_START_SIZE; |
|
2021 |
} |
|
2022 |
||
1732 | 2023 |
if (P2PHASE(zio->io_size, align) != 0) { |
2024 |
uint64_t asize = P2ROUNDUP(zio->io_size, align); |
|
2025 |
char *abuf = zio_buf_alloc(asize); |
|
2026 |
ASSERT(vd == tvd); |
|
2027 |
if (zio->io_type == ZIO_TYPE_WRITE) { |
|
2028 |
bcopy(zio->io_data, abuf, zio->io_size); |
|
2029 |
bzero(abuf + zio->io_size, asize - zio->io_size); |
|
2030 |
} |
|
2031 |
zio_push_transform(zio, abuf, asize, asize); |
|
2032 |
ASSERT(!(zio->io_flags & ZIO_FLAG_SUBBLOCK)); |
|
2033 |
zio->io_flags |= ZIO_FLAG_SUBBLOCK; |
|
2034 |
} |
|
2035 |
||
2036 |
ASSERT(P2PHASE(zio->io_offset, align) == 0); |
|
2037 |
ASSERT(P2PHASE(zio->io_size, align) == 0); |
|
2038 |
ASSERT(bp == NULL || |
|
2039 |
P2ROUNDUP(ZIO_GET_IOSIZE(zio), align) == zio->io_size); |
|
789 | 2040 |
ASSERT(zio->io_type != ZIO_TYPE_WRITE || (spa_mode & FWRITE)); |
2041 |
||
5530 | 2042 |
return (vd->vdev_ops->vdev_op_io_start(zio)); |
789 | 2043 |
} |
2044 |
||
5530 | 2045 |
static int |
789 | 2046 |
zio_vdev_io_done(zio_t *zio) |
2047 |
{ |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
2048 |
if (zio->io_vd == NULL) |
5530 | 2049 |
return (vdev_mirror_ops.vdev_op_io_done(zio)); |
2050 |
||
2051 |
return (zio->io_vd->vdev_ops->vdev_op_io_done(zio)); |
|
789 | 2052 |
} |
2053 |
||
2054 |
/* XXPOLICY */ |
|
1544 | 2055 |
boolean_t |
789 | 2056 |
zio_should_retry(zio_t *zio) |
2057 |
{ |
|
2058 |
vdev_t *vd = zio->io_vd; |
|
2059 |
||
2060 |
if (zio->io_error == 0) |
|
2061 |
return (B_FALSE); |
|
2062 |
if (zio->io_delegate_list != NULL) |
|
2063 |
return (B_FALSE); |
|
6976
cae5f06df471
PSARC 2008/388 Short circuit for vdev probe failure
eschrock
parents:
6523
diff
changeset
|
2064 |
if (vd != NULL) { |
cae5f06df471
PSARC 2008/388 Short circuit for vdev probe failure
eschrock
parents:
6523
diff
changeset
|
2065 |
if (vd != vd->vdev_top) |
cae5f06df471
PSARC 2008/388 Short circuit for vdev probe failure
eschrock
parents:
6523
diff
changeset
|
2066 |
return (B_FALSE); |
cae5f06df471
PSARC 2008/388 Short circuit for vdev probe failure
eschrock
parents:
6523
diff
changeset
|
2067 |
if (vd->vdev_is_failing) |
cae5f06df471
PSARC 2008/388 Short circuit for vdev probe failure
eschrock
parents:
6523
diff
changeset
|
2068 |
return (B_FALSE); |
cae5f06df471
PSARC 2008/388 Short circuit for vdev probe failure
eschrock
parents:
6523
diff
changeset
|
2069 |
} |
789 | 2070 |
if (zio->io_flags & ZIO_FLAG_DONT_RETRY) |
2071 |
return (B_FALSE); |
|
1544 | 2072 |
if (zio->io_retries > 0) |
789 | 2073 |
return (B_FALSE); |
2074 |
||
2075 |
return (B_TRUE); |
|
2076 |
} |
|
2077 |
||
5530 | 2078 |
static int |
789 | 2079 |
zio_vdev_io_assess(zio_t *zio) |
2080 |
{ |
|
2081 |
vdev_t *vd = zio->io_vd; |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
2082 |
vdev_t *tvd = vd ? vd->vdev_top : NULL; |
789 | 2083 |
|
1544 | 2084 |
ASSERT(zio->io_vsd == NULL); |
789 | 2085 |
|
1732 | 2086 |
if (zio->io_flags & ZIO_FLAG_SUBBLOCK) { |
2087 |
void *abuf; |
|
2088 |
uint64_t asize; |
|
2089 |
ASSERT(vd == tvd); |
|
2090 |
zio_pop_transform(zio, &abuf, &asize, &asize); |
|
2091 |
if (zio->io_type == ZIO_TYPE_READ) |
|
2092 |
bcopy(abuf, zio->io_data, zio->io_size); |
|
2093 |
zio_buf_free(abuf, asize); |
|
2094 |
zio->io_flags &= ~ZIO_FLAG_SUBBLOCK; |
|
2095 |
} |
|
2096 |
||
1544 | 2097 |
if (zio_injection_enabled && !zio->io_error) |
2098 |
zio->io_error = zio_handle_fault_injection(zio, EIO); |
|
789 | 2099 |
|
2100 |
/* |
|
2101 |
* If the I/O failed, determine whether we should attempt to retry it. |
|
2102 |
*/ |
|
2103 |
/* XXPOLICY */ |
|
2104 |
if (zio_should_retry(zio)) { |
|
2105 |
ASSERT(tvd == vd); |
|
2106 |
||
2107 |
zio->io_retries++; |
|
2108 |
zio->io_error = 0; |
|
5688
c0b02c8fd2c0
6640580 spa_get_random() is insanely slow in userland
bonwick
parents:
5530
diff
changeset
|
2109 |
zio->io_flags &= ZIO_FLAG_RETRY_INHERIT; |
789 | 2110 |
/* XXPOLICY */ |
2111 |
zio->io_flags &= ~ZIO_FLAG_FAILFAST; |
|
2112 |
zio->io_flags |= ZIO_FLAG_DONT_CACHE; |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
2113 |
zio->io_stage = ZIO_STAGE_VDEV_IO_START - 1; |
789 | 2114 |
|
5530 | 2115 |
return (ZIO_PIPELINE_CONTINUE); |
1544 | 2116 |
} |
789 | 2117 |
|
5530 | 2118 |
return (ZIO_PIPELINE_CONTINUE); |
789 | 2119 |
} |
2120 |
||
2121 |
void |
|
2122 |
zio_vdev_io_reissue(zio_t *zio) |
|
2123 |
{ |
|
2124 |
ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); |
|
2125 |
ASSERT(zio->io_error == 0); |
|
2126 |
||
2127 |
zio->io_stage--; |
|
2128 |
} |
|
2129 |
||
2130 |
void |
|
2131 |
zio_vdev_io_redone(zio_t *zio) |
|
2132 |
{ |
|
2133 |
ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); |
|
2134 |
||
2135 |
zio->io_stage--; |
|
2136 |
} |
|
2137 |
||
2138 |
void |
|
2139 |
zio_vdev_io_bypass(zio_t *zio) |
|
2140 |
{ |
|
2141 |
ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); |
|
2142 |
ASSERT(zio->io_error == 0); |
|
2143 |
||
2144 |
zio->io_flags |= ZIO_FLAG_IO_BYPASS; |
|
2145 |
zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS - 1; |
|
2146 |
} |
|
2147 |
||
2148 |
/* |
|
2149 |
* ========================================================================== |
|
2150 |
* Generate and verify checksums |
|
2151 |
* ========================================================================== |
|
2152 |
*/ |
|
5530 | 2153 |
static int |
789 | 2154 |
zio_checksum_generate(zio_t *zio) |
2155 |
{ |
|
2156 |
int checksum = zio->io_checksum; |
|
2157 |
blkptr_t *bp = zio->io_bp; |
|
2158 |
||
2159 |
ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); |
|
2160 |
||
2161 |
BP_SET_CHECKSUM(bp, checksum); |
|
2162 |
BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); |
|
2163 |
||
2164 |
zio_checksum(checksum, &bp->blk_cksum, zio->io_data, zio->io_size); |
|
2165 |
||
5530 | 2166 |
return (ZIO_PIPELINE_CONTINUE); |
789 | 2167 |
} |
2168 |
||
5530 | 2169 |
static int |
789 | 2170 |
zio_gang_checksum_generate(zio_t *zio) |
2171 |
{ |
|
2172 |
zio_cksum_t zc; |
|
2173 |
zio_gbh_phys_t *gbh = zio->io_data; |
|
2174 |
||
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
2175 |
ASSERT(BP_IS_GANG(zio->io_bp)); |
789 | 2176 |
ASSERT3U(zio->io_size, ==, SPA_GANGBLOCKSIZE); |
2177 |
||
2178 |
zio_set_gang_verifier(zio, &gbh->zg_tail.zbt_cksum); |
|
2179 |
||
2180 |
zio_checksum(ZIO_CHECKSUM_GANG_HEADER, &zc, zio->io_data, zio->io_size); |
|
2181 |
||
5530 | 2182 |
return (ZIO_PIPELINE_CONTINUE); |
789 | 2183 |
} |
2184 |
||
5530 | 2185 |
static int |
789 | 2186 |
zio_checksum_verify(zio_t *zio) |
2187 |
{ |
|
2188 |
if (zio->io_bp != NULL) { |
|
2189 |
zio->io_error = zio_checksum_error(zio); |
|
1544 | 2190 |
if (zio->io_error && !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) |
2191 |
zfs_ereport_post(FM_EREPORT_ZFS_CHECKSUM, |
|
2192 |
zio->io_spa, zio->io_vd, zio, 0, 0); |
|
789 | 2193 |
} |
2194 |
||
5530 | 2195 |
return (ZIO_PIPELINE_CONTINUE); |
789 | 2196 |
} |
2197 |
||
2198 |
/* |
|
2199 |
* Called by RAID-Z to ensure we don't compute the checksum twice. |
|
2200 |
*/ |
|
2201 |
void |
|
2202 |
zio_checksum_verified(zio_t *zio) |
|
2203 |
{ |
|
2204 |
zio->io_pipeline &= ~(1U << ZIO_STAGE_CHECKSUM_VERIFY); |
|
2205 |
} |
|
2206 |
||
2207 |
/* |
|
2208 |
* Set the external verifier for a gang block based on stuff in the bp |
|
2209 |
*/ |
|
2210 |
void |
|
2211 |
zio_set_gang_verifier(zio_t *zio, zio_cksum_t *zcp) |
|
2212 |
{ |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
2213 |
blkptr_t *bp = zio->io_bp; |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
2214 |
|
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
2215 |
zcp->zc_word[0] = DVA_GET_VDEV(BP_IDENTITY(bp)); |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
2216 |
zcp->zc_word[1] = DVA_GET_OFFSET(BP_IDENTITY(bp)); |
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
2217 |
zcp->zc_word[2] = bp->blk_birth; |
789 | 2218 |
zcp->zc_word[3] = 0; |
2219 |
} |
|
2220 |
||
2221 |
/* |
|
2222 |
* ========================================================================== |
|
2223 |
* Define the pipeline |
|
2224 |
* ========================================================================== |
|
2225 |
*/ |
|
5530 | 2226 |
typedef int zio_pipe_stage_t(zio_t *zio); |
789 | 2227 |
|
2228 |
zio_pipe_stage_t *zio_pipeline[ZIO_STAGE_DONE + 2] = { |
|
5530 | 2229 |
NULL, |
2230 |
zio_wait_for_children_ready, |
|
2231 |
zio_read_init, |
|
2232 |
zio_issue_async, |
|
789 | 2233 |
zio_write_compress, |
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
2234 |
zio_write_encrypt, |
789 | 2235 |
zio_checksum_generate, |
2236 |
zio_get_gang_header, |
|
2237 |
zio_rewrite_gang_members, |
|
2238 |
zio_free_gang_members, |
|
2239 |
zio_claim_gang_members, |
|
2240 |
zio_dva_allocate, |
|
2241 |
zio_dva_free, |
|
2242 |
zio_dva_claim, |
|
2243 |
zio_gang_checksum_generate, |
|
2244 |
zio_ready, |
|
2245 |
zio_vdev_io_start, |
|
2246 |
zio_vdev_io_done, |
|
2247 |
zio_vdev_io_assess, |
|
5530 | 2248 |
zio_wait_for_children_done, |
789 | 2249 |
zio_checksum_verify, |
2250 |
zio_read_gang_members, |
|
13150
4d89e6c6d8c8
Import ZFS Crypto Alpha+ bits into refesh repository due to Mecurial screwup.
Darren Moffat <darrenm@opensolaris.org>
parents:
5450
diff
changeset
|
2251 |
zio_read_decrypt, |
789 | 2252 |
zio_read_decompress, |
5329 | 2253 |
zio_assess, |
789 | 2254 |
zio_done, |
5530 | 2255 |
NULL |
789 | 2256 |
}; |
2257 |
||
2258 |
/* |
|
5530 | 2259 |
* Execute the I/O pipeline until one of the following occurs: |
2260 |
* (1) the I/O completes; (2) the pipeline stalls waiting for |
|
2261 |
* dependent child I/Os; (3) the I/O issues, so we're waiting |
|
2262 |
* for an I/O completion interrupt; (4) the I/O is delegated by |
|
2263 |
* vdev-level caching or aggregation; (5) the I/O is deferred |
|
2264 |
* due to vdev-level queueing; (6) the I/O is handed off to |
|
2265 |
* another thread. In all cases, the pipeline stops whenever |
|
2266 |
* there's no CPU work; it never burns a thread in cv_wait(). |
|
2267 |
* |
|
2268 |
* There's no locking on io_stage because there's no legitimate way |
|
2269 |
* for multiple threads to be attempting to process the same I/O. |
|
789 | 2270 |
*/ |
2271 |
void |
|
5530 | 2272 |
zio_execute(zio_t *zio) |
789 | 2273 |
{ |
5530 | 2274 |
while (zio->io_stage < ZIO_STAGE_DONE) { |
2275 |
uint32_t pipeline = zio->io_pipeline; |
|
2276 |
int rv; |
|
789 | 2277 |
|
5530 | 2278 |
ASSERT(!MUTEX_HELD(&zio->io_lock)); |
789 | 2279 |
|
5530 | 2280 |
/* |
2281 |
* If an error occurred outside the vdev stack, |
|
2282 |
* just execute the interlock stages to clean up. |
|
2283 |
*/ |
|
2284 |
if (zio->io_error && |
|
2285 |
((1U << zio->io_stage) & ZIO_VDEV_IO_STAGES) == 0) |
|
789 | 2286 |
pipeline &= ZIO_ERROR_PIPELINE_MASK; |
2287 |
||
5530 | 2288 |
while (((1U << ++zio->io_stage) & pipeline) == 0) |
2289 |
continue; |
|
789 | 2290 |
|
5530 | 2291 |
ASSERT(zio->io_stage <= ZIO_STAGE_DONE); |
2292 |
ASSERT(zio->io_stalled == 0); |
|
2293 |
||
2294 |
rv = zio_pipeline[zio->io_stage](zio); |
|
2295 |
||
2296 |
if (rv == ZIO_PIPELINE_STOP) |
|
2297 |
return; |
|
2298 |
||
2299 |
ASSERT(rv == ZIO_PIPELINE_CONTINUE); |
|
789 | 2300 |
} |
2301 |
} |
|
2302 |
||
3668 | 2303 |
static boolean_t |
5329 | 2304 |
zio_io_should_fail(uint16_t range) |
3668 | 2305 |
{ |
2306 |
static uint16_t allocs = 0; |
|
2307 |
||
5329 | 2308 |
return (P2PHASE(allocs++, 1U<<range) == 0); |
3668 | 2309 |
} |
2310 |
||
789 | 2311 |
/* |
2312 |
* Try to allocate an intent log block. Return 0 on success, errno on failure. |
|
2313 |
*/ |
|
2314 |
int |
|
3063
b252896b372b
6341569 zio_alloc_blk() vdev distribution performs badly
perrin
parents:
2981
diff
changeset
|
2315 |
zio_alloc_blk(spa_t *spa, uint64_t size, blkptr_t *new_bp, blkptr_t *old_bp, |
b252896b372b
6341569 zio_alloc_blk() vdev distribution performs badly
perrin
parents:
2981
diff
changeset
|
2316 |
uint64_t txg) |
789 | 2317 |
{ |
2318 |
int error; |
|
2319 |
||
1544 | 2320 |
spa_config_enter(spa, RW_READER, FTAG); |
789 | 2321 |
|
5329 | 2322 |
if (zio_zil_fail_shift && zio_io_should_fail(zio_zil_fail_shift)) { |
3668 | 2323 |
spa_config_exit(spa, FTAG); |
2324 |
return (ENOSPC); |
|
2325 |
} |
|
2326 |
||
3063
b252896b372b
6341569 zio_alloc_blk() vdev distribution performs badly
perrin
parents:
2981
diff
changeset
|
2327 |
/* |
4527 | 2328 |
* We were passed the previous log block's DVA in bp->blk_dva[0]. |
2329 |
* We use that as a hint for which vdev to allocate from next. |
|
3063
b252896b372b
6341569 zio_alloc_blk() vdev distribution performs badly
perrin
parents:
2981
diff
changeset
|
2330 |
*/ |
4527 | 2331 |
error = metaslab_alloc(spa, spa->spa_log_class, size, |
2332 |
new_bp, 1, txg, old_bp, B_TRUE); |
|
2333 |
||
2334 |
if (error) |
|
2335 |
error = metaslab_alloc(spa, spa->spa_normal_class, size, |
|
2336 |
new_bp, 1, txg, old_bp, B_TRUE); |
|
789 | 2337 |
|
2338 |
if (error == 0) { |
|
3063
b252896b372b
6341569 zio_alloc_blk() vdev distribution performs badly
perrin
parents:
2981
diff
changeset
|
2339 |
BP_SET_LSIZE(new_bp, size); |
b252896b372b
6341569 zio_alloc_blk() vdev distribution performs badly
perrin
parents:
2981
diff
changeset
|
2340 |
BP_SET_PSIZE(new_bp, size); |
b252896b372b
6341569 zio_alloc_blk() vdev distribution performs badly
perrin
parents:
2981
diff
changeset
|
2341 |
BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF); |
b252896b372b
6341569 zio_alloc_blk() vdev distribution performs badly
perrin
parents:
2981
diff
changeset
|
2342 |
BP_SET_CHECKSUM(new_bp, ZIO_CHECKSUM_ZILOG); |
13251
f037017b1f9b
1701 zio_write_encrypt ASSERT trips on old spa version
Darren Moffat <darrenm@opensolaris.org>
parents:
13230
diff
changeset
|
2343 |
BP_SET_CRYPT(new_bp, ZIO_CRYPT_OFF); |
3063
b252896b372b
6341569 zio_alloc_blk() vdev distribution performs badly
perrin
parents:
2981
diff
changeset
|
2344 |
BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); |
b252896b372b
6341569 zio_alloc_blk() vdev distribution performs badly
perrin
parents:
2981
diff
changeset
|
2345 |
BP_SET_LEVEL(new_bp, 0); |
b252896b372b
6341569 zio_alloc_blk() vdev distribution performs badly
perrin
parents:
2981
diff
changeset
|
2346 |
BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER); |
b252896b372b
6341569 zio_alloc_blk() vdev distribution performs badly
perrin
parents:
2981
diff
changeset
|
2347 |
new_bp->blk_birth = txg; |
789 | 2348 |
} |
2349 |
||
1544 | 2350 |
spa_config_exit(spa, FTAG); |
789 | 2351 |
|
2352 |
return (error); |
|
2353 |
} |
|
2354 |
||
2355 |
/* |
|
2356 |
* Free an intent log block. We know it can't be a gang block, so there's |
|
2357 |
* nothing to do except metaslab_free() it. |
|
2358 |
*/ |
|
2359 |
void |
|
2360 |
zio_free_blk(spa_t *spa, blkptr_t *bp, uint64_t txg) |
|
2361 |
{ |
|
1775
e51e26b432c0
6410698 ZFS metadata needs to be more highly replicated (ditto blocks)
billm
parents:
1732
diff
changeset
|
2362 |
ASSERT(!BP_IS_GANG(bp)); |
789 | 2363 |
|
1544 | 2364 |
spa_config_enter(spa, RW_READER, FTAG); |
789 | 2365 |
|
1807
35c8b566d7af
6410711 intent log blocks don't get invited to pool parties
bonwick
parents:
1775
diff
changeset
|
2366 |
metaslab_free(spa, bp, txg, B_FALSE); |
789 | 2367 |
|
1544 | 2368 |
spa_config_exit(spa, FTAG); |
789 | 2369 |
} |
4469
7d15cd6c3854
6566482 panic replacing a pool device whilst intent log being used
perrin
parents:
4451
diff
changeset
|
2370 |
|
7d15cd6c3854
6566482 panic replacing a pool device whilst intent log being used
perrin
parents:
4451
diff
changeset
|
2371 |
/* |
7d15cd6c3854
6566482 panic replacing a pool device whilst intent log being used
perrin
parents:
4451
diff
changeset
|
2372 |
* start an async flush of the write cache for this vdev |
7d15cd6c3854
6566482 panic replacing a pool device whilst intent log being used
perrin
parents:
4451
diff
changeset
|
2373 |
*/ |
7d15cd6c3854
6566482 panic replacing a pool device whilst intent log being used
perrin
parents:
4451
diff
changeset
|
2374 |
void |
5688
c0b02c8fd2c0
6640580 spa_get_random() is insanely slow in userland
bonwick
parents:
5530
diff
changeset
|
2375 |
zio_flush(zio_t *zio, vdev_t *vd) |
4469
7d15cd6c3854
6566482 panic replacing a pool device whilst intent log being used
perrin
parents:
4451
diff
changeset
|
2376 |
{ |
5688
c0b02c8fd2c0
6640580 spa_get_random() is insanely slow in userland
bonwick
parents:
5530
diff
changeset
|
2377 |
zio_nowait(zio_ioctl(zio, zio->io_spa, vd, DKIOCFLUSHWRITECACHE, |
4469
7d15cd6c3854
6566482 panic replacing a pool device whilst intent log being used
perrin
parents:
4451
diff
changeset
|
2378 |
NULL, NULL, ZIO_PRIORITY_NOW, |
7d15cd6c3854
6566482 panic replacing a pool device whilst intent log being used
perrin
parents:
4451
diff
changeset
|
2379 |
ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_RETRY)); |
7d15cd6c3854
6566482 panic replacing a pool device whilst intent log being used
perrin
parents:
4451
diff
changeset
|
2380 |
} |