32 #ifdef __cplusplus |
32 #ifdef __cplusplus |
33 extern "C" { |
33 extern "C" { |
34 #endif |
34 #endif |
35 |
35 |
36 /* |
36 /* |
37 * I/O Groups: pipeline stage definitions. |
37 * zio pipeline stage definitions |
38 */ |
38 */ |
39 typedef enum zio_stage { |
39 enum zio_stage { |
40 ZIO_STAGE_OPEN = 0, /* RWFCI */ |
40 ZIO_STAGE_OPEN = 1 << 0, /* RWFCI */ |
41 |
41 |
42 ZIO_STAGE_ISSUE_ASYNC, /* -W--- */ |
42 ZIO_STAGE_READ_BP_INIT = 1 << 1, /* R---- */ |
|
43 ZIO_STAGE_FREE_BP_INIT = 1 << 2, /* --F-- */ |
|
44 ZIO_STAGE_ISSUE_ASYNC = 1 << 3, /* RWF-- */ |
|
45 ZIO_STAGE_WRITE_BP_INIT = 1 << 4, /* -W--- */ |
43 |
46 |
44 ZIO_STAGE_READ_BP_INIT, /* R---- */ |
47 ZIO_STAGE_CHECKSUM_GENERATE = 1 << 5, /* -W--- */ |
45 ZIO_STAGE_WRITE_BP_INIT, /* -W--- */ |
|
46 |
48 |
47 ZIO_STAGE_CHECKSUM_GENERATE, /* -W--- */ |
49 ZIO_STAGE_DDT_READ_START = 1 << 6, /* R---- */ |
|
50 ZIO_STAGE_DDT_READ_DONE = 1 << 7, /* R---- */ |
|
51 ZIO_STAGE_DDT_WRITE = 1 << 8, /* -W--- */ |
|
52 ZIO_STAGE_DDT_FREE = 1 << 9, /* --F-- */ |
48 |
53 |
49 ZIO_STAGE_GANG_ASSEMBLE, /* RWFC- */ |
54 ZIO_STAGE_GANG_ASSEMBLE = 1 << 10, /* RWFC- */ |
50 ZIO_STAGE_GANG_ISSUE, /* RWFC- */ |
55 ZIO_STAGE_GANG_ISSUE = 1 << 11, /* RWFC- */ |
51 |
56 |
52 ZIO_STAGE_DVA_ALLOCATE, /* -W--- */ |
57 ZIO_STAGE_DVA_ALLOCATE = 1 << 12, /* -W--- */ |
53 ZIO_STAGE_DVA_FREE, /* --F-- */ |
58 ZIO_STAGE_DVA_FREE = 1 << 13, /* --F-- */ |
54 ZIO_STAGE_DVA_CLAIM, /* ---C- */ |
59 ZIO_STAGE_DVA_CLAIM = 1 << 14, /* ---C- */ |
55 |
60 |
56 ZIO_STAGE_READY, /* RWFCI */ |
61 ZIO_STAGE_READY = 1 << 15, /* RWFCI */ |
57 |
62 |
58 ZIO_STAGE_VDEV_IO_START, /* RW--I */ |
63 ZIO_STAGE_VDEV_IO_START = 1 << 16, /* RW--I */ |
59 ZIO_STAGE_VDEV_IO_DONE, /* RW--I */ |
64 ZIO_STAGE_VDEV_IO_DONE = 1 << 17, /* RW--I */ |
60 ZIO_STAGE_VDEV_IO_ASSESS, /* RW--I */ |
65 ZIO_STAGE_VDEV_IO_ASSESS = 1 << 18, /* RW--I */ |
61 |
66 |
62 ZIO_STAGE_CHECKSUM_VERIFY, /* R---- */ |
67 ZIO_STAGE_CHECKSUM_VERIFY = 1 << 19, /* R---- */ |
63 |
68 |
64 ZIO_STAGE_DONE, /* RWFCI */ |
69 ZIO_STAGE_DONE = 1 << 20 /* RWFCI */ |
65 ZIO_STAGES |
70 }; |
66 } zio_stage_t; |
|
67 |
71 |
68 #define ZIO_INTERLOCK_STAGES \ |
72 #define ZIO_INTERLOCK_STAGES \ |
69 ((1U << ZIO_STAGE_READY) | \ |
73 (ZIO_STAGE_READY | \ |
70 (1U << ZIO_STAGE_DONE)) |
74 ZIO_STAGE_DONE) |
71 |
75 |
72 #define ZIO_INTERLOCK_PIPELINE \ |
76 #define ZIO_INTERLOCK_PIPELINE \ |
73 ZIO_INTERLOCK_STAGES |
77 ZIO_INTERLOCK_STAGES |
74 |
78 |
75 #define ZIO_VDEV_IO_STAGES \ |
79 #define ZIO_VDEV_IO_STAGES \ |
76 ((1U << ZIO_STAGE_VDEV_IO_START) | \ |
80 (ZIO_STAGE_VDEV_IO_START | \ |
77 (1U << ZIO_STAGE_VDEV_IO_DONE) | \ |
81 ZIO_STAGE_VDEV_IO_DONE | \ |
78 (1U << ZIO_STAGE_VDEV_IO_ASSESS)) |
82 ZIO_STAGE_VDEV_IO_ASSESS) |
79 |
83 |
80 #define ZIO_VDEV_CHILD_PIPELINE \ |
84 #define ZIO_VDEV_CHILD_PIPELINE \ |
81 (ZIO_VDEV_IO_STAGES | \ |
85 (ZIO_VDEV_IO_STAGES | \ |
82 (1U << ZIO_STAGE_DONE)) |
86 ZIO_STAGE_DONE) |
83 |
87 |
84 #define ZIO_READ_COMMON_STAGES \ |
88 #define ZIO_READ_COMMON_STAGES \ |
85 (ZIO_INTERLOCK_STAGES | \ |
89 (ZIO_INTERLOCK_STAGES | \ |
86 ZIO_VDEV_IO_STAGES | \ |
90 ZIO_VDEV_IO_STAGES | \ |
87 (1U << ZIO_STAGE_CHECKSUM_VERIFY)) |
91 ZIO_STAGE_CHECKSUM_VERIFY) |
88 |
92 |
89 #define ZIO_READ_PHYS_PIPELINE \ |
93 #define ZIO_READ_PHYS_PIPELINE \ |
90 ZIO_READ_COMMON_STAGES |
94 ZIO_READ_COMMON_STAGES |
91 |
95 |
92 #define ZIO_READ_PIPELINE \ |
96 #define ZIO_READ_PIPELINE \ |
93 (ZIO_READ_COMMON_STAGES | \ |
97 (ZIO_READ_COMMON_STAGES | \ |
94 (1U << ZIO_STAGE_READ_BP_INIT)) |
98 ZIO_STAGE_READ_BP_INIT) |
95 |
99 |
96 #define ZIO_WRITE_COMMON_STAGES \ |
100 #define ZIO_DDT_CHILD_READ_PIPELINE \ |
97 (ZIO_INTERLOCK_STAGES | \ |
101 ZIO_READ_COMMON_STAGES |
98 ZIO_VDEV_IO_STAGES | \ |
|
99 (1U << ZIO_STAGE_ISSUE_ASYNC) | \ |
|
100 (1U << ZIO_STAGE_CHECKSUM_GENERATE)) |
|
101 |
102 |
102 #define ZIO_WRITE_PHYS_PIPELINE \ |
103 #define ZIO_DDT_READ_PIPELINE \ |
|
104 (ZIO_INTERLOCK_STAGES | \ |
|
105 ZIO_STAGE_READ_BP_INIT | \ |
|
106 ZIO_STAGE_DDT_READ_START | \ |
|
107 ZIO_STAGE_DDT_READ_DONE) |
|
108 |
|
109 #define ZIO_WRITE_COMMON_STAGES \ |
|
110 (ZIO_INTERLOCK_STAGES | \ |
|
111 ZIO_VDEV_IO_STAGES | \ |
|
112 ZIO_STAGE_ISSUE_ASYNC | \ |
|
113 ZIO_STAGE_CHECKSUM_GENERATE) |
|
114 |
|
115 #define ZIO_WRITE_PHYS_PIPELINE \ |
103 ZIO_WRITE_COMMON_STAGES |
116 ZIO_WRITE_COMMON_STAGES |
104 |
117 |
105 #define ZIO_REWRITE_PIPELINE \ |
118 #define ZIO_REWRITE_PIPELINE \ |
106 (ZIO_WRITE_COMMON_STAGES | \ |
119 (ZIO_WRITE_COMMON_STAGES | \ |
107 (1U << ZIO_STAGE_WRITE_BP_INIT)) |
120 ZIO_STAGE_WRITE_BP_INIT) |
108 |
121 |
109 #define ZIO_WRITE_PIPELINE \ |
122 #define ZIO_WRITE_PIPELINE \ |
110 (ZIO_WRITE_COMMON_STAGES | \ |
123 (ZIO_WRITE_COMMON_STAGES | \ |
111 (1U << ZIO_STAGE_WRITE_BP_INIT) | \ |
124 ZIO_STAGE_WRITE_BP_INIT | \ |
112 (1U << ZIO_STAGE_DVA_ALLOCATE)) |
125 ZIO_STAGE_DVA_ALLOCATE) |
113 |
126 |
114 #define ZIO_GANG_STAGES \ |
127 #define ZIO_DDT_CHILD_WRITE_PIPELINE \ |
115 ((1U << ZIO_STAGE_GANG_ASSEMBLE) | \ |
128 (ZIO_INTERLOCK_STAGES | \ |
116 (1U << ZIO_STAGE_GANG_ISSUE)) |
129 ZIO_VDEV_IO_STAGES | \ |
|
130 ZIO_STAGE_DVA_ALLOCATE) |
117 |
131 |
118 #define ZIO_FREE_PIPELINE \ |
132 #define ZIO_DDT_WRITE_PIPELINE \ |
119 (ZIO_INTERLOCK_STAGES | \ |
133 (ZIO_INTERLOCK_STAGES | \ |
120 (1U << ZIO_STAGE_DVA_FREE)) |
134 ZIO_STAGE_ISSUE_ASYNC | \ |
|
135 ZIO_STAGE_WRITE_BP_INIT | \ |
|
136 ZIO_STAGE_CHECKSUM_GENERATE | \ |
|
137 ZIO_STAGE_DDT_WRITE) |
121 |
138 |
122 #define ZIO_CLAIM_PIPELINE \ |
139 #define ZIO_GANG_STAGES \ |
123 (ZIO_INTERLOCK_STAGES | \ |
140 (ZIO_STAGE_GANG_ASSEMBLE | \ |
124 (1U << ZIO_STAGE_DVA_CLAIM)) |
141 ZIO_STAGE_GANG_ISSUE) |
125 |
142 |
126 #define ZIO_IOCTL_PIPELINE \ |
143 #define ZIO_FREE_PIPELINE \ |
127 (ZIO_INTERLOCK_STAGES | \ |
144 (ZIO_INTERLOCK_STAGES | \ |
128 (1U << ZIO_STAGE_VDEV_IO_START) | \ |
145 ZIO_STAGE_FREE_BP_INIT | \ |
129 (1U << ZIO_STAGE_VDEV_IO_ASSESS)) |
146 ZIO_STAGE_DVA_FREE) |
130 |
147 |
131 #define ZIO_CONFIG_LOCK_BLOCKING_STAGES \ |
148 #define ZIO_DDT_FREE_PIPELINE \ |
132 ((1U << ZIO_STAGE_VDEV_IO_START) | \ |
149 (ZIO_INTERLOCK_STAGES | \ |
133 (1U << ZIO_STAGE_DVA_ALLOCATE) | \ |
150 ZIO_STAGE_FREE_BP_INIT | \ |
134 (1U << ZIO_STAGE_DVA_CLAIM)) |
151 ZIO_STAGE_ISSUE_ASYNC | \ |
|
152 ZIO_STAGE_DDT_FREE) |
|
153 |
|
154 #define ZIO_CLAIM_PIPELINE \ |
|
155 (ZIO_INTERLOCK_STAGES | \ |
|
156 ZIO_STAGE_DVA_CLAIM) |
|
157 |
|
158 #define ZIO_IOCTL_PIPELINE \ |
|
159 (ZIO_INTERLOCK_STAGES | \ |
|
160 ZIO_STAGE_VDEV_IO_START | \ |
|
161 ZIO_STAGE_VDEV_IO_ASSESS) |
|
162 |
|
163 #define ZIO_BLOCKING_STAGES \ |
|
164 (ZIO_STAGE_DVA_ALLOCATE | \ |
|
165 ZIO_STAGE_DVA_CLAIM | \ |
|
166 ZIO_STAGE_VDEV_IO_START) |
135 |
167 |
136 extern void zio_inject_init(void); |
168 extern void zio_inject_init(void); |
137 extern void zio_inject_fini(void); |
169 extern void zio_inject_fini(void); |
138 |
170 |
139 #ifdef __cplusplus |
171 #ifdef __cplusplus |