]>
Commit | Line | Data |
---|---|---|
55716d26 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
a4ffc152 MP |
2 | /* |
3 | * Copyright (C) 2012 Red Hat, Inc. | |
4 | * | |
5 | * Author: Mikulas Patocka <mpatocka@redhat.com> | |
6 | * | |
7 | * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors | |
8 | * | |
a4ffc152 MP |
9 | * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set |
10 | * default prefetch value. Data are read in "prefetch_cluster" chunks from the | |
11 | * hash device. Setting this greatly improves performance when data and hash | |
12 | * are on the same disk on different partitions on devices with poor random | |
13 | * access behavior. | |
14 | */ | |
15 | ||
ffa39380 | 16 | #include "dm-verity.h" |
a739ff3f | 17 | #include "dm-verity-fec.h" |
88cd3e6c | 18 | #include "dm-verity-verify-sig.h" |
074c4466 | 19 | #include "dm-audit.h" |
a4ffc152 | 20 | #include <linux/module.h> |
65ff5b7d | 21 | #include <linux/reboot.h> |
24b83deb | 22 | #include <linux/scatterlist.h> |
b6c1c574 | 23 | #include <linux/string.h> |
ba2cce82 | 24 | #include <linux/jump_label.h> |
a4ffc152 MP |
25 | |
26 | #define DM_MSG_PREFIX "verity" | |
27 | ||
65ff5b7d ST |
28 | #define DM_VERITY_ENV_LENGTH 42 |
29 | #define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR" | |
30 | ||
a4ffc152 MP |
31 | #define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144 |
32 | ||
65ff5b7d ST |
33 | #define DM_VERITY_MAX_CORRUPTED_ERRS 100 |
34 | ||
35 | #define DM_VERITY_OPT_LOGGING "ignore_corruption" | |
36 | #define DM_VERITY_OPT_RESTART "restart_on_corruption" | |
e1fef0b0 | 37 | #define DM_VERITY_OPT_PANIC "panic_on_corruption" |
0cc37c2d | 38 | #define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks" |
843f38d3 | 39 | #define DM_VERITY_OPT_AT_MOST_ONCE "check_at_most_once" |
5721d4e5 | 40 | #define DM_VERITY_OPT_TASKLET_VERIFY "try_verify_in_tasklet" |
a4ffc152 | 41 | |
8c22816d | 42 | #define DM_VERITY_OPTS_MAX (4 + DM_VERITY_OPTS_FEC + \ |
88cd3e6c | 43 | DM_VERITY_ROOT_HASH_VERIFICATION_OPTS) |
753c1fd0 | 44 | |
86a3238c | 45 | static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE; |
a4ffc152 | 46 | |
6a808034 | 47 | module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644); |
a4ffc152 | 48 | |
ba2cce82 MS |
49 | static DEFINE_STATIC_KEY_FALSE(use_tasklet_enabled); |
50 | ||
3b6b7813 MP |
51 | struct dm_verity_prefetch_work { |
52 | struct work_struct work; | |
53 | struct dm_verity *v; | |
54 | sector_t block; | |
86a3238c | 55 | unsigned int n_blocks; |
3b6b7813 MP |
56 | }; |
57 | ||
a4ffc152 MP |
58 | /* |
59 | * Auxiliary structure appended to each dm-bufio buffer. If the value | |
60 | * hash_verified is nonzero, hash of the block has been verified. | |
61 | * | |
62 | * The variable hash_verified is set to 0 when allocating the buffer, then | |
63 | * it can be changed to 1 and it is never reset to 0 again. | |
64 | * | |
65 | * There is no lock around this value, a race condition can at worst cause | |
66 | * that multiple processes verify the hash of the same buffer simultaneously | |
67 | * and write 1 to hash_verified simultaneously. | |
68 | * This condition is harmless, so we don't need locking. | |
69 | */ | |
70 | struct buffer_aux { | |
71 | int hash_verified; | |
72 | }; | |
73 | ||
74 | /* | |
75 | * Initialize struct buffer_aux for a freshly created buffer. | |
76 | */ | |
77 | static void dm_bufio_alloc_callback(struct dm_buffer *buf) | |
78 | { | |
79 | struct buffer_aux *aux = dm_bufio_get_aux_data(buf); | |
80 | ||
81 | aux->hash_verified = 0; | |
82 | } | |
83 | ||
84 | /* | |
85 | * Translate input sector number to the sector number on the target device. | |
86 | */ | |
87 | static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector) | |
88 | { | |
89 | return v->data_start + dm_target_offset(v->ti, bi_sector); | |
90 | } | |
91 | ||
92 | /* | |
93 | * Return hash position of a specified block at a specified tree level | |
94 | * (0 is the lowest level). | |
95 | * The lowest "hash_per_block_bits"-bits of the result denote hash position | |
96 | * inside a hash block. The remaining bits denote location of the hash block. | |
97 | */ | |
98 | static sector_t verity_position_at_level(struct dm_verity *v, sector_t block, | |
99 | int level) | |
100 | { | |
101 | return block >> (level * v->hash_per_block_bits); | |
102 | } | |
103 | ||
d1ac3ff0 GBY |
104 | static int verity_hash_update(struct dm_verity *v, struct ahash_request *req, |
105 | const u8 *data, size_t len, | |
12f1ffc4 | 106 | struct crypto_wait *wait) |
6dbeda34 | 107 | { |
d1ac3ff0 | 108 | struct scatterlist sg; |
6dbeda34 | 109 | |
e4b069e0 MP |
110 | if (likely(!is_vmalloc_addr(data))) { |
111 | sg_init_one(&sg, data, len); | |
112 | ahash_request_set_crypt(req, &sg, NULL, len); | |
113 | return crypto_wait_req(crypto_ahash_update(req), wait); | |
e4b069e0 | 114 | } |
1c3fe2fa HM |
115 | |
116 | do { | |
117 | int r; | |
118 | size_t this_step = min_t(size_t, len, PAGE_SIZE - offset_in_page(data)); | |
119 | ||
120 | flush_kernel_vmap_range((void *)data, this_step); | |
121 | sg_init_table(&sg, 1); | |
122 | sg_set_page(&sg, vmalloc_to_page(data), this_step, offset_in_page(data)); | |
123 | ahash_request_set_crypt(req, &sg, NULL, this_step); | |
124 | r = crypto_wait_req(crypto_ahash_update(req), wait); | |
125 | if (unlikely(r)) | |
126 | return r; | |
127 | data += this_step; | |
128 | len -= this_step; | |
129 | } while (len); | |
130 | ||
131 | return 0; | |
d1ac3ff0 GBY |
132 | } |
133 | ||
134 | /* | |
135 | * Wrapper for crypto_ahash_init, which handles verity salting. | |
136 | */ | |
137 | static int verity_hash_init(struct dm_verity *v, struct ahash_request *req, | |
28f07f2a | 138 | struct crypto_wait *wait, bool may_sleep) |
d1ac3ff0 GBY |
139 | { |
140 | int r; | |
141 | ||
142 | ahash_request_set_tfm(req, v->tfm); | |
28f07f2a MP |
143 | ahash_request_set_callback(req, |
144 | may_sleep ? CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG : 0, | |
145 | crypto_req_done, (void *)wait); | |
12f1ffc4 | 146 | crypto_init_wait(wait); |
d1ac3ff0 | 147 | |
12f1ffc4 | 148 | r = crypto_wait_req(crypto_ahash_init(req), wait); |
d1ac3ff0 GBY |
149 | |
150 | if (unlikely(r < 0)) { | |
28f07f2a MP |
151 | if (r != -ENOMEM) |
152 | DMERR("crypto_ahash_init failed: %d", r); | |
d1ac3ff0 GBY |
153 | return r; |
154 | } | |
155 | ||
f52236e0 | 156 | if (likely(v->salt_size && (v->version >= 1))) |
12f1ffc4 | 157 | r = verity_hash_update(v, req, v->salt, v->salt_size, wait); |
6dbeda34 ST |
158 | |
159 | return r; | |
160 | } | |
161 | ||
d1ac3ff0 | 162 | static int verity_hash_final(struct dm_verity *v, struct ahash_request *req, |
12f1ffc4 | 163 | u8 *digest, struct crypto_wait *wait) |
6dbeda34 ST |
164 | { |
165 | int r; | |
166 | ||
f52236e0 | 167 | if (unlikely(v->salt_size && (!v->version))) { |
12f1ffc4 | 168 | r = verity_hash_update(v, req, v->salt, v->salt_size, wait); |
6dbeda34 ST |
169 | |
170 | if (r < 0) { | |
1c131886 | 171 | DMERR("%s failed updating salt: %d", __func__, r); |
d1ac3ff0 | 172 | goto out; |
6dbeda34 ST |
173 | } |
174 | } | |
175 | ||
d1ac3ff0 | 176 | ahash_request_set_crypt(req, NULL, digest, 0); |
12f1ffc4 | 177 | r = crypto_wait_req(crypto_ahash_final(req), wait); |
d1ac3ff0 | 178 | out: |
6dbeda34 ST |
179 | return r; |
180 | } | |
181 | ||
d1ac3ff0 | 182 | int verity_hash(struct dm_verity *v, struct ahash_request *req, |
28f07f2a | 183 | const u8 *data, size_t len, u8 *digest, bool may_sleep) |
6dbeda34 ST |
184 | { |
185 | int r; | |
12f1ffc4 | 186 | struct crypto_wait wait; |
6dbeda34 | 187 | |
28f07f2a | 188 | r = verity_hash_init(v, req, &wait, may_sleep); |
6dbeda34 | 189 | if (unlikely(r < 0)) |
d1ac3ff0 | 190 | goto out; |
6dbeda34 | 191 | |
12f1ffc4 | 192 | r = verity_hash_update(v, req, data, len, &wait); |
6dbeda34 | 193 | if (unlikely(r < 0)) |
d1ac3ff0 GBY |
194 | goto out; |
195 | ||
12f1ffc4 | 196 | r = verity_hash_final(v, req, digest, &wait); |
6dbeda34 | 197 | |
d1ac3ff0 GBY |
198 | out: |
199 | return r; | |
6dbeda34 ST |
200 | } |
201 | ||
a4ffc152 | 202 | static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level, |
86a3238c | 203 | sector_t *hash_block, unsigned int *offset) |
a4ffc152 MP |
204 | { |
205 | sector_t position = verity_position_at_level(v, block, level); | |
86a3238c | 206 | unsigned int idx; |
a4ffc152 MP |
207 | |
208 | *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits); | |
209 | ||
210 | if (!offset) | |
211 | return; | |
212 | ||
213 | idx = position & ((1 << v->hash_per_block_bits) - 1); | |
214 | if (!v->version) | |
215 | *offset = idx * v->digest_size; | |
216 | else | |
217 | *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits); | |
218 | } | |
219 | ||
65ff5b7d ST |
220 | /* |
221 | * Handle verification errors. | |
222 | */ | |
223 | static int verity_handle_err(struct dm_verity *v, enum verity_block_type type, | |
224 | unsigned long long block) | |
225 | { | |
226 | char verity_env[DM_VERITY_ENV_LENGTH]; | |
227 | char *envp[] = { verity_env, NULL }; | |
228 | const char *type_str = ""; | |
229 | struct mapped_device *md = dm_table_get_md(v->ti->table); | |
230 | ||
231 | /* Corruption should be visible in device status in all modes */ | |
5721d4e5 | 232 | v->hash_failed = true; |
65ff5b7d ST |
233 | |
234 | if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS) | |
235 | goto out; | |
236 | ||
237 | v->corrupted_errs++; | |
238 | ||
239 | switch (type) { | |
240 | case DM_VERITY_BLOCK_TYPE_DATA: | |
241 | type_str = "data"; | |
242 | break; | |
243 | case DM_VERITY_BLOCK_TYPE_METADATA: | |
244 | type_str = "metadata"; | |
245 | break; | |
246 | default: | |
247 | BUG(); | |
248 | } | |
249 | ||
2eba4e64 MB |
250 | DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name, |
251 | type_str, block); | |
65ff5b7d | 252 | |
074c4466 | 253 | if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) { |
65ff5b7d | 254 | DMERR("%s: reached maximum errors", v->data_dev->name); |
074c4466 MW |
255 | dm_audit_log_target(DM_MSG_PREFIX, "max-corrupted-errors", v->ti, 0); |
256 | } | |
65ff5b7d ST |
257 | |
258 | snprintf(verity_env, DM_VERITY_ENV_LENGTH, "%s=%d,%llu", | |
259 | DM_VERITY_ENV_VAR_NAME, type, block); | |
260 | ||
261 | kobject_uevent_env(&disk_to_dev(dm_disk(md))->kobj, KOBJ_CHANGE, envp); | |
262 | ||
263 | out: | |
264 | if (v->mode == DM_VERITY_MODE_LOGGING) | |
265 | return 0; | |
266 | ||
267 | if (v->mode == DM_VERITY_MODE_RESTART) | |
268 | kernel_restart("dm-verity device corrupted"); | |
269 | ||
e1fef0b0 JL |
270 | if (v->mode == DM_VERITY_MODE_PANIC) |
271 | panic("dm-verity device corrupted"); | |
272 | ||
65ff5b7d ST |
273 | return 1; |
274 | } | |
275 | ||
a4ffc152 MP |
276 | /* |
277 | * Verify hash of a metadata block pertaining to the specified data block | |
278 | * ("block" argument) at a specified level ("level" argument). | |
279 | * | |
ffa39380 ST |
280 | * On successful return, verity_io_want_digest(v, io) contains the hash value |
281 | * for a lower tree level or for the data block (if we're at the lowest level). | |
a4ffc152 MP |
282 | * |
283 | * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned. | |
284 | * If "skip_unverified" is false, unverified buffer is hashed and verified | |
ffa39380 | 285 | * against current value of verity_io_want_digest(v, io). |
a4ffc152 | 286 | */ |
6dbeda34 ST |
287 | static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, |
288 | sector_t block, int level, bool skip_unverified, | |
289 | u8 *want_digest) | |
a4ffc152 | 290 | { |
a4ffc152 MP |
291 | struct dm_buffer *buf; |
292 | struct buffer_aux *aux; | |
293 | u8 *data; | |
294 | int r; | |
295 | sector_t hash_block; | |
86a3238c | 296 | unsigned int offset; |
a4ffc152 MP |
297 | |
298 | verity_hash_at_level(v, block, level, &hash_block, &offset); | |
299 | ||
ba2cce82 | 300 | if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) { |
5721d4e5 NH |
301 | data = dm_bufio_get(v->bufio, hash_block, &buf); |
302 | if (data == NULL) { | |
303 | /* | |
304 | * In tasklet and the hash was not in the bufio cache. | |
305 | * Return early and resume execution from a work-queue | |
306 | * to read the hash from disk. | |
307 | */ | |
308 | return -EAGAIN; | |
309 | } | |
310 | } else | |
311 | data = dm_bufio_read(v->bufio, hash_block, &buf); | |
312 | ||
fc0a4461 | 313 | if (IS_ERR(data)) |
a4ffc152 MP |
314 | return PTR_ERR(data); |
315 | ||
316 | aux = dm_bufio_get_aux_data(buf); | |
317 | ||
318 | if (!aux->hash_verified) { | |
a4ffc152 MP |
319 | if (skip_unverified) { |
320 | r = 1; | |
321 | goto release_ret_r; | |
322 | } | |
323 | ||
d1ac3ff0 | 324 | r = verity_hash(v, verity_io_hash_req(v, io), |
6dbeda34 | 325 | data, 1 << v->hash_dev_block_bits, |
28f07f2a | 326 | verity_io_real_digest(v, io), !io->in_tasklet); |
6dbeda34 | 327 | if (unlikely(r < 0)) |
a4ffc152 | 328 | goto release_ret_r; |
a4ffc152 | 329 | |
ffa39380 | 330 | if (likely(memcmp(verity_io_real_digest(v, io), want_digest, |
6dbeda34 ST |
331 | v->digest_size) == 0)) |
332 | aux->hash_verified = 1; | |
ba2cce82 MS |
333 | else if (static_branch_unlikely(&use_tasklet_enabled) && |
334 | io->in_tasklet) { | |
5721d4e5 NH |
335 | /* |
336 | * Error handling code (FEC included) cannot be run in a | |
337 | * tasklet since it may sleep, so fallback to work-queue. | |
338 | */ | |
339 | r = -EAGAIN; | |
340 | goto release_ret_r; | |
03b18887 HM |
341 | } else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_METADATA, |
342 | hash_block, data, NULL) == 0) | |
a739ff3f | 343 | aux->hash_verified = 1; |
6dbeda34 ST |
344 | else if (verity_handle_err(v, |
345 | DM_VERITY_BLOCK_TYPE_METADATA, | |
346 | hash_block)) { | |
074c4466 MW |
347 | struct bio *bio = |
348 | dm_bio_from_per_bio_data(io, | |
349 | v->ti->per_io_data_size); | |
350 | dm_audit_log_bio(DM_MSG_PREFIX, "verify-metadata", bio, | |
351 | block, 0); | |
6dbeda34 | 352 | r = -EIO; |
a4ffc152 MP |
353 | goto release_ret_r; |
354 | } | |
a4ffc152 MP |
355 | } |
356 | ||
357 | data += offset; | |
6dbeda34 ST |
358 | memcpy(want_digest, data, v->digest_size); |
359 | r = 0; | |
a4ffc152 MP |
360 | |
361 | release_ret_r: | |
362 | dm_bufio_release(buf); | |
a4ffc152 MP |
363 | return r; |
364 | } | |
365 | ||
6dbeda34 ST |
366 | /* |
367 | * Find a hash for a given block, write it to digest and verify the integrity | |
368 | * of the hash tree if necessary. | |
369 | */ | |
ffa39380 | 370 | int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, |
0cc37c2d | 371 | sector_t block, u8 *digest, bool *is_zero) |
6dbeda34 | 372 | { |
0cc37c2d | 373 | int r = 0, i; |
6dbeda34 ST |
374 | |
375 | if (likely(v->levels)) { | |
376 | /* | |
377 | * First, we try to get the requested hash for | |
378 | * the current block. If the hash block itself is | |
379 | * verified, zero is returned. If it isn't, this | |
380 | * function returns 1 and we fall back to whole | |
381 | * chain verification. | |
382 | */ | |
383 | r = verity_verify_level(v, io, block, 0, true, digest); | |
384 | if (likely(r <= 0)) | |
0cc37c2d | 385 | goto out; |
6dbeda34 ST |
386 | } |
387 | ||
388 | memcpy(digest, v->root_digest, v->digest_size); | |
389 | ||
390 | for (i = v->levels - 1; i >= 0; i--) { | |
391 | r = verity_verify_level(v, io, block, i, false, digest); | |
392 | if (unlikely(r)) | |
0cc37c2d | 393 | goto out; |
6dbeda34 | 394 | } |
0cc37c2d ST |
395 | out: |
396 | if (!r && v->zero_digest) | |
397 | *is_zero = !memcmp(v->zero_digest, digest, v->digest_size); | |
398 | else | |
399 | *is_zero = false; | |
6dbeda34 | 400 | |
0cc37c2d | 401 | return r; |
6dbeda34 ST |
402 | } |
403 | ||
d1ac3ff0 GBY |
404 | /* |
405 | * Calculates the digest for the given bio | |
406 | */ | |
d4b1aaf5 | 407 | static int verity_for_io_block(struct dm_verity *v, struct dm_verity_io *io, |
408 | struct bvec_iter *iter, struct crypto_wait *wait) | |
d1ac3ff0 GBY |
409 | { |
410 | unsigned int todo = 1 << v->data_dev_block_bits; | |
411 | struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); | |
412 | struct scatterlist sg; | |
413 | struct ahash_request *req = verity_io_hash_req(v, io); | |
414 | ||
415 | do { | |
416 | int r; | |
417 | unsigned int len; | |
418 | struct bio_vec bv = bio_iter_iovec(bio, *iter); | |
419 | ||
420 | sg_init_table(&sg, 1); | |
421 | ||
422 | len = bv.bv_len; | |
423 | ||
424 | if (likely(len >= todo)) | |
425 | len = todo; | |
426 | /* | |
427 | * Operating on a single page at a time looks suboptimal | |
428 | * until you consider the typical block size is 4,096B. | |
429 | * Going through this loops twice should be very rare. | |
430 | */ | |
431 | sg_set_page(&sg, bv.bv_page, len, bv.bv_offset); | |
432 | ahash_request_set_crypt(req, &sg, NULL, len); | |
12f1ffc4 | 433 | r = crypto_wait_req(crypto_ahash_update(req), wait); |
d1ac3ff0 GBY |
434 | |
435 | if (unlikely(r < 0)) { | |
1c131886 | 436 | DMERR("%s crypto op failed: %d", __func__, r); |
d1ac3ff0 GBY |
437 | return r; |
438 | } | |
439 | ||
440 | bio_advance_iter(bio, iter, len); | |
441 | todo -= len; | |
442 | } while (todo); | |
443 | ||
444 | return 0; | |
445 | } | |
446 | ||
bb4d73ac ST |
447 | /* |
448 | * Calls function process for 1 << v->data_dev_block_bits bytes in the bio_vec | |
449 | * starting from iter. | |
450 | */ | |
451 | int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io, | |
452 | struct bvec_iter *iter, | |
453 | int (*process)(struct dm_verity *v, | |
454 | struct dm_verity_io *io, u8 *data, | |
455 | size_t len)) | |
456 | { | |
86a3238c | 457 | unsigned int todo = 1 << v->data_dev_block_bits; |
30187e1d | 458 | struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); |
bb4d73ac ST |
459 | |
460 | do { | |
461 | int r; | |
462 | u8 *page; | |
86a3238c | 463 | unsigned int len; |
bb4d73ac ST |
464 | struct bio_vec bv = bio_iter_iovec(bio, *iter); |
465 | ||
30495e68 | 466 | page = bvec_kmap_local(&bv); |
bb4d73ac ST |
467 | len = bv.bv_len; |
468 | ||
469 | if (likely(len >= todo)) | |
470 | len = todo; | |
471 | ||
30495e68 CH |
472 | r = process(v, io, page, len); |
473 | kunmap_local(page); | |
bb4d73ac ST |
474 | |
475 | if (r < 0) | |
476 | return r; | |
477 | ||
478 | bio_advance_iter(bio, iter, len); | |
479 | todo -= len; | |
480 | } while (todo); | |
481 | ||
482 | return 0; | |
483 | } | |
484 | ||
0cc37c2d ST |
485 | static int verity_bv_zero(struct dm_verity *v, struct dm_verity_io *io, |
486 | u8 *data, size_t len) | |
487 | { | |
488 | memset(data, 0, len); | |
489 | return 0; | |
490 | } | |
491 | ||
843f38d3 PT |
492 | /* |
493 | * Moves the bio iter one data block forward. | |
494 | */ | |
495 | static inline void verity_bv_skip_block(struct dm_verity *v, | |
496 | struct dm_verity_io *io, | |
497 | struct bvec_iter *iter) | |
498 | { | |
499 | struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); | |
500 | ||
501 | bio_advance_iter(bio, iter, 1 << v->data_dev_block_bits); | |
502 | } | |
503 | ||
a4ffc152 MP |
504 | /* |
505 | * Verify one "dm_verity_io" structure. | |
506 | */ | |
507 | static int verity_verify_io(struct dm_verity_io *io) | |
508 | { | |
0cc37c2d | 509 | bool is_zero; |
a4ffc152 | 510 | struct dm_verity *v = io->v; |
0a36463f | 511 | #if defined(CONFIG_DM_VERITY_FEC) |
bb4d73ac | 512 | struct bvec_iter start; |
0a36463f | 513 | #endif |
e9307e3d MS |
514 | struct bvec_iter iter_copy; |
515 | struct bvec_iter *iter; | |
12f1ffc4 | 516 | struct crypto_wait wait; |
2c0468e0 | 517 | struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); |
5721d4e5 | 518 | unsigned int b; |
a4ffc152 | 519 | |
e9307e3d MS |
520 | if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) { |
521 | /* | |
522 | * Copy the iterator in case we need to restart | |
523 | * verification in a work-queue. | |
524 | */ | |
525 | iter_copy = io->iter; | |
526 | iter = &iter_copy; | |
527 | } else | |
528 | iter = &io->iter; | |
a4ffc152 MP |
529 | |
530 | for (b = 0; b < io->n_blocks; b++) { | |
a4ffc152 | 531 | int r; |
843f38d3 | 532 | sector_t cur_block = io->block + b; |
d1ac3ff0 | 533 | struct ahash_request *req = verity_io_hash_req(v, io); |
a4ffc152 | 534 | |
e8c5d45f | 535 | if (v->validated_blocks && bio->bi_status == BLK_STS_OK && |
843f38d3 | 536 | likely(test_bit(cur_block, v->validated_blocks))) { |
e9307e3d | 537 | verity_bv_skip_block(v, io, iter); |
843f38d3 PT |
538 | continue; |
539 | } | |
540 | ||
541 | r = verity_hash_for_block(v, io, cur_block, | |
0cc37c2d ST |
542 | verity_io_want_digest(v, io), |
543 | &is_zero); | |
6dbeda34 ST |
544 | if (unlikely(r < 0)) |
545 | return r; | |
a4ffc152 | 546 | |
0cc37c2d ST |
547 | if (is_zero) { |
548 | /* | |
549 | * If we expect a zero block, don't validate, just | |
550 | * return zeros. | |
551 | */ | |
e9307e3d | 552 | r = verity_for_bv_block(v, io, iter, |
0cc37c2d ST |
553 | verity_bv_zero); |
554 | if (unlikely(r < 0)) | |
555 | return r; | |
556 | ||
557 | continue; | |
558 | } | |
559 | ||
28f07f2a | 560 | r = verity_hash_init(v, req, &wait, !io->in_tasklet); |
6dbeda34 | 561 | if (unlikely(r < 0)) |
a4ffc152 | 562 | return r; |
a4ffc152 | 563 | |
0a36463f MS |
564 | #if defined(CONFIG_DM_VERITY_FEC) |
565 | if (verity_fec_is_enabled(v)) | |
e9307e3d | 566 | start = *iter; |
0a36463f | 567 | #endif |
e9307e3d | 568 | r = verity_for_io_block(v, io, iter, &wait); |
bb4d73ac ST |
569 | if (unlikely(r < 0)) |
570 | return r; | |
a4ffc152 | 571 | |
d1ac3ff0 | 572 | r = verity_hash_final(v, req, verity_io_real_digest(v, io), |
12f1ffc4 | 573 | &wait); |
6dbeda34 | 574 | if (unlikely(r < 0)) |
a4ffc152 | 575 | return r; |
6dbeda34 | 576 | |
ffa39380 | 577 | if (likely(memcmp(verity_io_real_digest(v, io), |
843f38d3 PT |
578 | verity_io_want_digest(v, io), v->digest_size) == 0)) { |
579 | if (v->validated_blocks) | |
580 | set_bit(cur_block, v->validated_blocks); | |
6dbeda34 | 581 | continue; |
ba2cce82 MS |
582 | } else if (static_branch_unlikely(&use_tasklet_enabled) && |
583 | io->in_tasklet) { | |
5721d4e5 NH |
584 | /* |
585 | * Error handling code (FEC included) cannot be run in a | |
586 | * tasklet since it may sleep, so fallback to work-queue. | |
587 | */ | |
588 | return -EAGAIN; | |
0a36463f | 589 | #if defined(CONFIG_DM_VERITY_FEC) |
20e6fc85 | 590 | } else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA, |
5721d4e5 | 591 | cur_block, NULL, &start) == 0) { |
a739ff3f | 592 | continue; |
0a36463f | 593 | #endif |
20e6fc85 | 594 | } else { |
2c0468e0 AK |
595 | if (bio->bi_status) { |
596 | /* | |
597 | * Error correction failed; Just return error | |
598 | */ | |
599 | return -EIO; | |
600 | } | |
601 | if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA, | |
074c4466 MW |
602 | cur_block)) { |
603 | dm_audit_log_bio(DM_MSG_PREFIX, "verify-data", | |
604 | bio, cur_block, 0); | |
2c0468e0 | 605 | return -EIO; |
074c4466 | 606 | } |
2c0468e0 | 607 | } |
a4ffc152 | 608 | } |
a4ffc152 MP |
609 | |
610 | return 0; | |
611 | } | |
612 | ||
252bd125 HK |
613 | /* |
614 | * Skip verity work in response to I/O error when system is shutting down. | |
615 | */ | |
616 | static inline bool verity_is_system_shutting_down(void) | |
617 | { | |
618 | return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF | |
619 | || system_state == SYSTEM_RESTART; | |
620 | } | |
621 | ||
a4ffc152 MP |
622 | /* |
623 | * End one "io" structure with a given error. | |
624 | */ | |
4e4cbee9 | 625 | static void verity_finish_io(struct dm_verity_io *io, blk_status_t status) |
a4ffc152 | 626 | { |
a4ffc152 | 627 | struct dm_verity *v = io->v; |
30187e1d | 628 | struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); |
a4ffc152 MP |
629 | |
630 | bio->bi_end_io = io->orig_bi_end_io; | |
4e4cbee9 | 631 | bio->bi_status = status; |
a4ffc152 | 632 | |
ba2cce82 | 633 | if (!static_branch_unlikely(&use_tasklet_enabled) || !io->in_tasklet) |
5721d4e5 | 634 | verity_fec_finish_io(io); |
a739ff3f | 635 | |
4246a0b6 | 636 | bio_endio(bio); |
a4ffc152 MP |
637 | } |
638 | ||
639 | static void verity_work(struct work_struct *w) | |
640 | { | |
641 | struct dm_verity_io *io = container_of(w, struct dm_verity_io, work); | |
642 | ||
5721d4e5 NH |
643 | io->in_tasklet = false; |
644 | ||
4e4cbee9 | 645 | verity_finish_io(io, errno_to_blk_status(verity_verify_io(io))); |
a4ffc152 MP |
646 | } |
647 | ||
5721d4e5 NH |
648 | static void verity_tasklet(unsigned long data) |
649 | { | |
650 | struct dm_verity_io *io = (struct dm_verity_io *)data; | |
651 | int err; | |
652 | ||
653 | io->in_tasklet = true; | |
654 | err = verity_verify_io(io); | |
28f07f2a | 655 | if (err == -EAGAIN || err == -ENOMEM) { |
5721d4e5 NH |
656 | /* fallback to retrying with work-queue */ |
657 | INIT_WORK(&io->work, verity_work); | |
658 | queue_work(io->v->verify_wq, &io->work); | |
659 | return; | |
660 | } | |
661 | ||
662 | verity_finish_io(io, errno_to_blk_status(err)); | |
663 | } | |
664 | ||
4246a0b6 | 665 | static void verity_end_io(struct bio *bio) |
a4ffc152 MP |
666 | { |
667 | struct dm_verity_io *io = bio->bi_private; | |
668 | ||
252bd125 | 669 | if (bio->bi_status && |
0193e396 WB |
670 | (!verity_fec_is_enabled(io->v) || |
671 | verity_is_system_shutting_down() || | |
672 | (bio->bi_opf & REQ_RAHEAD))) { | |
4e4cbee9 | 673 | verity_finish_io(io, bio->bi_status); |
a4ffc152 MP |
674 | return; |
675 | } | |
676 | ||
ba2cce82 | 677 | if (static_branch_unlikely(&use_tasklet_enabled) && io->v->use_tasklet) { |
5721d4e5 NH |
678 | tasklet_init(&io->tasklet, verity_tasklet, (unsigned long)io); |
679 | tasklet_schedule(&io->tasklet); | |
680 | } else { | |
681 | INIT_WORK(&io->work, verity_work); | |
682 | queue_work(io->v->verify_wq, &io->work); | |
683 | } | |
a4ffc152 MP |
684 | } |
685 | ||
686 | /* | |
687 | * Prefetch buffers for the specified io. | |
688 | * The root buffer is not prefetched, it is assumed that it will be cached | |
689 | * all the time. | |
690 | */ | |
3b6b7813 | 691 | static void verity_prefetch_io(struct work_struct *work) |
a4ffc152 | 692 | { |
3b6b7813 MP |
693 | struct dm_verity_prefetch_work *pw = |
694 | container_of(work, struct dm_verity_prefetch_work, work); | |
695 | struct dm_verity *v = pw->v; | |
a4ffc152 MP |
696 | int i; |
697 | ||
698 | for (i = v->levels - 2; i >= 0; i--) { | |
699 | sector_t hash_block_start; | |
700 | sector_t hash_block_end; | |
0ef0b471 | 701 | |
3b6b7813 MP |
702 | verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL); |
703 | verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL); | |
0ef0b471 | 704 | |
a4ffc152 | 705 | if (!i) { |
86a3238c | 706 | unsigned int cluster = READ_ONCE(dm_verity_prefetch_cluster); |
a4ffc152 MP |
707 | |
708 | cluster >>= v->data_dev_block_bits; | |
709 | if (unlikely(!cluster)) | |
710 | goto no_prefetch_cluster; | |
711 | ||
712 | if (unlikely(cluster & (cluster - 1))) | |
553d8fe0 | 713 | cluster = 1 << __fls(cluster); |
a4ffc152 MP |
714 | |
715 | hash_block_start &= ~(sector_t)(cluster - 1); | |
716 | hash_block_end |= cluster - 1; | |
717 | if (unlikely(hash_block_end >= v->hash_blocks)) | |
718 | hash_block_end = v->hash_blocks - 1; | |
719 | } | |
720 | no_prefetch_cluster: | |
721 | dm_bufio_prefetch(v->bufio, hash_block_start, | |
722 | hash_block_end - hash_block_start + 1); | |
723 | } | |
3b6b7813 MP |
724 | |
725 | kfree(pw); | |
726 | } | |
727 | ||
728 | static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io) | |
729 | { | |
0a531c5a | 730 | sector_t block = io->block; |
731 | unsigned int n_blocks = io->n_blocks; | |
3b6b7813 MP |
732 | struct dm_verity_prefetch_work *pw; |
733 | ||
0a531c5a | 734 | if (v->validated_blocks) { |
735 | while (n_blocks && test_bit(block, v->validated_blocks)) { | |
736 | block++; | |
737 | n_blocks--; | |
738 | } | |
739 | while (n_blocks && test_bit(block + n_blocks - 1, | |
740 | v->validated_blocks)) | |
741 | n_blocks--; | |
742 | if (!n_blocks) | |
743 | return; | |
744 | } | |
745 | ||
3b6b7813 MP |
746 | pw = kmalloc(sizeof(struct dm_verity_prefetch_work), |
747 | GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); | |
748 | ||
749 | if (!pw) | |
750 | return; | |
751 | ||
752 | INIT_WORK(&pw->work, verity_prefetch_io); | |
753 | pw->v = v; | |
0a531c5a | 754 | pw->block = block; |
755 | pw->n_blocks = n_blocks; | |
3b6b7813 | 756 | queue_work(v->verify_wq, &pw->work); |
a4ffc152 MP |
757 | } |
758 | ||
759 | /* | |
760 | * Bio map function. It allocates dm_verity_io structure and bio vector and | |
761 | * fills them. Then it issues prefetches and the I/O. | |
762 | */ | |
7de3ee57 | 763 | static int verity_map(struct dm_target *ti, struct bio *bio) |
a4ffc152 MP |
764 | { |
765 | struct dm_verity *v = ti->private; | |
766 | struct dm_verity_io *io; | |
767 | ||
74d46992 | 768 | bio_set_dev(bio, v->data_dev->bdev); |
4f024f37 | 769 | bio->bi_iter.bi_sector = verity_map_sector(v, bio->bi_iter.bi_sector); |
a4ffc152 | 770 | |
86a3238c | 771 | if (((unsigned int)bio->bi_iter.bi_sector | bio_sectors(bio)) & |
a4ffc152 MP |
772 | ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) { |
773 | DMERR_LIMIT("unaligned io"); | |
846785e6 | 774 | return DM_MAPIO_KILL; |
a4ffc152 MP |
775 | } |
776 | ||
f73a1c7d | 777 | if (bio_end_sector(bio) >> |
a4ffc152 MP |
778 | (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) { |
779 | DMERR_LIMIT("io out of range"); | |
846785e6 | 780 | return DM_MAPIO_KILL; |
a4ffc152 MP |
781 | } |
782 | ||
783 | if (bio_data_dir(bio) == WRITE) | |
846785e6 | 784 | return DM_MAPIO_KILL; |
a4ffc152 | 785 | |
30187e1d | 786 | io = dm_per_bio_data(bio, ti->per_io_data_size); |
a4ffc152 | 787 | io->v = v; |
a4ffc152 | 788 | io->orig_bi_end_io = bio->bi_end_io; |
4f024f37 KO |
789 | io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT); |
790 | io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits; | |
a4ffc152 MP |
791 | |
792 | bio->bi_end_io = verity_end_io; | |
793 | bio->bi_private = io; | |
003b5c57 | 794 | io->iter = bio->bi_iter; |
a4ffc152 | 795 | |
7be05bdf WB |
796 | verity_fec_init_io(io); |
797 | ||
3b6b7813 | 798 | verity_submit_prefetch(v, io); |
a4ffc152 | 799 | |
ed00aabd | 800 | submit_bio_noacct(bio); |
a4ffc152 MP |
801 | |
802 | return DM_MAPIO_SUBMITTED; | |
803 | } | |
804 | ||
805 | /* | |
806 | * Status: V (valid) or C (corruption found) | |
807 | */ | |
fd7c092e | 808 | static void verity_status(struct dm_target *ti, status_type_t type, |
86a3238c | 809 | unsigned int status_flags, char *result, unsigned int maxlen) |
a4ffc152 MP |
810 | { |
811 | struct dm_verity *v = ti->private; | |
86a3238c HM |
812 | unsigned int args = 0; |
813 | unsigned int sz = 0; | |
814 | unsigned int x; | |
a4ffc152 MP |
815 | |
816 | switch (type) { | |
817 | case STATUSTYPE_INFO: | |
818 | DMEMIT("%c", v->hash_failed ? 'C' : 'V'); | |
819 | break; | |
820 | case STATUSTYPE_TABLE: | |
821 | DMEMIT("%u %s %s %u %u %llu %llu %s ", | |
822 | v->version, | |
823 | v->data_dev->name, | |
824 | v->hash_dev->name, | |
825 | 1 << v->data_dev_block_bits, | |
826 | 1 << v->hash_dev_block_bits, | |
827 | (unsigned long long)v->data_blocks, | |
828 | (unsigned long long)v->hash_start, | |
829 | v->alg_name | |
830 | ); | |
831 | for (x = 0; x < v->digest_size; x++) | |
832 | DMEMIT("%02x", v->root_digest[x]); | |
833 | DMEMIT(" "); | |
834 | if (!v->salt_size) | |
835 | DMEMIT("-"); | |
836 | else | |
837 | for (x = 0; x < v->salt_size; x++) | |
838 | DMEMIT("%02x", v->salt[x]); | |
a739ff3f ST |
839 | if (v->mode != DM_VERITY_MODE_EIO) |
840 | args++; | |
841 | if (verity_fec_is_enabled(v)) | |
842 | args += DM_VERITY_OPTS_FEC; | |
0cc37c2d ST |
843 | if (v->zero_digest) |
844 | args++; | |
843f38d3 PT |
845 | if (v->validated_blocks) |
846 | args++; | |
5721d4e5 NH |
847 | if (v->use_tasklet) |
848 | args++; | |
88cd3e6c JK |
849 | if (v->signature_key_desc) |
850 | args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS; | |
a739ff3f ST |
851 | if (!args) |
852 | return; | |
853 | DMEMIT(" %u", args); | |
65ff5b7d | 854 | if (v->mode != DM_VERITY_MODE_EIO) { |
a739ff3f | 855 | DMEMIT(" "); |
65ff5b7d ST |
856 | switch (v->mode) { |
857 | case DM_VERITY_MODE_LOGGING: | |
858 | DMEMIT(DM_VERITY_OPT_LOGGING); | |
859 | break; | |
860 | case DM_VERITY_MODE_RESTART: | |
861 | DMEMIT(DM_VERITY_OPT_RESTART); | |
862 | break; | |
e1fef0b0 JL |
863 | case DM_VERITY_MODE_PANIC: |
864 | DMEMIT(DM_VERITY_OPT_PANIC); | |
865 | break; | |
65ff5b7d ST |
866 | default: |
867 | BUG(); | |
868 | } | |
869 | } | |
0cc37c2d ST |
870 | if (v->zero_digest) |
871 | DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES); | |
843f38d3 PT |
872 | if (v->validated_blocks) |
873 | DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE); | |
5721d4e5 NH |
874 | if (v->use_tasklet) |
875 | DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY); | |
a739ff3f | 876 | sz = verity_fec_status_table(v, sz, result, maxlen); |
88cd3e6c JK |
877 | if (v->signature_key_desc) |
878 | DMEMIT(" " DM_VERITY_ROOT_HASH_VERIFICATION_OPT_SIG_KEY | |
879 | " %s", v->signature_key_desc); | |
a4ffc152 | 880 | break; |
8ec45662 TS |
881 | |
882 | case STATUSTYPE_IMA: | |
883 | DMEMIT_TARGET_NAME_VERSION(ti->type); | |
884 | DMEMIT(",hash_failed=%c", v->hash_failed ? 'C' : 'V'); | |
885 | DMEMIT(",verity_version=%u", v->version); | |
886 | DMEMIT(",data_device_name=%s", v->data_dev->name); | |
887 | DMEMIT(",hash_device_name=%s", v->hash_dev->name); | |
888 | DMEMIT(",verity_algorithm=%s", v->alg_name); | |
889 | ||
890 | DMEMIT(",root_digest="); | |
891 | for (x = 0; x < v->digest_size; x++) | |
892 | DMEMIT("%02x", v->root_digest[x]); | |
893 | ||
894 | DMEMIT(",salt="); | |
895 | if (!v->salt_size) | |
896 | DMEMIT("-"); | |
897 | else | |
898 | for (x = 0; x < v->salt_size; x++) | |
899 | DMEMIT("%02x", v->salt[x]); | |
900 | ||
901 | DMEMIT(",ignore_zero_blocks=%c", v->zero_digest ? 'y' : 'n'); | |
902 | DMEMIT(",check_at_most_once=%c", v->validated_blocks ? 'y' : 'n'); | |
33ace4ca TS |
903 | if (v->signature_key_desc) |
904 | DMEMIT(",root_hash_sig_key_desc=%s", v->signature_key_desc); | |
8ec45662 TS |
905 | |
906 | if (v->mode != DM_VERITY_MODE_EIO) { | |
907 | DMEMIT(",verity_mode="); | |
908 | switch (v->mode) { | |
909 | case DM_VERITY_MODE_LOGGING: | |
910 | DMEMIT(DM_VERITY_OPT_LOGGING); | |
911 | break; | |
912 | case DM_VERITY_MODE_RESTART: | |
913 | DMEMIT(DM_VERITY_OPT_RESTART); | |
914 | break; | |
915 | case DM_VERITY_MODE_PANIC: | |
916 | DMEMIT(DM_VERITY_OPT_PANIC); | |
917 | break; | |
918 | default: | |
919 | DMEMIT("invalid"); | |
920 | } | |
921 | } | |
922 | DMEMIT(";"); | |
923 | break; | |
a4ffc152 | 924 | } |
a4ffc152 MP |
925 | } |
926 | ||
5bd5e8d8 | 927 | static int verity_prepare_ioctl(struct dm_target *ti, struct block_device **bdev) |
a4ffc152 MP |
928 | { |
929 | struct dm_verity *v = ti->private; | |
e56f81e0 CH |
930 | |
931 | *bdev = v->data_dev->bdev; | |
a4ffc152 | 932 | |
6dcbb52c | 933 | if (v->data_start || ti->len != bdev_nr_sectors(v->data_dev->bdev)) |
e56f81e0 CH |
934 | return 1; |
935 | return 0; | |
a4ffc152 MP |
936 | } |
937 | ||
a4ffc152 MP |
938 | static int verity_iterate_devices(struct dm_target *ti, |
939 | iterate_devices_callout_fn fn, void *data) | |
940 | { | |
941 | struct dm_verity *v = ti->private; | |
942 | ||
943 | return fn(ti, v->data_dev, v->data_start, ti->len, data); | |
944 | } | |
945 | ||
946 | static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) | |
947 | { | |
948 | struct dm_verity *v = ti->private; | |
949 | ||
950 | if (limits->logical_block_size < 1 << v->data_dev_block_bits) | |
951 | limits->logical_block_size = 1 << v->data_dev_block_bits; | |
952 | ||
953 | if (limits->physical_block_size < 1 << v->data_dev_block_bits) | |
954 | limits->physical_block_size = 1 << v->data_dev_block_bits; | |
955 | ||
956 | blk_limits_io_min(limits, limits->logical_block_size); | |
957 | } | |
958 | ||
959 | static void verity_dtr(struct dm_target *ti) | |
960 | { | |
961 | struct dm_verity *v = ti->private; | |
962 | ||
963 | if (v->verify_wq) | |
964 | destroy_workqueue(v->verify_wq); | |
965 | ||
a4ffc152 MP |
966 | if (v->bufio) |
967 | dm_bufio_client_destroy(v->bufio); | |
968 | ||
843f38d3 | 969 | kvfree(v->validated_blocks); |
a4ffc152 MP |
970 | kfree(v->salt); |
971 | kfree(v->root_digest); | |
0cc37c2d | 972 | kfree(v->zero_digest); |
a4ffc152 MP |
973 | |
974 | if (v->tfm) | |
d1ac3ff0 | 975 | crypto_free_ahash(v->tfm); |
a4ffc152 MP |
976 | |
977 | kfree(v->alg_name); | |
978 | ||
979 | if (v->hash_dev) | |
980 | dm_put_device(ti, v->hash_dev); | |
981 | ||
982 | if (v->data_dev) | |
983 | dm_put_device(ti, v->data_dev); | |
984 | ||
a739ff3f ST |
985 | verity_fec_dtr(v); |
986 | ||
88cd3e6c JK |
987 | kfree(v->signature_key_desc); |
988 | ||
ba2cce82 MS |
989 | if (v->use_tasklet) |
990 | static_branch_dec(&use_tasklet_enabled); | |
991 | ||
a4ffc152 | 992 | kfree(v); |
074c4466 MW |
993 | |
994 | dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1); | |
a4ffc152 MP |
995 | } |
996 | ||
843f38d3 PT |
997 | static int verity_alloc_most_once(struct dm_verity *v) |
998 | { | |
999 | struct dm_target *ti = v->ti; | |
1000 | ||
1001 | /* the bitset can only handle INT_MAX blocks */ | |
1002 | if (v->data_blocks > INT_MAX) { | |
1003 | ti->error = "device too large to use check_at_most_once"; | |
1004 | return -E2BIG; | |
1005 | } | |
1006 | ||
778e1cdd KC |
1007 | v->validated_blocks = kvcalloc(BITS_TO_LONGS(v->data_blocks), |
1008 | sizeof(unsigned long), | |
1009 | GFP_KERNEL); | |
843f38d3 PT |
1010 | if (!v->validated_blocks) { |
1011 | ti->error = "failed to allocate bitset for check_at_most_once"; | |
1012 | return -ENOMEM; | |
1013 | } | |
1014 | ||
1015 | return 0; | |
1016 | } | |
1017 | ||
0cc37c2d ST |
1018 | static int verity_alloc_zero_digest(struct dm_verity *v) |
1019 | { | |
1020 | int r = -ENOMEM; | |
d1ac3ff0 | 1021 | struct ahash_request *req; |
0cc37c2d ST |
1022 | u8 *zero_data; |
1023 | ||
1024 | v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL); | |
1025 | ||
1026 | if (!v->zero_digest) | |
1027 | return r; | |
1028 | ||
d1ac3ff0 | 1029 | req = kmalloc(v->ahash_reqsize, GFP_KERNEL); |
0cc37c2d | 1030 | |
d1ac3ff0 | 1031 | if (!req) |
0cc37c2d ST |
1032 | return r; /* verity_dtr will free zero_digest */ |
1033 | ||
1034 | zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL); | |
1035 | ||
1036 | if (!zero_data) | |
1037 | goto out; | |
1038 | ||
d1ac3ff0 | 1039 | r = verity_hash(v, req, zero_data, 1 << v->data_dev_block_bits, |
28f07f2a | 1040 | v->zero_digest, true); |
0cc37c2d ST |
1041 | |
1042 | out: | |
d1ac3ff0 | 1043 | kfree(req); |
0cc37c2d ST |
1044 | kfree(zero_data); |
1045 | ||
1046 | return r; | |
1047 | } | |
1048 | ||
219a9b5e JL |
1049 | static inline bool verity_is_verity_mode(const char *arg_name) |
1050 | { | |
1051 | return (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING) || | |
1052 | !strcasecmp(arg_name, DM_VERITY_OPT_RESTART) || | |
1053 | !strcasecmp(arg_name, DM_VERITY_OPT_PANIC)); | |
1054 | } | |
1055 | ||
1056 | static int verity_parse_verity_mode(struct dm_verity *v, const char *arg_name) | |
1057 | { | |
1058 | if (v->mode) | |
1059 | return -EINVAL; | |
1060 | ||
1061 | if (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING)) | |
1062 | v->mode = DM_VERITY_MODE_LOGGING; | |
1063 | else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART)) | |
1064 | v->mode = DM_VERITY_MODE_RESTART; | |
1065 | else if (!strcasecmp(arg_name, DM_VERITY_OPT_PANIC)) | |
1066 | v->mode = DM_VERITY_MODE_PANIC; | |
1067 | ||
1068 | return 0; | |
1069 | } | |
1070 | ||
88cd3e6c | 1071 | static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, |
df326e7a MS |
1072 | struct dm_verity_sig_opts *verify_args, |
1073 | bool only_modifier_opts) | |
753c1fd0 | 1074 | { |
f876df9f | 1075 | int r = 0; |
86a3238c | 1076 | unsigned int argc; |
753c1fd0 ST |
1077 | struct dm_target *ti = v->ti; |
1078 | const char *arg_name; | |
1079 | ||
5916a22b | 1080 | static const struct dm_arg _args[] = { |
753c1fd0 ST |
1081 | {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"}, |
1082 | }; | |
1083 | ||
1084 | r = dm_read_arg_group(_args, as, &argc, &ti->error); | |
1085 | if (r) | |
1086 | return -EINVAL; | |
1087 | ||
1088 | if (!argc) | |
1089 | return 0; | |
1090 | ||
1091 | do { | |
1092 | arg_name = dm_shift_arg(as); | |
1093 | argc--; | |
1094 | ||
219a9b5e | 1095 | if (verity_is_verity_mode(arg_name)) { |
df326e7a MS |
1096 | if (only_modifier_opts) |
1097 | continue; | |
219a9b5e JL |
1098 | r = verity_parse_verity_mode(v, arg_name); |
1099 | if (r) { | |
1100 | ti->error = "Conflicting error handling parameters"; | |
1101 | return r; | |
1102 | } | |
e1fef0b0 JL |
1103 | continue; |
1104 | ||
0cc37c2d | 1105 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_IGN_ZEROES)) { |
df326e7a MS |
1106 | if (only_modifier_opts) |
1107 | continue; | |
0cc37c2d ST |
1108 | r = verity_alloc_zero_digest(v); |
1109 | if (r) { | |
1110 | ti->error = "Cannot allocate zero digest"; | |
1111 | return r; | |
1112 | } | |
1113 | continue; | |
1114 | ||
843f38d3 | 1115 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_AT_MOST_ONCE)) { |
df326e7a MS |
1116 | if (only_modifier_opts) |
1117 | continue; | |
843f38d3 PT |
1118 | r = verity_alloc_most_once(v); |
1119 | if (r) | |
1120 | return r; | |
1121 | continue; | |
1122 | ||
5721d4e5 NH |
1123 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) { |
1124 | v->use_tasklet = true; | |
ba2cce82 | 1125 | static_branch_inc(&use_tasklet_enabled); |
5721d4e5 NH |
1126 | continue; |
1127 | ||
a739ff3f | 1128 | } else if (verity_is_fec_opt_arg(arg_name)) { |
df326e7a MS |
1129 | if (only_modifier_opts) |
1130 | continue; | |
a739ff3f ST |
1131 | r = verity_fec_parse_opt_args(as, v, &argc, arg_name); |
1132 | if (r) | |
1133 | return r; | |
1134 | continue; | |
5721d4e5 | 1135 | |
88cd3e6c | 1136 | } else if (verity_verify_is_sig_opt_arg(arg_name)) { |
df326e7a MS |
1137 | if (only_modifier_opts) |
1138 | continue; | |
88cd3e6c JK |
1139 | r = verity_verify_sig_parse_opt_args(as, v, |
1140 | verify_args, | |
1141 | &argc, arg_name); | |
1142 | if (r) | |
1143 | return r; | |
1144 | continue; | |
f876df9f MS |
1145 | |
1146 | } else if (only_modifier_opts) { | |
1147 | /* | |
1148 | * Ignore unrecognized opt, could easily be an extra | |
1149 | * argument to an option whose parsing was skipped. | |
1150 | * Normal parsing (@only_modifier_opts=false) will | |
1151 | * properly parse all options (and their extra args). | |
1152 | */ | |
1153 | continue; | |
753c1fd0 ST |
1154 | } |
1155 | ||
f876df9f | 1156 | DMERR("Unrecognized verity feature request: %s", arg_name); |
753c1fd0 ST |
1157 | ti->error = "Unrecognized verity feature request"; |
1158 | return -EINVAL; | |
1159 | } while (argc && !r); | |
1160 | ||
1161 | return r; | |
1162 | } | |
1163 | ||
a4ffc152 MP |
1164 | /* |
1165 | * Target parameters: | |
1166 | * <version> The current format is version 1. | |
1167 | * Vsn 0 is compatible with original Chromium OS releases. | |
1168 | * <data device> | |
1169 | * <hash device> | |
1170 | * <data block size> | |
1171 | * <hash block size> | |
1172 | * <the number of data blocks> | |
1173 | * <hash start block> | |
1174 | * <algorithm> | |
1175 | * <digest> | |
1176 | * <salt> Hex string or "-" if no salt. | |
1177 | */ | |
86a3238c | 1178 | static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) |
a4ffc152 MP |
1179 | { |
1180 | struct dm_verity *v; | |
88cd3e6c | 1181 | struct dm_verity_sig_opts verify_args = {0}; |
65ff5b7d | 1182 | struct dm_arg_set as; |
753c1fd0 | 1183 | unsigned int num; |
a4ffc152 MP |
1184 | unsigned long long num_ll; |
1185 | int r; | |
1186 | int i; | |
1187 | sector_t hash_position; | |
1188 | char dummy; | |
88cd3e6c | 1189 | char *root_hash_digest_to_validate; |
a4ffc152 MP |
1190 | |
1191 | v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL); | |
1192 | if (!v) { | |
1193 | ti->error = "Cannot allocate verity structure"; | |
1194 | return -ENOMEM; | |
1195 | } | |
1196 | ti->private = v; | |
1197 | v->ti = ti; | |
1198 | ||
a739ff3f ST |
1199 | r = verity_fec_ctr_alloc(v); |
1200 | if (r) | |
1201 | goto bad; | |
1202 | ||
05bdb996 | 1203 | if ((dm_table_get_mode(ti->table) & ~BLK_OPEN_READ)) { |
a4ffc152 MP |
1204 | ti->error = "Device must be readonly"; |
1205 | r = -EINVAL; | |
1206 | goto bad; | |
1207 | } | |
1208 | ||
65ff5b7d ST |
1209 | if (argc < 10) { |
1210 | ti->error = "Not enough arguments"; | |
a4ffc152 MP |
1211 | r = -EINVAL; |
1212 | goto bad; | |
1213 | } | |
1214 | ||
df326e7a MS |
1215 | /* Parse optional parameters that modify primary args */ |
1216 | if (argc > 10) { | |
1217 | as.argc = argc - 10; | |
1218 | as.argv = argv + 10; | |
1219 | r = verity_parse_opt_args(&as, v, &verify_args, true); | |
1220 | if (r < 0) | |
1221 | goto bad; | |
1222 | } | |
1223 | ||
5d8be843 MP |
1224 | if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 || |
1225 | num > 1) { | |
a4ffc152 MP |
1226 | ti->error = "Invalid version"; |
1227 | r = -EINVAL; | |
1228 | goto bad; | |
1229 | } | |
1230 | v->version = num; | |
1231 | ||
05bdb996 | 1232 | r = dm_get_device(ti, argv[1], BLK_OPEN_READ, &v->data_dev); |
a4ffc152 MP |
1233 | if (r) { |
1234 | ti->error = "Data device lookup failed"; | |
1235 | goto bad; | |
1236 | } | |
1237 | ||
05bdb996 | 1238 | r = dm_get_device(ti, argv[2], BLK_OPEN_READ, &v->hash_dev); |
a4ffc152 | 1239 | if (r) { |
21ffe552 | 1240 | ti->error = "Hash device lookup failed"; |
a4ffc152 MP |
1241 | goto bad; |
1242 | } | |
1243 | ||
1244 | if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 || | |
1245 | !num || (num & (num - 1)) || | |
1246 | num < bdev_logical_block_size(v->data_dev->bdev) || | |
1247 | num > PAGE_SIZE) { | |
1248 | ti->error = "Invalid data device block size"; | |
1249 | r = -EINVAL; | |
1250 | goto bad; | |
1251 | } | |
553d8fe0 | 1252 | v->data_dev_block_bits = __ffs(num); |
a4ffc152 MP |
1253 | |
1254 | if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 || | |
1255 | !num || (num & (num - 1)) || | |
1256 | num < bdev_logical_block_size(v->hash_dev->bdev) || | |
1257 | num > INT_MAX) { | |
1258 | ti->error = "Invalid hash device block size"; | |
1259 | r = -EINVAL; | |
1260 | goto bad; | |
1261 | } | |
553d8fe0 | 1262 | v->hash_dev_block_bits = __ffs(num); |
a4ffc152 MP |
1263 | |
1264 | if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 || | |
1d55f6bc MP |
1265 | (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) |
1266 | >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) { | |
a4ffc152 MP |
1267 | ti->error = "Invalid data blocks"; |
1268 | r = -EINVAL; | |
1269 | goto bad; | |
1270 | } | |
1271 | v->data_blocks = num_ll; | |
1272 | ||
1273 | if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) { | |
1274 | ti->error = "Data device is too small"; | |
1275 | r = -EINVAL; | |
1276 | goto bad; | |
1277 | } | |
1278 | ||
1279 | if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 || | |
1d55f6bc MP |
1280 | (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT)) |
1281 | >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) { | |
a4ffc152 MP |
1282 | ti->error = "Invalid hash start"; |
1283 | r = -EINVAL; | |
1284 | goto bad; | |
1285 | } | |
1286 | v->hash_start = num_ll; | |
1287 | ||
1288 | v->alg_name = kstrdup(argv[7], GFP_KERNEL); | |
1289 | if (!v->alg_name) { | |
1290 | ti->error = "Cannot allocate algorithm name"; | |
1291 | r = -ENOMEM; | |
1292 | goto bad; | |
1293 | } | |
1294 | ||
df326e7a MS |
1295 | v->tfm = crypto_alloc_ahash(v->alg_name, 0, |
1296 | v->use_tasklet ? CRYPTO_ALG_ASYNC : 0); | |
a4ffc152 MP |
1297 | if (IS_ERR(v->tfm)) { |
1298 | ti->error = "Cannot initialize hash function"; | |
1299 | r = PTR_ERR(v->tfm); | |
1300 | v->tfm = NULL; | |
1301 | goto bad; | |
1302 | } | |
bbf6a566 EB |
1303 | |
1304 | /* | |
1305 | * dm-verity performance can vary greatly depending on which hash | |
1306 | * algorithm implementation is used. Help people debug performance | |
1307 | * problems by logging the ->cra_driver_name. | |
1308 | */ | |
1309 | DMINFO("%s using implementation \"%s\"", v->alg_name, | |
1310 | crypto_hash_alg_common(v->tfm)->base.cra_driver_name); | |
1311 | ||
d1ac3ff0 | 1312 | v->digest_size = crypto_ahash_digestsize(v->tfm); |
a4ffc152 MP |
1313 | if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) { |
1314 | ti->error = "Digest size too big"; | |
1315 | r = -EINVAL; | |
1316 | goto bad; | |
1317 | } | |
d1ac3ff0 GBY |
1318 | v->ahash_reqsize = sizeof(struct ahash_request) + |
1319 | crypto_ahash_reqsize(v->tfm); | |
a4ffc152 MP |
1320 | |
1321 | v->root_digest = kmalloc(v->digest_size, GFP_KERNEL); | |
1322 | if (!v->root_digest) { | |
1323 | ti->error = "Cannot allocate root digest"; | |
1324 | r = -ENOMEM; | |
1325 | goto bad; | |
1326 | } | |
1327 | if (strlen(argv[8]) != v->digest_size * 2 || | |
1328 | hex2bin(v->root_digest, argv[8], v->digest_size)) { | |
1329 | ti->error = "Invalid root digest"; | |
1330 | r = -EINVAL; | |
1331 | goto bad; | |
1332 | } | |
88cd3e6c | 1333 | root_hash_digest_to_validate = argv[8]; |
a4ffc152 MP |
1334 | |
1335 | if (strcmp(argv[9], "-")) { | |
1336 | v->salt_size = strlen(argv[9]) / 2; | |
1337 | v->salt = kmalloc(v->salt_size, GFP_KERNEL); | |
1338 | if (!v->salt) { | |
1339 | ti->error = "Cannot allocate salt"; | |
1340 | r = -ENOMEM; | |
1341 | goto bad; | |
1342 | } | |
1343 | if (strlen(argv[9]) != v->salt_size * 2 || | |
1344 | hex2bin(v->salt, argv[9], v->salt_size)) { | |
1345 | ti->error = "Invalid salt"; | |
1346 | r = -EINVAL; | |
1347 | goto bad; | |
1348 | } | |
1349 | } | |
1350 | ||
65ff5b7d ST |
1351 | argv += 10; |
1352 | argc -= 10; | |
1353 | ||
1354 | /* Optional parameters */ | |
1355 | if (argc) { | |
1356 | as.argc = argc; | |
1357 | as.argv = argv; | |
df326e7a | 1358 | r = verity_parse_opt_args(&as, v, &verify_args, false); |
753c1fd0 | 1359 | if (r < 0) |
65ff5b7d | 1360 | goto bad; |
65ff5b7d ST |
1361 | } |
1362 | ||
88cd3e6c JK |
1363 | /* Root hash signature is a optional parameter*/ |
1364 | r = verity_verify_root_hash(root_hash_digest_to_validate, | |
1365 | strlen(root_hash_digest_to_validate), | |
1366 | verify_args.sig, | |
1367 | verify_args.sig_size); | |
1368 | if (r < 0) { | |
1369 | ti->error = "Root hash verification failed"; | |
1370 | goto bad; | |
1371 | } | |
a4ffc152 | 1372 | v->hash_per_block_bits = |
553d8fe0 | 1373 | __fls((1 << v->hash_dev_block_bits) / v->digest_size); |
a4ffc152 MP |
1374 | |
1375 | v->levels = 0; | |
1376 | if (v->data_blocks) | |
1377 | while (v->hash_per_block_bits * v->levels < 64 && | |
1378 | (unsigned long long)(v->data_blocks - 1) >> | |
1379 | (v->hash_per_block_bits * v->levels)) | |
1380 | v->levels++; | |
1381 | ||
1382 | if (v->levels > DM_VERITY_MAX_LEVELS) { | |
1383 | ti->error = "Too many tree levels"; | |
1384 | r = -E2BIG; | |
1385 | goto bad; | |
1386 | } | |
1387 | ||
1388 | hash_position = v->hash_start; | |
1389 | for (i = v->levels - 1; i >= 0; i--) { | |
1390 | sector_t s; | |
0ef0b471 | 1391 | |
a4ffc152 | 1392 | v->hash_level_block[i] = hash_position; |
b1bf2de0 MP |
1393 | s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1) |
1394 | >> ((i + 1) * v->hash_per_block_bits); | |
a4ffc152 MP |
1395 | if (hash_position + s < hash_position) { |
1396 | ti->error = "Hash device offset overflow"; | |
1397 | r = -E2BIG; | |
1398 | goto bad; | |
1399 | } | |
1400 | hash_position += s; | |
1401 | } | |
1402 | v->hash_blocks = hash_position; | |
1403 | ||
1404 | v->bufio = dm_bufio_client_create(v->hash_dev->bdev, | |
1405 | 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux), | |
5721d4e5 NH |
1406 | dm_bufio_alloc_callback, NULL, |
1407 | v->use_tasklet ? DM_BUFIO_CLIENT_NO_SLEEP : 0); | |
a4ffc152 MP |
1408 | if (IS_ERR(v->bufio)) { |
1409 | ti->error = "Cannot initialize dm-bufio"; | |
1410 | r = PTR_ERR(v->bufio); | |
1411 | v->bufio = NULL; | |
1412 | goto bad; | |
1413 | } | |
1414 | ||
1415 | if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) { | |
1416 | ti->error = "Hash device is too small"; | |
1417 | r = -E2BIG; | |
1418 | goto bad; | |
1419 | } | |
1420 | ||
afd41fff NH |
1421 | /* |
1422 | * Using WQ_HIGHPRI improves throughput and completion latency by | |
1423 | * reducing wait times when reading from a dm-verity device. | |
1424 | * | |
1425 | * Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI | |
1426 | * allows verify_wq to preempt softirq since verification in tasklet | |
1427 | * will fall-back to using it for error handling (or if the bufio cache | |
1428 | * doesn't have required hashes). | |
1429 | */ | |
c25da5b7 | 1430 | v->verify_wq = alloc_workqueue("kverityd", WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); |
a4ffc152 MP |
1431 | if (!v->verify_wq) { |
1432 | ti->error = "Cannot allocate workqueue"; | |
1433 | r = -ENOMEM; | |
1434 | goto bad; | |
1435 | } | |
1436 | ||
30187e1d | 1437 | ti->per_io_data_size = sizeof(struct dm_verity_io) + |
d1ac3ff0 | 1438 | v->ahash_reqsize + v->digest_size * 2; |
a739ff3f ST |
1439 | |
1440 | r = verity_fec_ctr(v); | |
1441 | if (r) | |
1442 | goto bad; | |
1443 | ||
30187e1d MS |
1444 | ti->per_io_data_size = roundup(ti->per_io_data_size, |
1445 | __alignof__(struct dm_verity_io)); | |
a739ff3f | 1446 | |
88cd3e6c JK |
1447 | verity_verify_sig_opts_cleanup(&verify_args); |
1448 | ||
074c4466 MW |
1449 | dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1); |
1450 | ||
a4ffc152 MP |
1451 | return 0; |
1452 | ||
1453 | bad: | |
88cd3e6c JK |
1454 | |
1455 | verity_verify_sig_opts_cleanup(&verify_args); | |
074c4466 | 1456 | dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0); |
a4ffc152 MP |
1457 | verity_dtr(ti); |
1458 | ||
1459 | return r; | |
1460 | } | |
1461 | ||
b6c1c574 MK |
1462 | /* |
1463 | * Check whether a DM target is a verity target. | |
1464 | */ | |
1465 | bool dm_is_verity_target(struct dm_target *ti) | |
1466 | { | |
1467 | return ti->type->module == THIS_MODULE; | |
1468 | } | |
1469 | ||
916ef623 MK |
1470 | /* |
1471 | * Get the verity mode (error behavior) of a verity target. | |
1472 | * | |
1473 | * Returns the verity mode of the target, or -EINVAL if 'ti' is not a verity | |
1474 | * target. | |
1475 | */ | |
1476 | int dm_verity_get_mode(struct dm_target *ti) | |
1477 | { | |
1478 | struct dm_verity *v = ti->private; | |
1479 | ||
1480 | if (!dm_is_verity_target(ti)) | |
1481 | return -EINVAL; | |
1482 | ||
1483 | return v->mode; | |
1484 | } | |
1485 | ||
b6c1c574 MK |
1486 | /* |
1487 | * Get the root digest of a verity target. | |
1488 | * | |
1489 | * Returns a copy of the root digest, the caller is responsible for | |
1490 | * freeing the memory of the digest. | |
1491 | */ | |
1492 | int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned int *digest_size) | |
1493 | { | |
1494 | struct dm_verity *v = ti->private; | |
1495 | ||
1496 | if (!dm_is_verity_target(ti)) | |
1497 | return -EINVAL; | |
1498 | ||
1499 | *root_digest = kmemdup(v->root_digest, v->digest_size, GFP_KERNEL); | |
1500 | if (*root_digest == NULL) | |
1501 | return -ENOMEM; | |
1502 | ||
1503 | *digest_size = v->digest_size; | |
1504 | ||
1505 | return 0; | |
1506 | } | |
1507 | ||
a4ffc152 MP |
1508 | static struct target_type verity_target = { |
1509 | .name = "verity", | |
4caae584 | 1510 | .features = DM_TARGET_IMMUTABLE, |
5721d4e5 | 1511 | .version = {1, 9, 0}, |
a4ffc152 MP |
1512 | .module = THIS_MODULE, |
1513 | .ctr = verity_ctr, | |
1514 | .dtr = verity_dtr, | |
1515 | .map = verity_map, | |
1516 | .status = verity_status, | |
e56f81e0 | 1517 | .prepare_ioctl = verity_prepare_ioctl, |
a4ffc152 MP |
1518 | .iterate_devices = verity_iterate_devices, |
1519 | .io_hints = verity_io_hints, | |
1520 | }; | |
3664ff82 | 1521 | module_dm(verity); |
a4ffc152 MP |
1522 | |
1523 | MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>"); | |
1524 | MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>"); | |
1525 | MODULE_AUTHOR("Will Drewry <wad@chromium.org>"); | |
1526 | MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking"); | |
1527 | MODULE_LICENSE("GPL"); |