diff options
Diffstat (limited to 'drivers/md/dm-verity-target.c')
-rw-r--r-- | drivers/md/dm-verity-target.c | 159 |
1 files changed, 119 insertions, 40 deletions
diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 14e58ae70521..bb5da66da4c1 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -46,11 +46,12 @@ static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644); -static DEFINE_STATIC_KEY_FALSE(use_tasklet_enabled); +static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled); struct dm_verity_prefetch_work { struct work_struct work; struct dm_verity *v; + unsigned short ioprio; sector_t block; unsigned int n_blocks; }; @@ -294,10 +295,11 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, int r; sector_t hash_block; unsigned int offset; + struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); verity_hash_at_level(v, block, level, &hash_block, &offset); - if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) { + if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { data = dm_bufio_get(v->bufio, hash_block, &buf); if (data == NULL) { /* @@ -307,8 +309,10 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, */ return -EAGAIN; } - } else - data = dm_bufio_read(v->bufio, hash_block, &buf); + } else { + data = dm_bufio_read_with_ioprio(v->bufio, hash_block, + &buf, bio_prio(bio)); + } if (IS_ERR(data)) return PTR_ERR(data); @@ -323,15 +327,14 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, r = verity_hash(v, verity_io_hash_req(v, io), data, 1 << v->hash_dev_block_bits, - verity_io_real_digest(v, io), !io->in_tasklet); + verity_io_real_digest(v, io), !io->in_bh); if (unlikely(r < 0)) goto release_ret_r; if (likely(memcmp(verity_io_real_digest(v, io), want_digest, v->digest_size) == 0)) aux->hash_verified = 1; - else if (static_branch_unlikely(&use_tasklet_enabled) && - io->in_tasklet) { + else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { /* * Error handling code (FEC included) cannot be run in a * tasklet since it may sleep, so fallback to work-queue. @@ -482,6 +485,63 @@ int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io, return 0; } +static int verity_recheck_copy(struct dm_verity *v, struct dm_verity_io *io, + u8 *data, size_t len) +{ + memcpy(data, io->recheck_buffer, len); + io->recheck_buffer += len; + + return 0; +} + +static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io, + struct bvec_iter start, sector_t cur_block) +{ + struct page *page; + void *buffer; + int r; + struct dm_io_request io_req; + struct dm_io_region io_loc; + + page = mempool_alloc(&v->recheck_pool, GFP_NOIO); + buffer = page_to_virt(page); + + io_req.bi_opf = REQ_OP_READ; + io_req.mem.type = DM_IO_KMEM; + io_req.mem.ptr.addr = buffer; + io_req.notify.fn = NULL; + io_req.client = v->io; + io_loc.bdev = v->data_dev->bdev; + io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT); + io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + if (unlikely(r)) + goto free_ret; + + r = verity_hash(v, verity_io_hash_req(v, io), buffer, + 1 << v->data_dev_block_bits, + verity_io_real_digest(v, io), true); + if (unlikely(r)) + goto free_ret; + + if (memcmp(verity_io_real_digest(v, io), + verity_io_want_digest(v, io), v->digest_size)) { + r = -EIO; + goto free_ret; + } + + io->recheck_buffer = buffer; + r = verity_for_bv_block(v, io, &start, verity_recheck_copy); + if (unlikely(r)) + goto free_ret; + + r = 0; +free_ret: + mempool_free(page, &v->recheck_pool); + + return r; +} + static int verity_bv_zero(struct dm_verity *v, struct dm_verity_io *io, u8 *data, size_t len) { @@ -508,16 +568,14 @@ static int verity_verify_io(struct dm_verity_io *io) { bool is_zero; struct dm_verity *v = io->v; -#if defined(CONFIG_DM_VERITY_FEC) struct bvec_iter start; -#endif struct bvec_iter iter_copy; struct bvec_iter *iter; struct crypto_wait wait; struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); unsigned int b; - if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) { + if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { /* * Copy the iterator in case we need to restart * verification in a work-queue. @@ -557,14 +615,11 @@ static int verity_verify_io(struct dm_verity_io *io) continue; } - r = verity_hash_init(v, req, &wait, !io->in_tasklet); + r = verity_hash_init(v, req, &wait, !io->in_bh); if (unlikely(r < 0)) return r; -#if defined(CONFIG_DM_VERITY_FEC) - if (verity_fec_is_enabled(v)) - start = *iter; -#endif + start = *iter; r = verity_for_io_block(v, io, iter, &wait); if (unlikely(r < 0)) return r; @@ -579,13 +634,16 @@ static int verity_verify_io(struct dm_verity_io *io) if (v->validated_blocks) set_bit(cur_block, v->validated_blocks); continue; - } else if (static_branch_unlikely(&use_tasklet_enabled) && - io->in_tasklet) { + } else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { /* * Error handling code (FEC included) cannot be run in a * tasklet since it may sleep, so fallback to work-queue. */ return -EAGAIN; + } else if (verity_recheck(v, io, start, cur_block) == 0) { + if (v->validated_blocks) + set_bit(cur_block, v->validated_blocks); + continue; #if defined(CONFIG_DM_VERITY_FEC) } else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA, cur_block, NULL, &start) == 0) { @@ -630,7 +688,7 @@ static void verity_finish_io(struct dm_verity_io *io, blk_status_t status) bio->bi_end_io = io->orig_bi_end_io; bio->bi_status = status; - if (!static_branch_unlikely(&use_tasklet_enabled) || !io->in_tasklet) + if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh) verity_fec_finish_io(io); bio_endio(bio); @@ -640,17 +698,17 @@ static void verity_work(struct work_struct *w) { struct dm_verity_io *io = container_of(w, struct dm_verity_io, work); - io->in_tasklet = false; + io->in_bh = false; verity_finish_io(io, errno_to_blk_status(verity_verify_io(io))); } -static void verity_tasklet(unsigned long data) +static void verity_bh_work(struct work_struct *w) { - struct dm_verity_io *io = (struct dm_verity_io *)data; + struct dm_verity_io *io = container_of(w, struct dm_verity_io, bh_work); int err; - io->in_tasklet = true; + io->in_bh = true; err = verity_verify_io(io); if (err == -EAGAIN || err == -ENOMEM) { /* fallback to retrying with work-queue */ @@ -674,9 +732,9 @@ static void verity_end_io(struct bio *bio) return; } - if (static_branch_unlikely(&use_tasklet_enabled) && io->v->use_tasklet) { - tasklet_init(&io->tasklet, verity_tasklet, (unsigned long)io); - tasklet_schedule(&io->tasklet); + if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq) { + INIT_WORK(&io->bh_work, verity_bh_work); + queue_work(system_bh_wq, &io->bh_work); } else { INIT_WORK(&io->work, verity_work); queue_work(io->v->verify_wq, &io->work); @@ -718,14 +776,16 @@ static void verity_prefetch_io(struct work_struct *work) hash_block_end = v->hash_blocks - 1; } no_prefetch_cluster: - dm_bufio_prefetch(v->bufio, hash_block_start, - hash_block_end - hash_block_start + 1); + dm_bufio_prefetch_with_ioprio(v->bufio, hash_block_start, + hash_block_end - hash_block_start + 1, + pw->ioprio); } kfree(pw); } -static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io) +static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io, + unsigned short ioprio) { sector_t block = io->block; unsigned int n_blocks = io->n_blocks; @@ -753,6 +813,7 @@ static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io) pw->v = v; pw->block = block; pw->n_blocks = n_blocks; + pw->ioprio = ioprio; queue_work(v->verify_wq, &pw->work); } @@ -795,7 +856,7 @@ static int verity_map(struct dm_target *ti, struct bio *bio) verity_fec_init_io(io); - verity_submit_prefetch(v, io); + verity_submit_prefetch(v, io, bio_prio(bio)); submit_bio_noacct(bio); @@ -844,7 +905,7 @@ static void verity_status(struct dm_target *ti, status_type_t type, args++; if (v->validated_blocks) args++; - if (v->use_tasklet) + if (v->use_bh_wq) args++; if (v->signature_key_desc) args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS; @@ -871,7 +932,7 @@ static void verity_status(struct dm_target *ti, status_type_t type, DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES); if (v->validated_blocks) DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE); - if (v->use_tasklet) + if (v->use_bh_wq) DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY); sz = verity_fec_status_table(v, sz, result, maxlen); if (v->signature_key_desc) @@ -963,6 +1024,10 @@ static void verity_dtr(struct dm_target *ti) if (v->verify_wq) destroy_workqueue(v->verify_wq); + mempool_exit(&v->recheck_pool); + if (v->io) + dm_io_client_destroy(v->io); + if (v->bufio) dm_bufio_client_destroy(v->bufio); @@ -986,8 +1051,8 @@ static void verity_dtr(struct dm_target *ti) kfree(v->signature_key_desc); - if (v->use_tasklet) - static_branch_dec(&use_tasklet_enabled); + if (v->use_bh_wq) + static_branch_dec(&use_bh_wq_enabled); kfree(v); @@ -1121,8 +1186,8 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, continue; } else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) { - v->use_tasklet = true; - static_branch_inc(&use_tasklet_enabled); + v->use_bh_wq = true; + static_branch_inc(&use_bh_wq_enabled); continue; } else if (verity_is_fec_opt_arg(arg_name)) { @@ -1293,7 +1358,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) } v->tfm = crypto_alloc_ahash(v->alg_name, 0, - v->use_tasklet ? CRYPTO_ALG_ASYNC : 0); + v->use_bh_wq ? CRYPTO_ALG_ASYNC : 0); if (IS_ERR(v->tfm)) { ti->error = "Cannot initialize hash function"; r = PTR_ERR(v->tfm); @@ -1401,10 +1466,24 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) } v->hash_blocks = hash_position; + r = mempool_init_page_pool(&v->recheck_pool, 1, 0); + if (unlikely(r)) { + ti->error = "Cannot allocate mempool"; + goto bad; + } + + v->io = dm_io_client_create(); + if (IS_ERR(v->io)) { + r = PTR_ERR(v->io); + v->io = NULL; + ti->error = "Cannot allocate dm io"; + goto bad; + } + v->bufio = dm_bufio_client_create(v->hash_dev->bdev, 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux), dm_bufio_alloc_callback, NULL, - v->use_tasklet ? DM_BUFIO_CLIENT_NO_SLEEP : 0); + v->use_bh_wq ? DM_BUFIO_CLIENT_NO_SLEEP : 0); if (IS_ERR(v->bufio)) { ti->error = "Cannot initialize dm-bufio"; r = PTR_ERR(v->bufio); @@ -1423,7 +1502,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) * reducing wait times when reading from a dm-verity device. * * Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI - * allows verify_wq to preempt softirq since verification in tasklet + * allows verify_wq to preempt softirq since verification in BH workqueue * will fall-back to using it for error handling (or if the bufio cache * doesn't have required hashes). */ @@ -1507,8 +1586,8 @@ int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned i static struct target_type verity_target = { .name = "verity", - .features = DM_TARGET_IMMUTABLE, - .version = {1, 9, 0}, + .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE, + .version = {1, 10, 0}, .module = THIS_MODULE, .ctr = verity_ctr, .dtr = verity_dtr, |