From 073b9bf9af463d32555c5ebaf7e28c3a44c715d0 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Wed, 17 Dec 2025 11:41:23 +0200 Subject: nvme-pci: Use size_t for length fields to handle larger sizes This patch changes the length variables from unsigned int to size_t. Using size_t ensures that we can handle larger sizes, as size_t is always equal to or larger than the previously used u32 type. Originally, u32 was used because blk-mq-dma code evolved from scatter-gather implementation, which uses unsigned int to describe length. This change will also allow us to reuse the existing struct phys_vec in places that don't need scatter-gather. Signed-off-by: Leon Romanovsky Reviewed-by: Chaitanya Kulkarni Signed-off-by: Jens Axboe --- block/blk-mq-dma.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/blk-mq-dma.c b/block/blk-mq-dma.c index fb018fffffdc..a2bedc8f8666 100644 --- a/block/blk-mq-dma.c +++ b/block/blk-mq-dma.c @@ -8,7 +8,7 @@ struct phys_vec { phys_addr_t paddr; - u32 len; + size_t len; }; static bool __blk_map_iter_next(struct blk_map_iter *iter) @@ -112,8 +112,8 @@ static bool blk_rq_dma_map_iova(struct request *req, struct device *dma_dev, struct phys_vec *vec) { enum dma_data_direction dir = rq_dma_dir(req); - unsigned int mapped = 0; unsigned int attrs = 0; + size_t mapped = 0; int error; iter->addr = state->addr; @@ -297,6 +297,8 @@ int __blk_rq_map_sg(struct request *rq, struct scatterlist *sglist, blk_rq_map_iter_init(rq, &iter); while (blk_map_iter_next(rq, &iter, &vec)) { *last_sg = blk_next_sg(last_sg, sglist); + + WARN_ON_ONCE(overflows_type(vec.len, unsigned int)); sg_set_page(*last_sg, phys_to_page(vec.paddr), vec.len, offset_in_page(vec.paddr)); nsegs++; @@ -417,6 +419,8 @@ int blk_rq_map_integrity_sg(struct request *rq, struct scatterlist *sglist) while (blk_map_iter_next(rq, &iter, &vec)) { sg = blk_next_sg(&sg, sglist); + + WARN_ON_ONCE(overflows_type(vec.len, unsigned int)); sg_set_page(sg, phys_to_page(vec.paddr), vec.len, offset_in_page(vec.paddr)); segments++; -- cgit v1.2.3 From fcf463b92a08686d1aeb1e66674a72eb7a8bfb9b Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Wed, 17 Dec 2025 11:41:24 +0200 Subject: types: move phys_vec definition to common header Move the struct phys_vec definition from block/blk-mq-dma.c to include/linux/types.h to make it available for use across the kernel. The phys_vec structure represents a physical address range with a length, which is used by the new physical address-based DMA mapping API. This structure is already used by the block layer and will be needed for DMA phys API users. Moving this definition to types.h provides a centralized location for this common data structure and eliminates code duplication across subsystems that need to work with physical address ranges. Signed-off-by: Leon Romanovsky Reviewed-by: Chaitanya Kulkarni Signed-off-by: Jens Axboe --- block/blk-mq-dma.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'block') diff --git a/block/blk-mq-dma.c b/block/blk-mq-dma.c index a2bedc8f8666..752060d7261c 100644 --- a/block/blk-mq-dma.c +++ b/block/blk-mq-dma.c @@ -6,11 +6,6 @@ #include #include "blk.h" -struct phys_vec { - phys_addr_t paddr; - size_t len; -}; - static bool __blk_map_iter_next(struct blk_map_iter *iter) { if (iter->iter.bi_size) -- cgit v1.2.3 From ee623c892aa59003fca173de0041abc2ccc2c72d Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Wed, 31 Dec 2025 11:00:55 +0800 Subject: block: use bvec iterator helper for bio_may_need_split() bio_may_need_split() uses bi_vcnt to determine if a bio has a single segment, but bi_vcnt is unreliable for cloned bios. Cloned bios share the parent's bi_io_vec array but iterate over a subset via bi_iter, so bi_vcnt may not reflect the actual segment count being iterated. Replace the bi_vcnt check with bvec iterator access via __bvec_iter_bvec(), comparing bi_iter.bi_size against the current bvec's length. This correctly handles both cloned and non-cloned bios. Move bi_io_vec into the first cache line adjacent to bi_iter. This is a sensible layout since bi_io_vec and bi_iter are commonly accessed together throughout the block layer - every bvec iteration requires both fields. This displaces bi_end_io to the second cache line, which is acceptable since bi_end_io and bi_private are always fetched together in bio_endio() anyway. The struct layout change requires bio_reset() to preserve and restore bi_io_vec across the memset, since it now falls within BIO_RESET_BYTES. Nitesh verified that this patch doesn't regress NVMe 512-byte IO perf [1]. Link: https://lore.kernel.org/linux-block/20251220081607.tvnrltcngl3cc2fh@green245.gost/ [1] Signed-off-by: Ming Lei Reviewed-by: Nitesh Shetty Signed-off-by: Jens Axboe --- block/bio.c | 3 +++ block/blk.h | 12 +++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index e726c0e280a8..0e936288034e 100644 --- a/block/bio.c +++ b/block/bio.c @@ -301,9 +301,12 @@ EXPORT_SYMBOL(bio_init); */ void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf) { + struct bio_vec *bv = bio->bi_io_vec; + bio_uninit(bio); memset(bio, 0, BIO_RESET_BYTES); atomic_set(&bio->__bi_remaining, 1); + bio->bi_io_vec = bv; bio->bi_bdev = bdev; if (bio->bi_bdev) bio_associate_blkg(bio); diff --git a/block/blk.h b/block/blk.h index e4c433f62dfc..98f4dfd4ec75 100644 --- a/block/blk.h +++ b/block/blk.h @@ -371,12 +371,18 @@ struct bio *bio_split_zone_append(struct bio *bio, static inline bool bio_may_need_split(struct bio *bio, const struct queue_limits *lim) { + const struct bio_vec *bv; + if (lim->chunk_sectors) return true; - if (bio->bi_vcnt != 1) + + if (!bio->bi_io_vec) + return true; + + bv = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); + if (bio->bi_iter.bi_size > bv->bv_len) return true; - return bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset > - lim->max_fast_segment_size; + return bv->bv_len + bv->bv_offset > lim->max_fast_segment_size; } /** -- cgit v1.2.3 From 641864314866dff382f64cd8b52fd6bf4c4d84f6 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Wed, 31 Dec 2025 11:00:56 +0800 Subject: block: don't initialize bi_vcnt for cloned bio in bio_iov_bvec_set() bio_iov_bvec_set() creates a cloned bio that borrows a bvec array from an iov_iter. For cloned bios, bi_vcnt is meaningless because iteration is controlled entirely by bi_iter (bi_idx, bi_size, bi_bvec_done), not by bi_vcnt. Remove the incorrect bi_vcnt assignment. Explicitly initialize bi_iter.bi_idx to 0 to ensure iteration starts at the first bvec. While bi_idx is typically already zero from bio initialization, making this explicit improves clarity and correctness. This change also avoids accessing iter->nr_segs, which is an iov_iter implementation detail that block code should not depend on. Signed-off-by: Ming Lei Reviewed-by: Nitesh Shetty Signed-off-by: Jens Axboe --- block/bio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index 0e936288034e..2359c0723b88 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1165,8 +1165,8 @@ void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter) { WARN_ON_ONCE(bio->bi_max_vecs); - bio->bi_vcnt = iter->nr_segs; bio->bi_io_vec = (struct bio_vec *)iter->bvec; + bio->bi_iter.bi_idx = 0; bio->bi_iter.bi_bvec_done = iter->iov_offset; bio->bi_iter.bi_size = iov_iter_count(iter); bio_set_flag(bio, BIO_CLONED); -- cgit v1.2.3 From a31bde687b10b1a3db9c61eba5abb662dda15277 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Thu, 8 Jan 2026 10:22:12 -0700 Subject: block: use pi_tuple_size in bi_offload_capable() bi_offload_capable() returns whether a block device's metadata size matches its PI tuple size. Use pi_tuple_size instead of switching on csum_type. This makes the code considerably simpler and less branchy. Signed-off-by: Caleb Sander Mateos Reviewed-by: Christoph Hellwig Reviewed-by: Anuj Gupta Reviewed-by: Martin K. Petersen Signed-off-by: Jens Axboe --- block/bio-integrity-auto.c | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) (limited to 'block') diff --git a/block/bio-integrity-auto.c b/block/bio-integrity-auto.c index 9850c338548d..eb95e29c93bc 100644 --- a/block/bio-integrity-auto.c +++ b/block/bio-integrity-auto.c @@ -52,19 +52,7 @@ static bool bip_should_check(struct bio_integrity_payload *bip) static bool bi_offload_capable(struct blk_integrity *bi) { - switch (bi->csum_type) { - case BLK_INTEGRITY_CSUM_CRC64: - return bi->metadata_size == sizeof(struct crc64_pi_tuple); - case BLK_INTEGRITY_CSUM_CRC: - case BLK_INTEGRITY_CSUM_IP: - return bi->metadata_size == sizeof(struct t10_pi_tuple); - default: - pr_warn_once("%s: unknown integrity checksum type:%d\n", - __func__, bi->csum_type); - fallthrough; - case BLK_INTEGRITY_CSUM_NONE: - return false; - } + return bi->metadata_size == bi->pi_tuple_size; } /** -- cgit v1.2.3 From f7ba87dfa8e42642d43faf29a71cee338086218b Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Sat, 10 Jan 2026 21:42:36 +0800 Subject: block: account for bi_bvec_done in bio_may_need_split() When checking if a bio fits in a single segment, bio_may_need_split() compares bi_size against the current bvec's bv_len. However, for partially consumed bvecs (bi_bvec_done > 0), such as in cloned or split bios, the remaining bytes in the current bvec is actually (bv_len - bi_bvec_done), not bv_len. This could cause bio_may_need_split() to incorrectly return false, leading to nr_phys_segments being set to 1 when the bio actually spans multiple segments. This triggers the WARN_ON in __blk_rq_map_sg() when the actual mapped segments exceed the expected count. Fix by subtracting bi_bvec_done from bv_len in the comparison. Reported-by: Venkat Rao Bagalkote Close: https://lore.kernel.org/linux-block/9687cf2b-1f32-44e1-b58d-2492dc6e7185@linux.ibm.com/ Repored-and-bisected-by: Christoph Hellwig Tested-by: Venkat Rao Bagalkote Tested-by: Christoph Hellwig Fixes: ee623c892aa5 ("block: use bvec iterator helper for bio_may_need_split()") Cc: Nitesh Shetty Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'block') diff --git a/block/blk.h b/block/blk.h index 98f4dfd4ec75..980eef1f5690 100644 --- a/block/blk.h +++ b/block/blk.h @@ -380,7 +380,7 @@ static inline bool bio_may_need_split(struct bio *bio, return true; bv = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); - if (bio->bi_iter.bi_size > bv->bv_len) + if (bio->bi_iter.bi_size > bv->bv_len - bio->bi_iter.bi_bvec_done) return true; return bv->bv_len + bv->bv_offset > lim->max_fast_segment_size; } -- cgit v1.2.3 From aefc2a1fa2edc2a486aaf857e48b3fd13062b0eb Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 9 Jan 2026 07:07:44 +0100 Subject: blk-crypto: submit the encrypted bio in blk_crypto_fallback_bio_prep Restructure blk_crypto_fallback_bio_prep so that it always submits the encrypted bio instead of passing it back to the caller, which allows to simplify the calling conventions for blk_crypto_fallback_bio_prep and blk_crypto_bio_prep so that they never have to return a bio, and can use a true return value to indicate that the caller should submit the bio, and false that the blk-crypto code consumed it. The submission is handled by the on-stack bio list in the current task_struct by the block layer and does not cause additional stack usage or major overhead. It also prepares for the following optimization and fixes for the blk-crypto fallback write path. Signed-off-by: Christoph Hellwig Reviewed-by: Eric Biggers Signed-off-by: Jens Axboe --- block/blk-core.c | 2 +- block/blk-crypto-fallback.c | 70 +++++++++++++++++++++------------------------ block/blk-crypto-internal.h | 19 ++++-------- block/blk-crypto.c | 53 +++++++++++++++++----------------- 4 files changed, 67 insertions(+), 77 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index 8387fe50ea15..f87e5f1a101f 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -628,7 +628,7 @@ static void __submit_bio(struct bio *bio) /* If plug is not used, add new plug here to cache nsecs time. */ struct blk_plug plug; - if (unlikely(!blk_crypto_bio_prep(&bio))) + if (unlikely(!blk_crypto_bio_prep(bio))) return; blk_start_plug(&plug); diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index 86b27f96051a..cc9e90be23b7 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -250,14 +250,14 @@ static void blk_crypto_dun_to_iv(const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], /* * The crypto API fallback's encryption routine. - * Allocate a bounce bio for encryption, encrypt the input bio using crypto API, - * and replace *bio_ptr with the bounce bio. May split input bio if it's too - * large. Returns true on success. Returns false and sets bio->bi_status on - * error. + * + * Allocate one or more bios for encryption, encrypt the input bio using the + * crypto API, and submit the encrypted bios. Sets bio->bi_status and + * completes the source bio on error */ -static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr) +static void blk_crypto_fallback_encrypt_bio(struct bio *src_bio) { - struct bio *src_bio, *enc_bio; + struct bio *enc_bio; struct bio_crypt_ctx *bc; struct blk_crypto_keyslot *slot; int data_unit_size; @@ -267,14 +267,12 @@ static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr) struct scatterlist src, dst; union blk_crypto_iv iv; unsigned int i, j; - bool ret = false; blk_status_t blk_st; /* Split the bio if it's too big for single page bvec */ - if (!blk_crypto_fallback_split_bio_if_needed(bio_ptr)) - return false; + if (!blk_crypto_fallback_split_bio_if_needed(&src_bio)) + goto out_endio; - src_bio = *bio_ptr; bc = src_bio->bi_crypt_context; data_unit_size = bc->bc_key->crypto_cfg.data_unit_size; @@ -282,7 +280,7 @@ static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr) enc_bio = blk_crypto_fallback_clone_bio(src_bio); if (!enc_bio) { src_bio->bi_status = BLK_STS_RESOURCE; - return false; + goto out_endio; } /* @@ -345,25 +343,23 @@ static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr) enc_bio->bi_private = src_bio; enc_bio->bi_end_io = blk_crypto_fallback_encrypt_endio; - *bio_ptr = enc_bio; - ret = true; - - enc_bio = NULL; - goto out_free_ciph_req; + skcipher_request_free(ciph_req); + blk_crypto_put_keyslot(slot); + submit_bio(enc_bio); + return; out_free_bounce_pages: while (i > 0) mempool_free(enc_bio->bi_io_vec[--i].bv_page, blk_crypto_bounce_page_pool); -out_free_ciph_req: skcipher_request_free(ciph_req); out_release_keyslot: blk_crypto_put_keyslot(slot); out_put_enc_bio: - if (enc_bio) - bio_uninit(enc_bio); + bio_uninit(enc_bio); kfree(enc_bio); - return ret; +out_endio: + bio_endio(src_bio); } /* @@ -466,44 +462,44 @@ static void blk_crypto_fallback_decrypt_endio(struct bio *bio) /** * blk_crypto_fallback_bio_prep - Prepare a bio to use fallback en/decryption + * @bio: bio to prepare * - * @bio_ptr: pointer to the bio to prepare - * - * If bio is doing a WRITE operation, this splits the bio into two parts if it's - * too big (see blk_crypto_fallback_split_bio_if_needed()). It then allocates a - * bounce bio for the first part, encrypts it, and updates bio_ptr to point to - * the bounce bio. + * If bio is doing a WRITE operation, allocate one or more bios to contain the + * encrypted payload and submit them. * - * For a READ operation, we mark the bio for decryption by using bi_private and + * For a READ operation, mark the bio for decryption by using bi_private and * bi_end_io. * - * In either case, this function will make the bio look like a regular bio (i.e. - * as if no encryption context was ever specified) for the purposes of the rest - * of the stack except for blk-integrity (blk-integrity and blk-crypto are not - * currently supported together). + * In either case, this function will make the submitted bio(s) look like + * regular bios (i.e. as if no encryption context was ever specified) for the + * purposes of the rest of the stack except for blk-integrity (blk-integrity and + * blk-crypto are not currently supported together). * - * Return: true on success. Sets bio->bi_status and returns false on error. + * Return: true if @bio should be submitted to the driver by the caller, else + * false. Sets bio->bi_status, calls bio_endio and returns false on error. */ -bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr) +bool blk_crypto_fallback_bio_prep(struct bio *bio) { - struct bio *bio = *bio_ptr; struct bio_crypt_ctx *bc = bio->bi_crypt_context; struct bio_fallback_crypt_ctx *f_ctx; if (WARN_ON_ONCE(!tfms_inited[bc->bc_key->crypto_cfg.crypto_mode])) { /* User didn't call blk_crypto_start_using_key() first */ - bio->bi_status = BLK_STS_IOERR; + bio_io_error(bio); return false; } if (!__blk_crypto_cfg_supported(blk_crypto_fallback_profile, &bc->bc_key->crypto_cfg)) { bio->bi_status = BLK_STS_NOTSUPP; + bio_endio(bio); return false; } - if (bio_data_dir(bio) == WRITE) - return blk_crypto_fallback_encrypt_bio(bio_ptr); + if (bio_data_dir(bio) == WRITE) { + blk_crypto_fallback_encrypt_bio(bio); + return false; + } /* * bio READ case: Set up a f_ctx in the bio's bi_private and set the diff --git a/block/blk-crypto-internal.h b/block/blk-crypto-internal.h index ccf6dff6ff6b..d65023120341 100644 --- a/block/blk-crypto-internal.h +++ b/block/blk-crypto-internal.h @@ -165,11 +165,11 @@ static inline void bio_crypt_do_front_merge(struct request *rq, #endif } -bool __blk_crypto_bio_prep(struct bio **bio_ptr); -static inline bool blk_crypto_bio_prep(struct bio **bio_ptr) +bool __blk_crypto_bio_prep(struct bio *bio); +static inline bool blk_crypto_bio_prep(struct bio *bio) { - if (bio_has_crypt_ctx(*bio_ptr)) - return __blk_crypto_bio_prep(bio_ptr); + if (bio_has_crypt_ctx(bio)) + return __blk_crypto_bio_prep(bio); return true; } @@ -215,12 +215,12 @@ static inline int blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio, return 0; } +bool blk_crypto_fallback_bio_prep(struct bio *bio); + #ifdef CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num); -bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr); - int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key); #else /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ @@ -232,13 +232,6 @@ blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) return -ENOPKG; } -static inline bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr) -{ - pr_warn_once("crypto API fallback disabled; failing request.\n"); - (*bio_ptr)->bi_status = BLK_STS_NOTSUPP; - return false; -} - static inline int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key) { diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 3e7bf1974cbd..69e869d1c9bd 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -260,54 +260,55 @@ void __blk_crypto_free_request(struct request *rq) /** * __blk_crypto_bio_prep - Prepare bio for inline encryption - * - * @bio_ptr: pointer to original bio pointer + * @bio: bio to prepare * * If the bio crypt context provided for the bio is supported by the underlying * device's inline encryption hardware, do nothing. * * Otherwise, try to perform en/decryption for this bio by falling back to the - * kernel crypto API. When the crypto API fallback is used for encryption, - * blk-crypto may choose to split the bio into 2 - the first one that will - * continue to be processed and the second one that will be resubmitted via - * submit_bio_noacct. A bounce bio will be allocated to encrypt the contents - * of the aforementioned "first one", and *bio_ptr will be updated to this - * bounce bio. + * kernel crypto API. For encryption this means submitting newly allocated + * bios for the encrypted payload while keeping back the source bio until they + * complete, while for reads the decryption happens in-place by a hooked in + * completion handler. * * Caller must ensure bio has bio_crypt_ctx. * - * Return: true on success; false on error (and bio->bi_status will be set - * appropriately, and bio_endio() will have been called so bio - * submission should abort). + * Return: true if @bio should be submitted to the driver by the caller, else + * false. Sets bio->bi_status, calls bio_endio and returns false on error. */ -bool __blk_crypto_bio_prep(struct bio **bio_ptr) +bool __blk_crypto_bio_prep(struct bio *bio) { - struct bio *bio = *bio_ptr; const struct blk_crypto_key *bc_key = bio->bi_crypt_context->bc_key; + struct block_device *bdev = bio->bi_bdev; /* Error if bio has no data. */ if (WARN_ON_ONCE(!bio_has_data(bio))) { - bio->bi_status = BLK_STS_IOERR; - goto fail; + bio_io_error(bio); + return false; } if (!bio_crypt_check_alignment(bio)) { bio->bi_status = BLK_STS_INVAL; - goto fail; + bio_endio(bio); + return false; } /* - * Success if device supports the encryption context, or if we succeeded - * in falling back to the crypto API. + * If the device does not natively support the encryption context, try to use + * the fallback if available. */ - if (blk_crypto_config_supported_natively(bio->bi_bdev, - &bc_key->crypto_cfg)) - return true; - if (blk_crypto_fallback_bio_prep(bio_ptr)) - return true; -fail: - bio_endio(*bio_ptr); - return false; + if (!blk_crypto_config_supported_natively(bdev, &bc_key->crypto_cfg)) { + if (!IS_ENABLED(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK)) { + pr_warn_once("%pg: crypto API fallback disabled; failing request.\n", + bdev); + bio->bi_status = BLK_STS_NOTSUPP; + bio_endio(bio); + return false; + } + return blk_crypto_fallback_bio_prep(bio); + } + + return true; } int __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio, -- cgit v1.2.3 From b37fbce460ad60b0c4449c1c7566cf24f3016713 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 9 Jan 2026 07:07:45 +0100 Subject: blk-crypto: optimize bio splitting in blk_crypto_fallback_encrypt_bio The current code in blk_crypto_fallback_encrypt_bio is inefficient and prone to deadlocks under memory pressure: It first walks the passed in plaintext bio to see how much of it can fit into a single encrypted bio using up to BIO_MAX_VEC PAGE_SIZE segments, and then allocates a plaintext clone that fits the size, only to allocate another bio for the ciphertext later. While the plaintext clone uses a bioset to avoid deadlocks when allocations could fail, the ciphertex one uses bio_kmalloc which is a no-go in the file system I/O path. Switch blk_crypto_fallback_encrypt_bio to walk the source plaintext bio while consuming bi_iter without cloning it, and instead allocate a ciphertext bio at the beginning and whenever we fille up the previous one. The existing bio_set for the plaintext clones is reused for the ciphertext bios to remove the deadlock risk. Signed-off-by: Christoph Hellwig Reviewed-by: Eric Biggers Signed-off-by: Jens Axboe --- block/blk-crypto-fallback.c | 189 +++++++++++++++++++------------------------- 1 file changed, 80 insertions(+), 109 deletions(-) (limited to 'block') diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index cc9e90be23b7..4ec7da342280 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -81,7 +81,7 @@ static struct blk_crypto_fallback_keyslot { static struct blk_crypto_profile *blk_crypto_fallback_profile; static struct workqueue_struct *blk_crypto_wq; static mempool_t *blk_crypto_bounce_page_pool; -static struct bio_set crypto_bio_split; +static struct bio_set enc_bio_set; /* * This is the key we set when evicting a keyslot. This *should* be the all 0's @@ -150,37 +150,29 @@ static void blk_crypto_fallback_encrypt_endio(struct bio *enc_bio) mempool_free(enc_bio->bi_io_vec[i].bv_page, blk_crypto_bounce_page_pool); - src_bio->bi_status = enc_bio->bi_status; + if (enc_bio->bi_status) + cmpxchg(&src_bio->bi_status, 0, enc_bio->bi_status); - bio_uninit(enc_bio); - kfree(enc_bio); + bio_put(enc_bio); bio_endio(src_bio); } -static struct bio *blk_crypto_fallback_clone_bio(struct bio *bio_src) +static struct bio *blk_crypto_alloc_enc_bio(struct bio *bio_src, + unsigned int nr_segs) { - unsigned int nr_segs = bio_segments(bio_src); - struct bvec_iter iter; - struct bio_vec bv; struct bio *bio; - bio = bio_kmalloc(nr_segs, GFP_NOIO); - if (!bio) - return NULL; - bio_init_inline(bio, bio_src->bi_bdev, nr_segs, bio_src->bi_opf); + bio = bio_alloc_bioset(bio_src->bi_bdev, nr_segs, bio_src->bi_opf, + GFP_NOIO, &enc_bio_set); if (bio_flagged(bio_src, BIO_REMAPPED)) bio_set_flag(bio, BIO_REMAPPED); + bio->bi_private = bio_src; + bio->bi_end_io = blk_crypto_fallback_encrypt_endio; bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_write_hint = bio_src->bi_write_hint; bio->bi_write_stream = bio_src->bi_write_stream; bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; - bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; - - bio_for_each_segment(bv, bio_src, iter) - bio->bi_io_vec[bio->bi_vcnt++] = bv; - bio_clone_blkg_association(bio, bio_src); - return bio; } @@ -208,32 +200,6 @@ blk_crypto_fallback_alloc_cipher_req(struct blk_crypto_keyslot *slot, return true; } -static bool blk_crypto_fallback_split_bio_if_needed(struct bio **bio_ptr) -{ - struct bio *bio = *bio_ptr; - unsigned int i = 0; - unsigned int num_sectors = 0; - struct bio_vec bv; - struct bvec_iter iter; - - bio_for_each_segment(bv, bio, iter) { - num_sectors += bv.bv_len >> SECTOR_SHIFT; - if (++i == BIO_MAX_VECS) - break; - } - - if (num_sectors < bio_sectors(bio)) { - bio = bio_submit_split_bioset(bio, num_sectors, - &crypto_bio_split); - if (!bio) - return false; - - *bio_ptr = bio; - } - - return true; -} - union blk_crypto_iv { __le64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; u8 bytes[BLK_CRYPTO_MAX_IV_SIZE]; @@ -257,46 +223,35 @@ static void blk_crypto_dun_to_iv(const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], */ static void blk_crypto_fallback_encrypt_bio(struct bio *src_bio) { - struct bio *enc_bio; - struct bio_crypt_ctx *bc; - struct blk_crypto_keyslot *slot; - int data_unit_size; + struct bio_crypt_ctx *bc = src_bio->bi_crypt_context; + int data_unit_size = bc->bc_key->crypto_cfg.data_unit_size; struct skcipher_request *ciph_req = NULL; + struct blk_crypto_keyslot *slot; DECLARE_CRYPTO_WAIT(wait); u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; struct scatterlist src, dst; union blk_crypto_iv iv; - unsigned int i, j; - blk_status_t blk_st; - - /* Split the bio if it's too big for single page bvec */ - if (!blk_crypto_fallback_split_bio_if_needed(&src_bio)) - goto out_endio; - - bc = src_bio->bi_crypt_context; - data_unit_size = bc->bc_key->crypto_cfg.data_unit_size; - - /* Allocate bounce bio for encryption */ - enc_bio = blk_crypto_fallback_clone_bio(src_bio); - if (!enc_bio) { - src_bio->bi_status = BLK_STS_RESOURCE; - goto out_endio; - } + unsigned int nr_enc_pages, enc_idx; + struct bio *enc_bio; + blk_status_t status; + unsigned int i; /* * Get a blk-crypto-fallback keyslot that contains a crypto_skcipher for * this bio's algorithm and key. */ - blk_st = blk_crypto_get_keyslot(blk_crypto_fallback_profile, + status = blk_crypto_get_keyslot(blk_crypto_fallback_profile, bc->bc_key, &slot); - if (blk_st != BLK_STS_OK) { - src_bio->bi_status = blk_st; - goto out_put_enc_bio; + if (status != BLK_STS_OK) { + src_bio->bi_status = status; + bio_endio(src_bio); + return; } /* and then allocate an skcipher_request for it */ if (!blk_crypto_fallback_alloc_cipher_req(slot, &ciph_req, &wait)) { src_bio->bi_status = BLK_STS_RESOURCE; + bio_endio(src_bio); goto out_release_keyslot; } @@ -307,59 +262,75 @@ static void blk_crypto_fallback_encrypt_bio(struct bio *src_bio) skcipher_request_set_crypt(ciph_req, &src, &dst, data_unit_size, iv.bytes); - /* Encrypt each page in the bounce bio */ - for (i = 0; i < enc_bio->bi_vcnt; i++) { - struct bio_vec *enc_bvec = &enc_bio->bi_io_vec[i]; - struct page *plaintext_page = enc_bvec->bv_page; - struct page *ciphertext_page = - mempool_alloc(blk_crypto_bounce_page_pool, GFP_NOIO); - - enc_bvec->bv_page = ciphertext_page; - - if (!ciphertext_page) { - src_bio->bi_status = BLK_STS_RESOURCE; - goto out_free_bounce_pages; - } - - sg_set_page(&src, plaintext_page, data_unit_size, - enc_bvec->bv_offset); - sg_set_page(&dst, ciphertext_page, data_unit_size, - enc_bvec->bv_offset); - - /* Encrypt each data unit in this page */ - for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) { + /* + * Encrypt each page in the source bio. Because the source bio could + * have bio_vecs that span more than a single page, but the encrypted + * bios are limited to a single page per bio_vec, this can generate + * more than a single encrypted bio per source bio. + */ +new_bio: + nr_enc_pages = min(bio_segments(src_bio), BIO_MAX_VECS); + enc_bio = blk_crypto_alloc_enc_bio(src_bio, nr_enc_pages); + enc_idx = 0; + for (;;) { + struct bio_vec src_bv = + bio_iter_iovec(src_bio, src_bio->bi_iter); + struct page *enc_page; + + enc_page = mempool_alloc(blk_crypto_bounce_page_pool, + GFP_NOIO); + __bio_add_page(enc_bio, enc_page, src_bv.bv_len, + src_bv.bv_offset); + + sg_set_page(&src, src_bv.bv_page, data_unit_size, + src_bv.bv_offset); + sg_set_page(&dst, enc_page, data_unit_size, src_bv.bv_offset); + + /* + * Increment the index now that the encrypted page is added to + * the bio. This is important for the error unwind path. + */ + enc_idx++; + + /* + * Encrypt each data unit in this page. + */ + for (i = 0; i < src_bv.bv_len; i += data_unit_size) { blk_crypto_dun_to_iv(curr_dun, &iv); if (crypto_wait_req(crypto_skcipher_encrypt(ciph_req), &wait)) { - i++; - src_bio->bi_status = BLK_STS_IOERR; - goto out_free_bounce_pages; + bio_io_error(enc_bio); + goto out_free_request; } bio_crypt_dun_increment(curr_dun, 1); src.offset += data_unit_size; dst.offset += data_unit_size; } + + bio_advance_iter_single(src_bio, &src_bio->bi_iter, + src_bv.bv_len); + if (!src_bio->bi_iter.bi_size) + break; + + if (enc_idx == nr_enc_pages) { + /* + * For each additional encrypted bio submitted, + * increment the source bio's remaining count. Each + * encrypted bio's completion handler calls bio_endio on + * the source bio, so this keeps the source bio from + * completing until the last encrypted bio does. + */ + bio_inc_remaining(src_bio); + submit_bio(enc_bio); + goto new_bio; + } } - enc_bio->bi_private = src_bio; - enc_bio->bi_end_io = blk_crypto_fallback_encrypt_endio; - skcipher_request_free(ciph_req); - blk_crypto_put_keyslot(slot); submit_bio(enc_bio); - return; - -out_free_bounce_pages: - while (i > 0) - mempool_free(enc_bio->bi_io_vec[--i].bv_page, - blk_crypto_bounce_page_pool); +out_free_request: skcipher_request_free(ciph_req); out_release_keyslot: blk_crypto_put_keyslot(slot); -out_put_enc_bio: - bio_uninit(enc_bio); - kfree(enc_bio); -out_endio: - bio_endio(src_bio); } /* @@ -533,7 +504,7 @@ static int blk_crypto_fallback_init(void) get_random_bytes(blank_key, sizeof(blank_key)); - err = bioset_init(&crypto_bio_split, 64, 0, 0); + err = bioset_init(&enc_bio_set, 64, 0, BIOSET_NEED_BVECS); if (err) goto out; @@ -603,7 +574,7 @@ fail_destroy_profile: fail_free_profile: kfree(blk_crypto_fallback_profile); fail_free_bioset: - bioset_exit(&crypto_bio_split); + bioset_exit(&enc_bio_set); out: return err; } -- cgit v1.2.3 From 2f655dcb2d925b55deb8c1ec8f42b522c6bc5698 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 9 Jan 2026 07:07:46 +0100 Subject: blk-crypto: use on-stack skcipher requests for fallback en/decryption Allocating a skcipher request dynamically can deadlock or cause unexpected I/O failures when called from writeback context. Avoid the allocation entirely by using on-stack skciphers, similar to what the non-blk-crypto fscrypt path already does. This drops the incomplete support for asynchronous algorithms, which previously could be used, but only synchronously. Signed-off-by: Christoph Hellwig Reviewed-by: Eric Biggers Signed-off-by: Jens Axboe --- block/blk-crypto-fallback.c | 179 +++++++++++++++++++------------------------- 1 file changed, 79 insertions(+), 100 deletions(-) (limited to 'block') diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index 4ec7da342280..4a682230c278 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -75,7 +75,7 @@ static bool tfms_inited[BLK_ENCRYPTION_MODE_MAX]; static struct blk_crypto_fallback_keyslot { enum blk_crypto_mode_num crypto_mode; - struct crypto_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX]; + struct crypto_sync_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX]; } *blk_crypto_keyslots; static struct blk_crypto_profile *blk_crypto_fallback_profile; @@ -98,7 +98,7 @@ static void blk_crypto_fallback_evict_keyslot(unsigned int slot) WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID); /* Clear the key in the skcipher */ - err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, + err = crypto_sync_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, blk_crypto_modes[crypto_mode].keysize); WARN_ON(err); slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID; @@ -119,7 +119,7 @@ blk_crypto_fallback_keyslot_program(struct blk_crypto_profile *profile, blk_crypto_fallback_evict_keyslot(slot); slotp->crypto_mode = crypto_mode; - err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key->bytes, + err = crypto_sync_skcipher_setkey(slotp->tfms[crypto_mode], key->bytes, key->size); if (err) { blk_crypto_fallback_evict_keyslot(slot); @@ -176,28 +176,13 @@ static struct bio *blk_crypto_alloc_enc_bio(struct bio *bio_src, return bio; } -static bool -blk_crypto_fallback_alloc_cipher_req(struct blk_crypto_keyslot *slot, - struct skcipher_request **ciph_req_ret, - struct crypto_wait *wait) +static struct crypto_sync_skcipher * +blk_crypto_fallback_tfm(struct blk_crypto_keyslot *slot) { - struct skcipher_request *ciph_req; - const struct blk_crypto_fallback_keyslot *slotp; - int keyslot_idx = blk_crypto_keyslot_index(slot); - - slotp = &blk_crypto_keyslots[keyslot_idx]; - ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode], - GFP_NOIO); - if (!ciph_req) - return false; - - skcipher_request_set_callback(ciph_req, - CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP, - crypto_req_done, wait); - *ciph_req_ret = ciph_req; + const struct blk_crypto_fallback_keyslot *slotp = + &blk_crypto_keyslots[blk_crypto_keyslot_index(slot)]; - return true; + return slotp->tfms[slotp->crypto_mode]; } union blk_crypto_iv { @@ -214,46 +199,22 @@ static void blk_crypto_dun_to_iv(const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], iv->dun[i] = cpu_to_le64(dun[i]); } -/* - * The crypto API fallback's encryption routine. - * - * Allocate one or more bios for encryption, encrypt the input bio using the - * crypto API, and submit the encrypted bios. Sets bio->bi_status and - * completes the source bio on error - */ -static void blk_crypto_fallback_encrypt_bio(struct bio *src_bio) +static void __blk_crypto_fallback_encrypt_bio(struct bio *src_bio, + struct crypto_sync_skcipher *tfm) { struct bio_crypt_ctx *bc = src_bio->bi_crypt_context; int data_unit_size = bc->bc_key->crypto_cfg.data_unit_size; - struct skcipher_request *ciph_req = NULL; - struct blk_crypto_keyslot *slot; - DECLARE_CRYPTO_WAIT(wait); + SYNC_SKCIPHER_REQUEST_ON_STACK(ciph_req, tfm); u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; struct scatterlist src, dst; union blk_crypto_iv iv; unsigned int nr_enc_pages, enc_idx; struct bio *enc_bio; - blk_status_t status; unsigned int i; - /* - * Get a blk-crypto-fallback keyslot that contains a crypto_skcipher for - * this bio's algorithm and key. - */ - status = blk_crypto_get_keyslot(blk_crypto_fallback_profile, - bc->bc_key, &slot); - if (status != BLK_STS_OK) { - src_bio->bi_status = status; - bio_endio(src_bio); - return; - } - - /* and then allocate an skcipher_request for it */ - if (!blk_crypto_fallback_alloc_cipher_req(slot, &ciph_req, &wait)) { - src_bio->bi_status = BLK_STS_RESOURCE; - bio_endio(src_bio); - goto out_release_keyslot; - } + skcipher_request_set_callback(ciph_req, + CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, + NULL, NULL); memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun)); sg_init_table(&src, 1); @@ -297,10 +258,9 @@ new_bio: */ for (i = 0; i < src_bv.bv_len; i += data_unit_size) { blk_crypto_dun_to_iv(curr_dun, &iv); - if (crypto_wait_req(crypto_skcipher_encrypt(ciph_req), - &wait)) { + if (crypto_skcipher_encrypt(ciph_req)) { bio_io_error(enc_bio); - goto out_free_request; + return; } bio_crypt_dun_increment(curr_dun, 1); src.offset += data_unit_size; @@ -327,50 +287,48 @@ new_bio: } submit_bio(enc_bio); -out_free_request: - skcipher_request_free(ciph_req); -out_release_keyslot: - blk_crypto_put_keyslot(slot); } /* - * The crypto API fallback's main decryption routine. - * Decrypts input bio in place, and calls bio_endio on the bio. + * The crypto API fallback's encryption routine. + * + * Allocate one or more bios for encryption, encrypt the input bio using the + * crypto API, and submit the encrypted bios. Sets bio->bi_status and + * completes the source bio on error */ -static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) +static void blk_crypto_fallback_encrypt_bio(struct bio *src_bio) { - struct bio_fallback_crypt_ctx *f_ctx = - container_of(work, struct bio_fallback_crypt_ctx, work); - struct bio *bio = f_ctx->bio; - struct bio_crypt_ctx *bc = &f_ctx->crypt_ctx; + struct bio_crypt_ctx *bc = src_bio->bi_crypt_context; struct blk_crypto_keyslot *slot; - struct skcipher_request *ciph_req = NULL; - DECLARE_CRYPTO_WAIT(wait); + blk_status_t status; + + status = blk_crypto_get_keyslot(blk_crypto_fallback_profile, + bc->bc_key, &slot); + if (status != BLK_STS_OK) { + src_bio->bi_status = status; + bio_endio(src_bio); + return; + } + __blk_crypto_fallback_encrypt_bio(src_bio, + blk_crypto_fallback_tfm(slot)); + blk_crypto_put_keyslot(slot); +} + +static blk_status_t __blk_crypto_fallback_decrypt_bio(struct bio *bio, + struct bio_crypt_ctx *bc, struct bvec_iter iter, + struct crypto_sync_skcipher *tfm) +{ + SYNC_SKCIPHER_REQUEST_ON_STACK(ciph_req, tfm); u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; union blk_crypto_iv iv; struct scatterlist sg; struct bio_vec bv; - struct bvec_iter iter; const int data_unit_size = bc->bc_key->crypto_cfg.data_unit_size; unsigned int i; - blk_status_t blk_st; - - /* - * Get a blk-crypto-fallback keyslot that contains a crypto_skcipher for - * this bio's algorithm and key. - */ - blk_st = blk_crypto_get_keyslot(blk_crypto_fallback_profile, - bc->bc_key, &slot); - if (blk_st != BLK_STS_OK) { - bio->bi_status = blk_st; - goto out_no_keyslot; - } - /* and then allocate an skcipher_request for it */ - if (!blk_crypto_fallback_alloc_cipher_req(slot, &ciph_req, &wait)) { - bio->bi_status = BLK_STS_RESOURCE; - goto out; - } + skcipher_request_set_callback(ciph_req, + CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, + NULL, NULL); memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun)); sg_init_table(&sg, 1); @@ -378,7 +336,7 @@ static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) iv.bytes); /* Decrypt each segment in the bio */ - __bio_for_each_segment(bv, bio, iter, f_ctx->crypt_iter) { + __bio_for_each_segment(bv, bio, iter, iter) { struct page *page = bv.bv_page; sg_set_page(&sg, page, data_unit_size, bv.bv_offset); @@ -386,21 +344,41 @@ static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) /* Decrypt each data unit in the segment */ for (i = 0; i < bv.bv_len; i += data_unit_size) { blk_crypto_dun_to_iv(curr_dun, &iv); - if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req), - &wait)) { - bio->bi_status = BLK_STS_IOERR; - goto out; - } + if (crypto_skcipher_decrypt(ciph_req)) + return BLK_STS_IOERR; bio_crypt_dun_increment(curr_dun, 1); sg.offset += data_unit_size; } } -out: - skcipher_request_free(ciph_req); - blk_crypto_put_keyslot(slot); -out_no_keyslot: + return BLK_STS_OK; +} + +/* + * The crypto API fallback's main decryption routine. + * + * Decrypts input bio in place, and calls bio_endio on the bio. + */ +static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) +{ + struct bio_fallback_crypt_ctx *f_ctx = + container_of(work, struct bio_fallback_crypt_ctx, work); + struct bio *bio = f_ctx->bio; + struct bio_crypt_ctx *bc = &f_ctx->crypt_ctx; + struct blk_crypto_keyslot *slot; + blk_status_t status; + + status = blk_crypto_get_keyslot(blk_crypto_fallback_profile, + bc->bc_key, &slot); + if (status == BLK_STS_OK) { + status = __blk_crypto_fallback_decrypt_bio(bio, bc, + f_ctx->crypt_iter, + blk_crypto_fallback_tfm(slot)); + blk_crypto_put_keyslot(slot); + } mempool_free(f_ctx, bio_fallback_crypt_ctx_pool); + + bio->bi_status = status; bio_endio(bio); } @@ -608,7 +586,8 @@ int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) for (i = 0; i < blk_crypto_num_keyslots; i++) { slotp = &blk_crypto_keyslots[i]; - slotp->tfms[mode_num] = crypto_alloc_skcipher(cipher_str, 0, 0); + slotp->tfms[mode_num] = crypto_alloc_sync_skcipher(cipher_str, + 0, 0); if (IS_ERR(slotp->tfms[mode_num])) { err = PTR_ERR(slotp->tfms[mode_num]); if (err == -ENOENT) { @@ -620,7 +599,7 @@ int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) goto out_free_tfms; } - crypto_skcipher_set_flags(slotp->tfms[mode_num], + crypto_sync_skcipher_set_flags(slotp->tfms[mode_num], CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); } @@ -634,7 +613,7 @@ int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) out_free_tfms: for (i = 0; i < blk_crypto_num_keyslots; i++) { slotp = &blk_crypto_keyslots[i]; - crypto_free_skcipher(slotp->tfms[mode_num]); + crypto_free_sync_skcipher(slotp->tfms[mode_num]); slotp->tfms[mode_num] = NULL; } out: -- cgit v1.2.3 From 3d939695e68218d420be2b5dbb2fa39ccb7e97ed Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 9 Jan 2026 07:07:47 +0100 Subject: blk-crypto: use mempool_alloc_bulk for encrypted bio page allocation Calling mempool_alloc in a loop is not safe unless the maximum allocation size times the maximum number of threads using it is less than the minimum pool size. Use the new mempool_alloc_bulk helper to allocate all missing elements in one pass to remove this deadlock risk. This also means that non-pool allocations now use alloc_pages_bulk which can be significantly faster than a loop over individual page allocations. Signed-off-by: Christoph Hellwig Reviewed-by: Eric Biggers Signed-off-by: Jens Axboe --- block/blk-crypto-fallback.c | 76 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 62 insertions(+), 14 deletions(-) (limited to 'block') diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index 4a682230c278..6be971859542 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -22,7 +22,7 @@ #include "blk-cgroup.h" #include "blk-crypto-internal.h" -static unsigned int num_prealloc_bounce_pg = 32; +static unsigned int num_prealloc_bounce_pg = BIO_MAX_VECS; module_param(num_prealloc_bounce_pg, uint, 0); MODULE_PARM_DESC(num_prealloc_bounce_pg, "Number of preallocated bounce pages for the blk-crypto crypto API fallback"); @@ -144,11 +144,21 @@ static const struct blk_crypto_ll_ops blk_crypto_fallback_ll_ops = { static void blk_crypto_fallback_encrypt_endio(struct bio *enc_bio) { struct bio *src_bio = enc_bio->bi_private; - int i; + struct page **pages = (struct page **)enc_bio->bi_io_vec; + struct bio_vec *bv; + unsigned int i; + + /* + * Use the same trick as the alloc side to avoid the need for an extra + * pages array. + */ + bio_for_each_bvec_all(bv, enc_bio, i) + pages[i] = bv->bv_page; - for (i = 0; i < enc_bio->bi_vcnt; i++) - mempool_free(enc_bio->bi_io_vec[i].bv_page, - blk_crypto_bounce_page_pool); + i = mempool_free_bulk(blk_crypto_bounce_page_pool, (void **)pages, + enc_bio->bi_vcnt); + if (i < enc_bio->bi_vcnt) + release_pages(pages + i, enc_bio->bi_vcnt - i); if (enc_bio->bi_status) cmpxchg(&src_bio->bi_status, 0, enc_bio->bi_status); @@ -157,9 +167,14 @@ static void blk_crypto_fallback_encrypt_endio(struct bio *enc_bio) bio_endio(src_bio); } +#define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *)) + static struct bio *blk_crypto_alloc_enc_bio(struct bio *bio_src, - unsigned int nr_segs) + unsigned int nr_segs, struct page ***pages_ret) { + unsigned int memflags = memalloc_noio_save(); + unsigned int nr_allocated; + struct page **pages; struct bio *bio; bio = bio_alloc_bioset(bio_src->bi_bdev, nr_segs, bio_src->bi_opf, @@ -173,6 +188,30 @@ static struct bio *blk_crypto_alloc_enc_bio(struct bio *bio_src, bio->bi_write_stream = bio_src->bi_write_stream; bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; bio_clone_blkg_association(bio, bio_src); + + /* + * Move page array up in the allocated memory for the bio vecs as far as + * possible so that we can start filling biovecs from the beginning + * without overwriting the temporary page array. + */ + static_assert(PAGE_PTRS_PER_BVEC > 1); + pages = (struct page **)bio->bi_io_vec; + pages += nr_segs * (PAGE_PTRS_PER_BVEC - 1); + + /* + * Try a bulk allocation first. This could leave random pages in the + * array unallocated, but we'll fix that up later in mempool_alloc_bulk. + * + * Note: alloc_pages_bulk needs the array to be zeroed, as it assumes + * any non-zero slot already contains a valid allocation. + */ + memset(pages, 0, sizeof(struct page *) * nr_segs); + nr_allocated = alloc_pages_bulk(GFP_KERNEL, nr_segs, pages); + if (nr_allocated < nr_segs) + mempool_alloc_bulk(blk_crypto_bounce_page_pool, (void **)pages, + nr_segs, nr_allocated); + memalloc_noio_restore(memflags); + *pages_ret = pages; return bio; } @@ -209,6 +248,7 @@ static void __blk_crypto_fallback_encrypt_bio(struct bio *src_bio, struct scatterlist src, dst; union blk_crypto_iv iv; unsigned int nr_enc_pages, enc_idx; + struct page **enc_pages; struct bio *enc_bio; unsigned int i; @@ -231,15 +271,13 @@ static void __blk_crypto_fallback_encrypt_bio(struct bio *src_bio, */ new_bio: nr_enc_pages = min(bio_segments(src_bio), BIO_MAX_VECS); - enc_bio = blk_crypto_alloc_enc_bio(src_bio, nr_enc_pages); + enc_bio = blk_crypto_alloc_enc_bio(src_bio, nr_enc_pages, &enc_pages); enc_idx = 0; for (;;) { struct bio_vec src_bv = bio_iter_iovec(src_bio, src_bio->bi_iter); - struct page *enc_page; + struct page *enc_page = enc_pages[enc_idx]; - enc_page = mempool_alloc(blk_crypto_bounce_page_pool, - GFP_NOIO); __bio_add_page(enc_bio, enc_page, src_bv.bv_len, src_bv.bv_offset); @@ -258,10 +296,8 @@ new_bio: */ for (i = 0; i < src_bv.bv_len; i += data_unit_size) { blk_crypto_dun_to_iv(curr_dun, &iv); - if (crypto_skcipher_encrypt(ciph_req)) { - bio_io_error(enc_bio); - return; - } + if (crypto_skcipher_encrypt(ciph_req)) + goto out_free_enc_bio; bio_crypt_dun_increment(curr_dun, 1); src.offset += data_unit_size; dst.offset += data_unit_size; @@ -287,6 +323,18 @@ new_bio: } submit_bio(enc_bio); + return; + +out_free_enc_bio: + /* + * Add the remaining pages to the bio so that the normal completion path + * in blk_crypto_fallback_encrypt_endio frees them. The exact data + * layout does not matter for that, so don't bother iterating the source + * bio. + */ + for (; enc_idx < nr_enc_pages; enc_idx++) + __bio_add_page(enc_bio, enc_pages[enc_idx], PAGE_SIZE, 0); + bio_io_error(enc_bio); } /* -- cgit v1.2.3 From 66e5a11d2ed6d58006d5cd8276de28751daaa230 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 9 Jan 2026 07:07:48 +0100 Subject: blk-crypto: optimize data unit alignment checking Avoid the relatively high overhead of constructing and walking per-page segment bio_vecs for data unit alignment checking by merging the checks into existing loops. For hardware support crypto, perform the check in bio_split_io_at, which already contains a similar alignment check applied for all I/O. This means bio-based drivers that do not call bio_split_to_limits, should they ever grow blk-crypto support, need to implement the check themselves, just like all other queue limits checks. For blk-crypto-fallback do it in the encryption/decryption loops. This means alignment errors for decryption will only be detected after I/O has completed, but that seems like a worthwhile trade off. Signed-off-by: Christoph Hellwig Reviewed-by: Eric Biggers Signed-off-by: Jens Axboe --- block/blk-crypto-fallback.c | 15 +++++++++++++-- block/blk-crypto.c | 22 ---------------------- block/blk-merge.c | 9 ++++++++- 3 files changed, 21 insertions(+), 25 deletions(-) (limited to 'block') diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index 6be971859542..a331b061dbf4 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -278,6 +278,12 @@ new_bio: bio_iter_iovec(src_bio, src_bio->bi_iter); struct page *enc_page = enc_pages[enc_idx]; + if (!IS_ALIGNED(src_bv.bv_len | src_bv.bv_offset, + data_unit_size)) { + enc_bio->bi_status = BLK_STS_INVAL; + goto out_free_enc_bio; + } + __bio_add_page(enc_bio, enc_page, src_bv.bv_len, src_bv.bv_offset); @@ -296,8 +302,10 @@ new_bio: */ for (i = 0; i < src_bv.bv_len; i += data_unit_size) { blk_crypto_dun_to_iv(curr_dun, &iv); - if (crypto_skcipher_encrypt(ciph_req)) + if (crypto_skcipher_encrypt(ciph_req)) { + enc_bio->bi_status = BLK_STS_IOERR; goto out_free_enc_bio; + } bio_crypt_dun_increment(curr_dun, 1); src.offset += data_unit_size; dst.offset += data_unit_size; @@ -334,7 +342,7 @@ out_free_enc_bio: */ for (; enc_idx < nr_enc_pages; enc_idx++) __bio_add_page(enc_bio, enc_pages[enc_idx], PAGE_SIZE, 0); - bio_io_error(enc_bio); + bio_endio(enc_bio); } /* @@ -387,6 +395,9 @@ static blk_status_t __blk_crypto_fallback_decrypt_bio(struct bio *bio, __bio_for_each_segment(bv, bio, iter, iter) { struct page *page = bv.bv_page; + if (!IS_ALIGNED(bv.bv_len | bv.bv_offset, data_unit_size)) + return BLK_STS_INVAL; + sg_set_page(&sg, page, data_unit_size, bv.bv_offset); /* Decrypt each data unit in the segment */ diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 69e869d1c9bd..0b2535d8dbcc 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -219,22 +219,6 @@ bool bio_crypt_ctx_mergeable(struct bio_crypt_ctx *bc1, unsigned int bc1_bytes, return !bc1 || bio_crypt_dun_is_contiguous(bc1, bc1_bytes, bc2->bc_dun); } -/* Check that all I/O segments are data unit aligned. */ -static bool bio_crypt_check_alignment(struct bio *bio) -{ - const unsigned int data_unit_size = - bio->bi_crypt_context->bc_key->crypto_cfg.data_unit_size; - struct bvec_iter iter; - struct bio_vec bv; - - bio_for_each_segment(bv, bio, iter) { - if (!IS_ALIGNED(bv.bv_len | bv.bv_offset, data_unit_size)) - return false; - } - - return true; -} - blk_status_t __blk_crypto_rq_get_keyslot(struct request *rq) { return blk_crypto_get_keyslot(rq->q->crypto_profile, @@ -287,12 +271,6 @@ bool __blk_crypto_bio_prep(struct bio *bio) return false; } - if (!bio_crypt_check_alignment(bio)) { - bio->bi_status = BLK_STS_INVAL; - bio_endio(bio); - return false; - } - /* * If the device does not natively support the encryption context, try to use * the fallback if available. diff --git a/block/blk-merge.c b/block/blk-merge.c index d3115d7469df..b82c6d304658 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -324,12 +324,19 @@ static inline unsigned int bvec_seg_gap(struct bio_vec *bvprv, int bio_split_io_at(struct bio *bio, const struct queue_limits *lim, unsigned *segs, unsigned max_bytes, unsigned len_align_mask) { + struct bio_crypt_ctx *bc = bio_crypt_ctx(bio); struct bio_vec bv, bvprv, *bvprvp = NULL; unsigned nsegs = 0, bytes = 0, gaps = 0; struct bvec_iter iter; + unsigned start_align_mask = lim->dma_alignment; + + if (bc) { + start_align_mask |= (bc->bc_key->crypto_cfg.data_unit_size - 1); + len_align_mask |= (bc->bc_key->crypto_cfg.data_unit_size - 1); + } bio_for_each_bvec(bv, bio, iter) { - if (bv.bv_offset & lim->dma_alignment || + if (bv.bv_offset & start_align_mask || bv.bv_len & len_align_mask) return -EINVAL; -- cgit v1.2.3 From bb8e2019ad613dd023a59bf91d1768018d17e09b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 9 Jan 2026 07:07:49 +0100 Subject: blk-crypto: handle the fallback above the block layer Add a blk_crypto_submit_bio helper that either submits the bio when it is not encrypted or inline encryption is provided, but otherwise handles the encryption before going down into the low-level driver. This reduces the risk from bio reordering and keeps memory allocation as high up in the stack as possible. Note that if the submitter knows that inline enctryption is known to be supported by the underyling driver, it can still use plain submit_bio. Signed-off-by: Christoph Hellwig Reviewed-by: Eric Biggers Signed-off-by: Jens Axboe --- block/blk-core.c | 10 +++++++--- block/blk-crypto-internal.h | 19 +++++++++++-------- block/blk-crypto.c | 23 ++++++----------------- 3 files changed, 24 insertions(+), 28 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index f87e5f1a101f..a0bf5174e9e9 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -628,9 +628,6 @@ static void __submit_bio(struct bio *bio) /* If plug is not used, add new plug here to cache nsecs time. */ struct blk_plug plug; - if (unlikely(!blk_crypto_bio_prep(bio))) - return; - blk_start_plug(&plug); if (!bdev_test_flag(bio->bi_bdev, BD_HAS_SUBMIT_BIO)) { @@ -794,6 +791,13 @@ void submit_bio_noacct(struct bio *bio) if ((bio->bi_opf & REQ_NOWAIT) && !bdev_nowait(bdev)) goto not_supported; + if (bio_has_crypt_ctx(bio)) { + if (WARN_ON_ONCE(!bio_has_data(bio))) + goto end_io; + if (!blk_crypto_supported(bio)) + goto not_supported; + } + if (should_fail_bio(bio)) goto end_io; bio_check_ro(bio); diff --git a/block/blk-crypto-internal.h b/block/blk-crypto-internal.h index d65023120341..742694213529 100644 --- a/block/blk-crypto-internal.h +++ b/block/blk-crypto-internal.h @@ -86,6 +86,12 @@ bool __blk_crypto_cfg_supported(struct blk_crypto_profile *profile, int blk_crypto_ioctl(struct block_device *bdev, unsigned int cmd, void __user *argp); +static inline bool blk_crypto_supported(struct bio *bio) +{ + return blk_crypto_config_supported_natively(bio->bi_bdev, + &bio->bi_crypt_context->bc_key->crypto_cfg); +} + #else /* CONFIG_BLK_INLINE_ENCRYPTION */ static inline int blk_crypto_sysfs_register(struct gendisk *disk) @@ -139,6 +145,11 @@ static inline int blk_crypto_ioctl(struct block_device *bdev, unsigned int cmd, return -ENOTTY; } +static inline bool blk_crypto_supported(struct bio *bio) +{ + return false; +} + #endif /* CONFIG_BLK_INLINE_ENCRYPTION */ void __bio_crypt_advance(struct bio *bio, unsigned int bytes); @@ -165,14 +176,6 @@ static inline void bio_crypt_do_front_merge(struct request *rq, #endif } -bool __blk_crypto_bio_prep(struct bio *bio); -static inline bool blk_crypto_bio_prep(struct bio *bio) -{ - if (bio_has_crypt_ctx(bio)) - return __blk_crypto_bio_prep(bio); - return true; -} - blk_status_t __blk_crypto_rq_get_keyslot(struct request *rq); static inline blk_status_t blk_crypto_rq_get_keyslot(struct request *rq) { diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 0b2535d8dbcc..856d3c5b1fa0 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -242,25 +242,13 @@ void __blk_crypto_free_request(struct request *rq) rq->crypt_ctx = NULL; } -/** - * __blk_crypto_bio_prep - Prepare bio for inline encryption - * @bio: bio to prepare - * - * If the bio crypt context provided for the bio is supported by the underlying - * device's inline encryption hardware, do nothing. - * - * Otherwise, try to perform en/decryption for this bio by falling back to the - * kernel crypto API. For encryption this means submitting newly allocated - * bios for the encrypted payload while keeping back the source bio until they - * complete, while for reads the decryption happens in-place by a hooked in - * completion handler. - * - * Caller must ensure bio has bio_crypt_ctx. +/* + * Process a bio with a crypto context. Returns true if the caller should + * submit the passed in bio, false if the bio is consumed. * - * Return: true if @bio should be submitted to the driver by the caller, else - * false. Sets bio->bi_status, calls bio_endio and returns false on error. + * See the kerneldoc comment for blk_crypto_submit_bio for further details. */ -bool __blk_crypto_bio_prep(struct bio *bio) +bool __blk_crypto_submit_bio(struct bio *bio) { const struct blk_crypto_key *bc_key = bio->bi_crypt_context->bc_key; struct block_device *bdev = bio->bi_bdev; @@ -288,6 +276,7 @@ bool __blk_crypto_bio_prep(struct bio *bio) return true; } +EXPORT_SYMBOL_GPL(__blk_crypto_submit_bio); int __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio, gfp_t gfp_mask) -- cgit v1.2.3 From 91e1c1bcf0f2376f40ac859cf17d0a64a605e662 Mon Sep 17 00:00:00 2001 From: Nitesh Shetty Date: Mon, 12 Jan 2026 20:08:08 +0530 Subject: block, nvme: remove unused dma_iova_state function parameter DMA IOVA state is not used inside blk_rq_dma_map_iter_next, get rid of the argument. Signed-off-by: Nitesh Shetty Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-mq-dma.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'block') diff --git a/block/blk-mq-dma.c b/block/blk-mq-dma.c index fb018fffffdc..4afeda45df15 100644 --- a/block/blk-mq-dma.c +++ b/block/blk-mq-dma.c @@ -238,7 +238,6 @@ EXPORT_SYMBOL_GPL(blk_rq_dma_map_iter_start); * blk_rq_dma_map_iter_next - map the next DMA segment for a request * @req: request to map * @dma_dev: device to map to - * @state: DMA IOVA state * @iter: block layer DMA iterator * * Iterate to the next mapping after a previous call to @@ -253,7 +252,7 @@ EXPORT_SYMBOL_GPL(blk_rq_dma_map_iter_start); * returned in @iter.status. */ bool blk_rq_dma_map_iter_next(struct request *req, struct device *dma_dev, - struct dma_iova_state *state, struct blk_dma_iter *iter) + struct blk_dma_iter *iter) { struct phys_vec vec; -- cgit v1.2.3 From 41ee77b75308354054f4fe03a05b8016a0d41573 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Tue, 6 Jan 2026 16:00:56 +0900 Subject: block: fix blk_zone_cond_str() comment Fix the comment for blk_zone_cond_str() by replacing the meaningless BLK_ZONE_ZONE_XXX comment with the correct BLK_ZONE_COND_name, thus also replacing the XXX with what that actually means. Signed-off-by: Damien Le Moal Reviewed-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Reviewed-by: Bart Van Assche Signed-off-by: Jens Axboe --- block/blk-zoned.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'block') diff --git a/block/blk-zoned.c b/block/blk-zoned.c index 1c54678fae6b..ef3872c53244 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -112,12 +112,12 @@ static inline unsigned int disk_zone_wplugs_hash_size(struct gendisk *disk) #define BLK_ZONE_WPLUG_UNHASHED (1U << 2) /** - * blk_zone_cond_str - Return string XXX in BLK_ZONE_COND_XXX. - * @zone_cond: BLK_ZONE_COND_XXX. + * blk_zone_cond_str - Return a zone condition name string + * @zone_cond: a zone condition BLK_ZONE_COND_name * - * Description: Centralize block layer function to convert BLK_ZONE_COND_XXX - * into string format. Useful in the debugging and tracing zone conditions. For - * invalid BLK_ZONE_COND_XXX it returns string "UNKNOWN". + * Convert a BLK_ZONE_COND_name zone condition into the string "name". Useful + * for the debugging and tracing zone conditions. For an invalid zone + * conditions, the string "UNKNOWN" is returned. */ const char *blk_zone_cond_str(enum blk_zone_cond zone_cond) { -- cgit v1.2.3 From 5e35a24c96185e1be4c24a713e53a49e92ab925b Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Tue, 6 Jan 2026 16:00:57 +0900 Subject: block: improve blk_op_str() comment Replace XXX with what it actually means. Signed-off-by: Damien Le Moal Reviewed-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Reviewed-by: Bart Van Assche Signed-off-by: Jens Axboe --- block/blk-core.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index a0bf5174e9e9..d6732dc69dd9 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -114,12 +114,12 @@ static const char *const blk_op_name[] = { #undef REQ_OP_NAME /** - * blk_op_str - Return string XXX in the REQ_OP_XXX. - * @op: REQ_OP_XXX. + * blk_op_str - Return the string "name" for an operation REQ_OP_name. + * @op: a request operation. * - * Description: Centralize block layer function to convert REQ_OP_XXX into - * string format. Useful in the debugging and tracing bio or request. For - * invalid REQ_OP_XXX it returns string "UNKNOWN". + * Convert a request operation REQ_OP_name into the string "name". Useful for + * debugging and tracing BIOs and requests. For an invalid request operation + * code, the string "UNKNOWN" is returned. */ inline const char *blk_op_str(enum req_op op) { -- cgit v1.2.3 From 5e2fde1a9433efc484a5feec36f748aa3ea58c85 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Fri, 16 Jan 2026 15:46:37 +0800 Subject: block: pass io_comp_batch to rq_end_io_fn callback Add a third parameter 'const struct io_comp_batch *' to the rq_end_io_fn callback signature. This allows end_io handlers to access the completion batch context when requests are completed via blk_mq_end_request_batch(). The io_comp_batch is passed from blk_mq_end_request_batch(), while NULL is passed from __blk_mq_end_request() and blk_mq_put_rq_ref() which don't have batch context. This infrastructure change enables drivers to detect whether they're being called from a batched completion path (like iopoll) and access additional context stored in the io_comp_batch. Update all rq_end_io_fn implementations: - block/blk-mq.c: blk_end_sync_rq - block/blk-flush.c: flush_end_io, mq_flush_data_end_io - drivers/nvme/host/ioctl.c: nvme_uring_cmd_end_io - drivers/nvme/host/core.c: nvme_keep_alive_end_io - drivers/nvme/host/pci.c: abort_endio, nvme_del_queue_end, nvme_del_cq_end - drivers/nvme/target/passthru.c: nvmet_passthru_req_done - drivers/scsi/scsi_error.c: eh_lock_door_done - drivers/scsi/sg.c: sg_rq_end_io - drivers/scsi/st.c: st_scsi_execute_end - drivers/target/target_core_pscsi.c: pscsi_req_done - drivers/md/dm-rq.c: end_clone_request Signed-off-by: Ming Lei Reviewed-by: Kanchan Joshi Signed-off-by: Jens Axboe --- block/blk-flush.c | 6 ++++-- block/blk-mq.c | 9 +++++---- 2 files changed, 9 insertions(+), 6 deletions(-) (limited to 'block') diff --git a/block/blk-flush.c b/block/blk-flush.c index 43d6152897a4..403a46c86411 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -199,7 +199,8 @@ static void blk_flush_complete_seq(struct request *rq, } static enum rq_end_io_ret flush_end_io(struct request *flush_rq, - blk_status_t error) + blk_status_t error, + const struct io_comp_batch *iob) { struct request_queue *q = flush_rq->q; struct list_head *running; @@ -335,7 +336,8 @@ static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq, } static enum rq_end_io_ret mq_flush_data_end_io(struct request *rq, - blk_status_t error) + blk_status_t error, + const struct io_comp_batch *iob) { struct request_queue *q = rq->q; struct blk_mq_hw_ctx *hctx = rq->mq_hctx; diff --git a/block/blk-mq.c b/block/blk-mq.c index a29d8ac9d3e3..cf1daedbb39f 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1156,7 +1156,7 @@ inline void __blk_mq_end_request(struct request *rq, blk_status_t error) if (rq->end_io) { rq_qos_done(rq->q, rq); - if (rq->end_io(rq, error) == RQ_END_IO_FREE) + if (rq->end_io(rq, error, NULL) == RQ_END_IO_FREE) blk_mq_free_request(rq); } else { blk_mq_free_request(rq); @@ -1211,7 +1211,7 @@ void blk_mq_end_request_batch(struct io_comp_batch *iob) * If end_io handler returns NONE, then it still has * ownership of the request. */ - if (rq->end_io && rq->end_io(rq, 0) == RQ_END_IO_NONE) + if (rq->end_io && rq->end_io(rq, 0, iob) == RQ_END_IO_NONE) continue; WRITE_ONCE(rq->state, MQ_RQ_IDLE); @@ -1458,7 +1458,8 @@ struct blk_rq_wait { blk_status_t ret; }; -static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret) +static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret, + const struct io_comp_batch *iob) { struct blk_rq_wait *wait = rq->end_io_data; @@ -1688,7 +1689,7 @@ static bool blk_mq_req_expired(struct request *rq, struct blk_expired_data *expi void blk_mq_put_rq_ref(struct request *rq) { if (is_flush_rq(rq)) { - if (rq->end_io(rq, 0) == RQ_END_IO_FREE) + if (rq->end_io(rq, 0, NULL) == RQ_END_IO_FREE) blk_mq_free_request(rq); } else if (req_ref_put_and_test(rq)) { __blk_mq_free_request(rq); -- cgit v1.2.3 From f46ebb910989a1db244f95bd1f937907591aa2ee Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Tue, 27 Jan 2026 23:47:01 +0100 Subject: block: Replace snprintf with strscpy in check_partition Replace snprintf("%s", ...) with the faster and more direct strscpy(). Signed-off-by: Thorsten Blum Signed-off-by: Jens Axboe --- block/partitions/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'block') diff --git a/block/partitions/core.c b/block/partitions/core.c index 815ed33caa1b..079057ab535a 100644 --- a/block/partitions/core.c +++ b/block/partitions/core.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -130,7 +131,7 @@ static struct parsed_partitions *check_partition(struct gendisk *hd) state->pp_buf[0] = '\0'; state->disk = hd; - snprintf(state->name, BDEVNAME_SIZE, "%s", hd->disk_name); + strscpy(state->name, hd->disk_name); snprintf(state->pp_buf, PAGE_SIZE, " %s:", state->name); if (isdigit(state->name[strlen(state->name)-1])) sprintf(state->name, "p"); -- cgit v1.2.3 From 2719bd1ee1a1cd0535bc62e89b52822f2bbd14eb Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Thu, 29 Jan 2026 16:27:15 +0900 Subject: block: introduce blk_queue_rot() To check if a request queue is for a rotational device, a double negation is needed with the pattern "!blk_queue_nonrot(q)". Simplify this with the introduction of the helper blk_queue_rot() which tests if a requests queue limit has the BLK_FEAT_ROTATIONAL feature set. All call sites of blk_queue_nonrot() are modified to use blk_queue_rot() and blk_queue_nonrot() definition removed. No functional changes. Signed-off-by: Damien Le Moal Reviewed-by: Nitesh Shetty Reviewed-by: Martin K. Petersen Signed-off-by: Jens Axboe --- block/bfq-iosched.c | 20 ++++++++++---------- block/blk-iocost.c | 2 +- block/blk-iolatency.c | 5 +---- block/blk-wbt.c | 5 ++--- 4 files changed, 14 insertions(+), 18 deletions(-) (limited to 'block') diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 6e54b1d3d8bc..3ebdec40e758 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -231,7 +231,7 @@ static struct kmem_cache *bfq_pool; #define BFQ_RQ_SEEKY(bfqd, last_pos, rq) \ (get_sdist(last_pos, rq) > \ BFQQ_SEEK_THR && \ - (!blk_queue_nonrot(bfqd->queue) || \ + (blk_queue_rot(bfqd->queue) || \ blk_rq_sectors(rq) < BFQQ_SECT_THR_NONROT)) #define BFQQ_CLOSE_THR (sector_t)(8 * 1024) #define BFQQ_SEEKY(bfqq) (hweight32(bfqq->seek_history) > 19) @@ -4165,7 +4165,7 @@ static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, /* don't use too short time intervals */ if (delta_usecs < 1000) { - if (blk_queue_nonrot(bfqd->queue)) + if (!blk_queue_rot(bfqd->queue)) /* * give same worst-case guarantees as idling * for seeky @@ -4487,7 +4487,7 @@ static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd, struct bfq_queue *bfqq) { bool rot_without_queueing = - !blk_queue_nonrot(bfqd->queue) && !bfqd->hw_tag, + blk_queue_rot(bfqd->queue) && !bfqd->hw_tag, bfqq_sequential_and_IO_bound, idling_boosts_thr; @@ -4521,7 +4521,7 @@ static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd, * flash-based device. */ idling_boosts_thr = rot_without_queueing || - ((!blk_queue_nonrot(bfqd->queue) || !bfqd->hw_tag) && + ((blk_queue_rot(bfqd->queue) || !bfqd->hw_tag) && bfqq_sequential_and_IO_bound); /* @@ -4722,7 +4722,7 @@ bfq_choose_bfqq_for_injection(struct bfq_data *bfqd) * there is only one in-flight large request * at a time. */ - if (blk_queue_nonrot(bfqd->queue) && + if (!blk_queue_rot(bfqd->queue) && blk_rq_sectors(bfqq->next_rq) >= BFQQ_SECT_THR_NONROT && bfqd->tot_rq_in_driver >= 1) @@ -6340,7 +6340,7 @@ static void bfq_update_hw_tag(struct bfq_data *bfqd) bfqd->hw_tag_samples = 0; bfqd->nonrot_with_queueing = - blk_queue_nonrot(bfqd->queue) && bfqd->hw_tag; + !blk_queue_rot(bfqd->queue) && bfqd->hw_tag; } static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) @@ -7293,7 +7293,7 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_queue *eq) INIT_HLIST_HEAD(&bfqd->burst_list); bfqd->hw_tag = -1; - bfqd->nonrot_with_queueing = blk_queue_nonrot(bfqd->queue); + bfqd->nonrot_with_queueing = !blk_queue_rot(bfqd->queue); bfqd->bfq_max_budget = bfq_default_max_budget; @@ -7328,9 +7328,9 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_queue *eq) * Begin by assuming, optimistically, that the device peak * rate is equal to 2/3 of the highest reference rate. */ - bfqd->rate_dur_prod = ref_rate[blk_queue_nonrot(bfqd->queue)] * - ref_wr_duration[blk_queue_nonrot(bfqd->queue)]; - bfqd->peak_rate = ref_rate[blk_queue_nonrot(bfqd->queue)] * 2 / 3; + bfqd->rate_dur_prod = ref_rate[!blk_queue_rot(bfqd->queue)] * + ref_wr_duration[!blk_queue_rot(bfqd->queue)]; + bfqd->peak_rate = ref_rate[!blk_queue_rot(bfqd->queue)] * 2 / 3; /* see comments on the definition of next field inside bfq_data */ bfqd->actuator_load_threshold = 4; diff --git a/block/blk-iocost.c b/block/blk-iocost.c index a0416927d33d..ef543d163d46 100644 --- a/block/blk-iocost.c +++ b/block/blk-iocost.c @@ -812,7 +812,7 @@ static int ioc_autop_idx(struct ioc *ioc, struct gendisk *disk) u64 now_ns; /* rotational? */ - if (!blk_queue_nonrot(disk->queue)) + if (blk_queue_rot(disk->queue)) return AUTOP_HDD; /* handle SATA SSDs w/ broken NCQ */ diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c index 45bd18f68541..f7434278cd29 100644 --- a/block/blk-iolatency.c +++ b/block/blk-iolatency.c @@ -988,10 +988,7 @@ static void iolatency_pd_init(struct blkg_policy_data *pd) u64 now = blk_time_get_ns(); int cpu; - if (blk_queue_nonrot(blkg->q)) - iolat->ssd = true; - else - iolat->ssd = false; + iolat->ssd = !blk_queue_rot(blkg->q); for_each_possible_cpu(cpu) { struct latency_stat *stat; diff --git a/block/blk-wbt.c b/block/blk-wbt.c index 0974875f77bd..8e025834f2fb 100644 --- a/block/blk-wbt.c +++ b/block/blk-wbt.c @@ -747,10 +747,9 @@ u64 wbt_default_latency_nsec(struct request_queue *q) * We default to 2msec for non-rotational storage, and 75msec * for rotational storage. */ - if (blk_queue_nonrot(q)) - return 2000000ULL; - else + if (blk_queue_rot(q)) return 75000000ULL; + return 2000000ULL; } static int wbt_data_dir(const struct request *rq) -- cgit v1.2.3 From da562d92e6755c00cd67845a8dbfb908dac51a9c Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Fri, 30 Jan 2026 15:28:45 +0900 Subject: block: introduce bdev_rot() Introduce the helper function bdev_rot() to test if a block device is a rotational one. The existing function bdev_nonrot() which tests for the opposite condition is redefined using this new helper. This avoids the double negation (operator and name) that appears when testing if a block device is a rotational device, thus making the code a little easier to read. Call sites of bdev_nonrot() in the block layer are updated to use this new helper. Remaining users in other subsystems are left unchanged for now. Signed-off-by: Damien Le Moal Reviewed-by: Johannes Thumshirn Signed-off-by: Jens Axboe --- block/ioctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'block') diff --git a/block/ioctl.c b/block/ioctl.c index 344478348a54..fd48f82f9f03 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -692,7 +692,7 @@ static int blkdev_common_ioctl(struct block_device *bdev, blk_mode_t mode, queue_max_sectors(bdev_get_queue(bdev))); return put_ushort(argp, max_sectors); case BLKROTATIONAL: - return put_ushort(argp, !bdev_nonrot(bdev)); + return put_ushort(argp, bdev_rot(bdev)); case BLKRASET: case BLKFRASET: if(!capable(CAP_SYS_ADMIN)) -- cgit v1.2.3 From 06564bae93d024e346c49304dfb4e2aaa68cf620 Mon Sep 17 00:00:00 2001 From: Ondrej Kozina Date: Fri, 30 Jan 2026 17:33:52 +0100 Subject: sed-opal: ignore locking ranges array when not enabling SUM. The locking ranges count and the array items are always ignored unless Single User Mode (SUM) is requested in the activate method. It is useless to enforce limits of unused array in the non-SUM case. Signed-off-by: Ondrej Kozina Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/sed-opal.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'block') diff --git a/block/sed-opal.c b/block/sed-opal.c index 5a28f23f7f22..23a19c92d791 100644 --- a/block/sed-opal.c +++ b/block/sed-opal.c @@ -2940,7 +2940,8 @@ static int opal_activate_lsp(struct opal_dev *dev, }; int ret; - if (!opal_lr_act->num_lrs || opal_lr_act->num_lrs > OPAL_MAX_LRS) + if (opal_lr_act->sum && + (!opal_lr_act->num_lrs || opal_lr_act->num_lrs > OPAL_MAX_LRS)) return -EINVAL; ret = opal_get_key(dev, &opal_lr_act->key); -- cgit v1.2.3 From 2751b90051a0211ed7c78f26eb2a9b7038804b9b Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 2 Feb 2026 16:05:16 +0800 Subject: blk-wbt: factor out a helper wbt_set_lat() To move implementation details inside blk-wbt.c, prepare to fix possible deadlock to call wbt_init() while queue is frozen in the next patch. Reviewed-by: Ming Lei Reviewed-by: Nilay Shroff Signed-off-by: Yu Kuai Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-sysfs.c | 39 ++------------------------------------- block/blk-wbt.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++--- block/blk-wbt.h | 7 ++----- 3 files changed, 51 insertions(+), 45 deletions(-) (limited to 'block') diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index e0a70d26972b..a580688c3ad5 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -636,11 +636,8 @@ out: static ssize_t queue_wb_lat_store(struct gendisk *disk, const char *page, size_t count) { - struct request_queue *q = disk->queue; - struct rq_qos *rqos; ssize_t ret; s64 val; - unsigned int memflags; ret = queue_var_store64(&val, page); if (ret < 0) @@ -648,40 +645,8 @@ static ssize_t queue_wb_lat_store(struct gendisk *disk, const char *page, if (val < -1) return -EINVAL; - /* - * Ensure that the queue is idled, in case the latency update - * ends up either enabling or disabling wbt completely. We can't - * have IO inflight if that happens. - */ - memflags = blk_mq_freeze_queue(q); - - rqos = wbt_rq_qos(q); - if (!rqos) { - ret = wbt_init(disk); - if (ret) - goto out; - } - - ret = count; - if (val == -1) - val = wbt_default_latency_nsec(q); - else if (val >= 0) - val *= 1000ULL; - - if (wbt_get_min_lat(q) == val) - goto out; - - blk_mq_quiesce_queue(q); - - mutex_lock(&disk->rqos_state_mutex); - wbt_set_min_lat(q, val); - mutex_unlock(&disk->rqos_state_mutex); - - blk_mq_unquiesce_queue(q); -out: - blk_mq_unfreeze_queue(q, memflags); - - return ret; + ret = wbt_set_lat(disk, val); + return ret ? ret : count; } QUEUE_RW_ENTRY(queue_wb_lat, "wbt_lat_usec"); diff --git a/block/blk-wbt.c b/block/blk-wbt.c index 8e025834f2fb..0a37d97bda75 100644 --- a/block/blk-wbt.c +++ b/block/blk-wbt.c @@ -93,6 +93,8 @@ struct rq_wb { struct rq_depth rq_depth; }; +static int wbt_init(struct gendisk *disk); + static inline struct rq_wb *RQWB(struct rq_qos *rqos) { return container_of(rqos, struct rq_wb, rqos); @@ -506,7 +508,7 @@ u64 wbt_get_min_lat(struct request_queue *q) return RQWB(rqos)->min_lat_nsec; } -void wbt_set_min_lat(struct request_queue *q, u64 val) +static void wbt_set_min_lat(struct request_queue *q, u64 val) { struct rq_qos *rqos = wbt_rq_qos(q); if (!rqos) @@ -741,7 +743,7 @@ void wbt_init_enable_default(struct gendisk *disk) WARN_ON_ONCE(wbt_init(disk)); } -u64 wbt_default_latency_nsec(struct request_queue *q) +static u64 wbt_default_latency_nsec(struct request_queue *q) { /* * We default to 2msec for non-rotational storage, and 75msec @@ -901,7 +903,7 @@ static const struct rq_qos_ops wbt_rqos_ops = { #endif }; -int wbt_init(struct gendisk *disk) +static int wbt_init(struct gendisk *disk) { struct request_queue *q = disk->queue; struct rq_wb *rwb; @@ -948,3 +950,45 @@ err_free: return ret; } + +int wbt_set_lat(struct gendisk *disk, s64 val) +{ + struct request_queue *q = disk->queue; + unsigned int memflags; + struct rq_qos *rqos; + int ret = 0; + + /* + * Ensure that the queue is idled, in case the latency update + * ends up either enabling or disabling wbt completely. We can't + * have IO inflight if that happens. + */ + memflags = blk_mq_freeze_queue(q); + + rqos = wbt_rq_qos(q); + if (!rqos) { + ret = wbt_init(disk); + if (ret) + goto out; + } + + if (val == -1) + val = wbt_default_latency_nsec(q); + else if (val >= 0) + val *= 1000ULL; + + if (wbt_get_min_lat(q) == val) + goto out; + + blk_mq_quiesce_queue(q); + + mutex_lock(&disk->rqos_state_mutex); + wbt_set_min_lat(q, val); + mutex_unlock(&disk->rqos_state_mutex); + + blk_mq_unquiesce_queue(q); +out: + blk_mq_unfreeze_queue(q, memflags); + + return ret; +} diff --git a/block/blk-wbt.h b/block/blk-wbt.h index 925f22475738..6e39da17218b 100644 --- a/block/blk-wbt.h +++ b/block/blk-wbt.h @@ -4,16 +4,13 @@ #ifdef CONFIG_BLK_WBT -int wbt_init(struct gendisk *disk); void wbt_init_enable_default(struct gendisk *disk); void wbt_disable_default(struct gendisk *disk); void wbt_enable_default(struct gendisk *disk); u64 wbt_get_min_lat(struct request_queue *q); -void wbt_set_min_lat(struct request_queue *q, u64 val); -bool wbt_disabled(struct request_queue *); - -u64 wbt_default_latency_nsec(struct request_queue *); +bool wbt_disabled(struct request_queue *q); +int wbt_set_lat(struct gendisk *disk, s64 val); #else -- cgit v1.2.3 From 41afaeeda5099d9cd07eaa7dc6c3d20c6f1dd9e9 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 2 Feb 2026 16:05:17 +0800 Subject: blk-wbt: fix possible deadlock to nest pcpu_alloc_mutex under q_usage_counter If wbt is disabled by default and user configures wbt by sysfs, queue will be frozen first and then pcpu_alloc_mutex will be held in blk_stat_alloc_callback(). Fix this problem by allocating memory first before queue frozen. Signed-off-by: Yu Kuai Reviewed-by: Nilay Shroff Reviewed-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-wbt.c | 108 +++++++++++++++++++++++++++++++++----------------------- 1 file changed, 63 insertions(+), 45 deletions(-) (limited to 'block') diff --git a/block/blk-wbt.c b/block/blk-wbt.c index 0a37d97bda75..665760274e60 100644 --- a/block/blk-wbt.c +++ b/block/blk-wbt.c @@ -93,7 +93,7 @@ struct rq_wb { struct rq_depth rq_depth; }; -static int wbt_init(struct gendisk *disk); +static int wbt_init(struct gendisk *disk, struct rq_wb *rwb); static inline struct rq_wb *RQWB(struct rq_qos *rqos) { @@ -698,6 +698,41 @@ static void wbt_requeue(struct rq_qos *rqos, struct request *rq) } } +static int wbt_data_dir(const struct request *rq) +{ + const enum req_op op = req_op(rq); + + if (op == REQ_OP_READ) + return READ; + else if (op_is_write(op)) + return WRITE; + + /* don't account */ + return -1; +} + +static struct rq_wb *wbt_alloc(void) +{ + struct rq_wb *rwb = kzalloc(sizeof(*rwb), GFP_KERNEL); + + if (!rwb) + return NULL; + + rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb); + if (!rwb->cb) { + kfree(rwb); + return NULL; + } + + return rwb; +} + +static void wbt_free(struct rq_wb *rwb) +{ + blk_stat_free_callback(rwb->cb); + kfree(rwb); +} + /* * Enable wbt if defaults are configured that way */ @@ -739,8 +774,17 @@ EXPORT_SYMBOL_GPL(wbt_enable_default); void wbt_init_enable_default(struct gendisk *disk) { - if (__wbt_enable_default(disk)) - WARN_ON_ONCE(wbt_init(disk)); + struct rq_wb *rwb; + + if (!__wbt_enable_default(disk)) + return; + + rwb = wbt_alloc(); + if (WARN_ON_ONCE(!rwb)) + return; + + if (WARN_ON_ONCE(wbt_init(disk, rwb))) + wbt_free(rwb); } static u64 wbt_default_latency_nsec(struct request_queue *q) @@ -754,19 +798,6 @@ static u64 wbt_default_latency_nsec(struct request_queue *q) return 2000000ULL; } -static int wbt_data_dir(const struct request *rq) -{ - const enum req_op op = req_op(rq); - - if (op == REQ_OP_READ) - return READ; - else if (op_is_write(op)) - return WRITE; - - /* don't account */ - return -1; -} - static void wbt_queue_depth_changed(struct rq_qos *rqos) { RQWB(rqos)->rq_depth.queue_depth = blk_queue_depth(rqos->disk->queue); @@ -778,8 +809,7 @@ static void wbt_exit(struct rq_qos *rqos) struct rq_wb *rwb = RQWB(rqos); blk_stat_remove_callback(rqos->disk->queue, rwb->cb); - blk_stat_free_callback(rwb->cb); - kfree(rwb); + wbt_free(rwb); } /* @@ -903,22 +933,11 @@ static const struct rq_qos_ops wbt_rqos_ops = { #endif }; -static int wbt_init(struct gendisk *disk) +static int wbt_init(struct gendisk *disk, struct rq_wb *rwb) { struct request_queue *q = disk->queue; - struct rq_wb *rwb; - int i; int ret; - - rwb = kzalloc(sizeof(*rwb), GFP_KERNEL); - if (!rwb) - return -ENOMEM; - - rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb); - if (!rwb->cb) { - kfree(rwb); - return -ENOMEM; - } + int i; for (i = 0; i < WBT_NUM_RWQ; i++) rq_wait_init(&rwb->rq_wait[i]); @@ -938,38 +957,38 @@ static int wbt_init(struct gendisk *disk) ret = rq_qos_add(&rwb->rqos, disk, RQ_QOS_WBT, &wbt_rqos_ops); mutex_unlock(&q->rq_qos_mutex); if (ret) - goto err_free; + return ret; blk_stat_add_callback(q, rwb->cb); - return 0; - -err_free: - blk_stat_free_callback(rwb->cb); - kfree(rwb); - return ret; - } int wbt_set_lat(struct gendisk *disk, s64 val) { struct request_queue *q = disk->queue; + struct rq_qos *rqos = wbt_rq_qos(q); + struct rq_wb *rwb = NULL; unsigned int memflags; - struct rq_qos *rqos; int ret = 0; + if (!rqos) { + rwb = wbt_alloc(); + if (!rwb) + return -ENOMEM; + } + /* * Ensure that the queue is idled, in case the latency update * ends up either enabling or disabling wbt completely. We can't * have IO inflight if that happens. */ memflags = blk_mq_freeze_queue(q); - - rqos = wbt_rq_qos(q); if (!rqos) { - ret = wbt_init(disk); - if (ret) + ret = wbt_init(disk, rwb); + if (ret) { + wbt_free(rwb); goto out; + } } if (val == -1) @@ -989,6 +1008,5 @@ int wbt_set_lat(struct gendisk *disk, s64 val) blk_mq_unquiesce_queue(q); out: blk_mq_unfreeze_queue(q, memflags); - return ret; } -- cgit v1.2.3 From 3f0bea9f3b9e7d9bdc3794103575168ef007d309 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 2 Feb 2026 16:05:18 +0800 Subject: blk-mq-debugfs: factor out a helper to register debugfs for all rq_qos There is already a helper blk_mq_debugfs_register_rqos() to register one rqos, however this helper is called synchronously when the rqos is created with queue frozen. Prepare to fix possible deadlock to create blk-mq debugfs entries while queue is still frozen. Reviewed-by: Ming Lei Reviewed-by: Nilay Shroff Signed-off-by: Yu Kuai Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-mq-debugfs.c | 23 +++++++++++++++-------- block/blk-mq-debugfs.h | 5 +++++ 2 files changed, 20 insertions(+), 8 deletions(-) (limited to 'block') diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 4896525b1c05..4fe164b6d648 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -631,14 +631,7 @@ void blk_mq_debugfs_register(struct request_queue *q) blk_mq_debugfs_register_hctx(q, hctx); } - if (q->rq_qos) { - struct rq_qos *rqos = q->rq_qos; - - while (rqos) { - blk_mq_debugfs_register_rqos(rqos); - rqos = rqos->next; - } - } + blk_mq_debugfs_register_rq_qos(q); } static void blk_mq_debugfs_register_ctx(struct blk_mq_hw_ctx *hctx, @@ -769,6 +762,20 @@ void blk_mq_debugfs_register_rqos(struct rq_qos *rqos) debugfs_create_files(rqos->debugfs_dir, rqos, rqos->ops->debugfs_attrs); } +void blk_mq_debugfs_register_rq_qos(struct request_queue *q) +{ + lockdep_assert_held(&q->debugfs_mutex); + + if (q->rq_qos) { + struct rq_qos *rqos = q->rq_qos; + + while (rqos) { + blk_mq_debugfs_register_rqos(rqos); + rqos = rqos->next; + } + } +} + void blk_mq_debugfs_register_sched_hctx(struct request_queue *q, struct blk_mq_hw_ctx *hctx) { diff --git a/block/blk-mq-debugfs.h b/block/blk-mq-debugfs.h index c80e453e3014..54948a266889 100644 --- a/block/blk-mq-debugfs.h +++ b/block/blk-mq-debugfs.h @@ -33,6 +33,7 @@ void blk_mq_debugfs_register_sched_hctx(struct request_queue *q, struct blk_mq_hw_ctx *hctx); void blk_mq_debugfs_unregister_sched_hctx(struct blk_mq_hw_ctx *hctx); +void blk_mq_debugfs_register_rq_qos(struct request_queue *q); void blk_mq_debugfs_register_rqos(struct rq_qos *rqos); void blk_mq_debugfs_unregister_rqos(struct rq_qos *rqos); #else @@ -78,6 +79,10 @@ static inline void blk_mq_debugfs_register_rqos(struct rq_qos *rqos) { } +static inline void blk_mq_debugfs_register_rq_qos(struct request_queue *q) +{ +} + static inline void blk_mq_debugfs_unregister_rqos(struct rq_qos *rqos) { } -- cgit v1.2.3 From 3c17a346ffc613615f48c6f1ed30cdf328bab805 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 2 Feb 2026 16:05:19 +0800 Subject: blk-rq-qos: fix possible debugfs_mutex deadlock Currently rq-qos debugfs entries are created from rq_qos_add(), while rq_qos_add() can be called while queue is still frozen. This can deadlock because creating new entries can trigger fs reclaim. Fix this problem by delaying creating rq-qos debugfs entries after queue is unfrozen. - For wbt, 1) it can be initialized by default, fix it by calling new helper after wbt_init() from wbt_init_enable_default(); 2) it can be initialized by sysfs, fix it by calling new helper after queue is unfrozen from wbt_set_lat(). - For iocost and iolatency, they can only be initialized by blkcg configuration, however, they don't have debugfs entries for now, hence they are not handled yet. Signed-off-by: Yu Kuai Reviewed-by: Nilay Shroff Reviewed-by: Ming Lei Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-rq-qos.c | 7 ------- block/blk-wbt.c | 13 ++++++++++++- 2 files changed, 12 insertions(+), 8 deletions(-) (limited to 'block') diff --git a/block/blk-rq-qos.c b/block/blk-rq-qos.c index 654478dfbc20..d7ce99ce2e80 100644 --- a/block/blk-rq-qos.c +++ b/block/blk-rq-qos.c @@ -347,13 +347,6 @@ int rq_qos_add(struct rq_qos *rqos, struct gendisk *disk, enum rq_qos_id id, blk_queue_flag_set(QUEUE_FLAG_QOS_ENABLED, q); blk_mq_unfreeze_queue(q, memflags); - - if (rqos->ops->debugfs_attrs) { - mutex_lock(&q->debugfs_mutex); - blk_mq_debugfs_register_rqos(rqos); - mutex_unlock(&q->debugfs_mutex); - } - return 0; ebusy: blk_mq_unfreeze_queue(q, memflags); diff --git a/block/blk-wbt.c b/block/blk-wbt.c index 665760274e60..1415f2bf8611 100644 --- a/block/blk-wbt.c +++ b/block/blk-wbt.c @@ -774,6 +774,7 @@ EXPORT_SYMBOL_GPL(wbt_enable_default); void wbt_init_enable_default(struct gendisk *disk) { + struct request_queue *q = disk->queue; struct rq_wb *rwb; if (!__wbt_enable_default(disk)) @@ -783,8 +784,14 @@ void wbt_init_enable_default(struct gendisk *disk) if (WARN_ON_ONCE(!rwb)) return; - if (WARN_ON_ONCE(wbt_init(disk, rwb))) + if (WARN_ON_ONCE(wbt_init(disk, rwb))) { wbt_free(rwb); + return; + } + + mutex_lock(&q->debugfs_mutex); + blk_mq_debugfs_register_rq_qos(q); + mutex_unlock(&q->debugfs_mutex); } static u64 wbt_default_latency_nsec(struct request_queue *q) @@ -1008,5 +1015,9 @@ int wbt_set_lat(struct gendisk *disk, s64 val) blk_mq_unquiesce_queue(q); out: blk_mq_unfreeze_queue(q, memflags); + mutex_lock(&q->debugfs_mutex); + blk_mq_debugfs_register_rq_qos(q); + mutex_unlock(&q->debugfs_mutex); + return ret; } -- cgit v1.2.3 From 70bafa5e31ff979c4c38ac9838cc960a32c04f49 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 2 Feb 2026 16:05:20 +0800 Subject: blk-mq-debugfs: make blk_mq_debugfs_register_rqos() static Because it's only used inside blk-mq-debugfs.c now. Reviewed-by: Nilay Shroff Signed-off-by: Yu Kuai Reviewed-by: Ming Lei Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-mq-debugfs.c | 2 +- block/blk-mq-debugfs.h | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) (limited to 'block') diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 4fe164b6d648..11f00a868541 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -744,7 +744,7 @@ void blk_mq_debugfs_unregister_rqos(struct rq_qos *rqos) rqos->debugfs_dir = NULL; } -void blk_mq_debugfs_register_rqos(struct rq_qos *rqos) +static void blk_mq_debugfs_register_rqos(struct rq_qos *rqos) { struct request_queue *q = rqos->disk->queue; const char *dir_name = rq_qos_id_to_name(rqos->id); diff --git a/block/blk-mq-debugfs.h b/block/blk-mq-debugfs.h index 54948a266889..d94daa66556b 100644 --- a/block/blk-mq-debugfs.h +++ b/block/blk-mq-debugfs.h @@ -34,7 +34,6 @@ void blk_mq_debugfs_register_sched_hctx(struct request_queue *q, void blk_mq_debugfs_unregister_sched_hctx(struct blk_mq_hw_ctx *hctx); void blk_mq_debugfs_register_rq_qos(struct request_queue *q); -void blk_mq_debugfs_register_rqos(struct rq_qos *rqos); void blk_mq_debugfs_unregister_rqos(struct rq_qos *rqos); #else static inline void blk_mq_debugfs_register(struct request_queue *q) @@ -75,10 +74,6 @@ static inline void blk_mq_debugfs_unregister_sched_hctx(struct blk_mq_hw_ctx *hc { } -static inline void blk_mq_debugfs_register_rqos(struct rq_qos *rqos) -{ -} - static inline void blk_mq_debugfs_register_rq_qos(struct request_queue *q) { } -- cgit v1.2.3 From 5ae4b12ee6422a816efca4ede8411e4d5503b5ac Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 2 Feb 2026 16:05:21 +0800 Subject: blk-mq-debugfs: remove blk_mq_debugfs_unregister_rqos() Because this helper is only used by iocost and iolatency, while they don't have debugfs entries. Signed-off-by: Yu Kuai Reviewed-by: Nilay Shroff Reviewed-by: Ming Lei Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-mq-debugfs.c | 10 ---------- block/blk-mq-debugfs.h | 4 ---- block/blk-rq-qos.c | 4 ---- 3 files changed, 18 deletions(-) (limited to 'block') diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 11f00a868541..22c182b40bc3 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -734,16 +734,6 @@ static const char *rq_qos_id_to_name(enum rq_qos_id id) return "unknown"; } -void blk_mq_debugfs_unregister_rqos(struct rq_qos *rqos) -{ - lockdep_assert_held(&rqos->disk->queue->debugfs_mutex); - - if (!rqos->disk->queue->debugfs_dir) - return; - debugfs_remove_recursive(rqos->debugfs_dir); - rqos->debugfs_dir = NULL; -} - static void blk_mq_debugfs_register_rqos(struct rq_qos *rqos) { struct request_queue *q = rqos->disk->queue; diff --git a/block/blk-mq-debugfs.h b/block/blk-mq-debugfs.h index d94daa66556b..49bb1aaa83dc 100644 --- a/block/blk-mq-debugfs.h +++ b/block/blk-mq-debugfs.h @@ -34,7 +34,6 @@ void blk_mq_debugfs_register_sched_hctx(struct request_queue *q, void blk_mq_debugfs_unregister_sched_hctx(struct blk_mq_hw_ctx *hctx); void blk_mq_debugfs_register_rq_qos(struct request_queue *q); -void blk_mq_debugfs_unregister_rqos(struct rq_qos *rqos); #else static inline void blk_mq_debugfs_register(struct request_queue *q) { @@ -78,9 +77,6 @@ static inline void blk_mq_debugfs_register_rq_qos(struct request_queue *q) { } -static inline void blk_mq_debugfs_unregister_rqos(struct rq_qos *rqos) -{ -} #endif #if defined(CONFIG_BLK_DEV_ZONED) && defined(CONFIG_BLK_DEBUG_FS) diff --git a/block/blk-rq-qos.c b/block/blk-rq-qos.c index d7ce99ce2e80..85cf74402a09 100644 --- a/block/blk-rq-qos.c +++ b/block/blk-rq-qos.c @@ -371,8 +371,4 @@ void rq_qos_del(struct rq_qos *rqos) if (!q->rq_qos) blk_queue_flag_clear(QUEUE_FLAG_QOS_ENABLED, q); blk_mq_unfreeze_queue(q, memflags); - - mutex_lock(&q->debugfs_mutex); - blk_mq_debugfs_unregister_rqos(rqos); - mutex_unlock(&q->debugfs_mutex); } -- cgit v1.2.3 From 9d20fd6ce1ba9733cd5ac96fcab32faa9fc404dd Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 2 Feb 2026 16:05:22 +0800 Subject: blk-mq-debugfs: add missing debugfs_mutex in blk_mq_debugfs_register_hctxs() In blk_mq_update_nr_hw_queues(), debugfs_mutex is not held while creating debugfs entries for hctxs. Hence add debugfs_mutex there, it's safe because queue is not frozen. Signed-off-by: Yu Kuai Reviewed-by: Nilay Shroff Reviewed-by: Ming Lei Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-mq-debugfs.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'block') diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 22c182b40bc3..5c7cadf51a88 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -679,8 +679,10 @@ void blk_mq_debugfs_register_hctxs(struct request_queue *q) struct blk_mq_hw_ctx *hctx; unsigned long i; + mutex_lock(&q->debugfs_mutex); queue_for_each_hw_ctx(q, hctx, i) blk_mq_debugfs_register_hctx(q, hctx); + mutex_unlock(&q->debugfs_mutex); } void blk_mq_debugfs_unregister_hctxs(struct request_queue *q) -- cgit v1.2.3 From 65d466b6298470ce21ab21ebfdb51309d515737d Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 2 Feb 2026 16:05:23 +0800 Subject: blk-mq-debugfs: warn about possible deadlock Creating new debugfs entries can trigger fs reclaim, hence we can't do this with queue frozen, meanwhile, other locks that can be held while queue is frozen should not be held as well. Signed-off-by: Yu Kuai Reviewed-by: Nilay Shroff Reviewed-by: Ming Lei Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-mq-debugfs.c | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) (limited to 'block') diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 5c7cadf51a88..faeaa1fc86a7 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -608,9 +608,23 @@ static const struct blk_mq_debugfs_attr blk_mq_debugfs_ctx_attrs[] = { {}, }; -static void debugfs_create_files(struct dentry *parent, void *data, +static void debugfs_create_files(struct request_queue *q, struct dentry *parent, + void *data, const struct blk_mq_debugfs_attr *attr) { + lockdep_assert_held(&q->debugfs_mutex); + /* + * Creating new debugfs entries with queue freezed has the risk of + * deadlock. + */ + WARN_ON_ONCE(q->mq_freeze_depth != 0); + /* + * debugfs_mutex should not be nested under other locks that can be + * grabbed while queue is frozen. + */ + lockdep_assert_not_held(&q->elevator_lock); + lockdep_assert_not_held(&q->rq_qos_mutex); + if (IS_ERR_OR_NULL(parent)) return; @@ -624,7 +638,7 @@ void blk_mq_debugfs_register(struct request_queue *q) struct blk_mq_hw_ctx *hctx; unsigned long i; - debugfs_create_files(q->debugfs_dir, q, blk_mq_debugfs_queue_attrs); + debugfs_create_files(q, q->debugfs_dir, q, blk_mq_debugfs_queue_attrs); queue_for_each_hw_ctx(q, hctx, i) { if (!hctx->debugfs_dir) @@ -643,7 +657,8 @@ static void blk_mq_debugfs_register_ctx(struct blk_mq_hw_ctx *hctx, snprintf(name, sizeof(name), "cpu%u", ctx->cpu); ctx_dir = debugfs_create_dir(name, hctx->debugfs_dir); - debugfs_create_files(ctx_dir, ctx, blk_mq_debugfs_ctx_attrs); + debugfs_create_files(hctx->queue, ctx_dir, ctx, + blk_mq_debugfs_ctx_attrs); } void blk_mq_debugfs_register_hctx(struct request_queue *q, @@ -659,7 +674,8 @@ void blk_mq_debugfs_register_hctx(struct request_queue *q, snprintf(name, sizeof(name), "hctx%u", hctx->queue_num); hctx->debugfs_dir = debugfs_create_dir(name, q->debugfs_dir); - debugfs_create_files(hctx->debugfs_dir, hctx, blk_mq_debugfs_hctx_attrs); + debugfs_create_files(q, hctx->debugfs_dir, hctx, + blk_mq_debugfs_hctx_attrs); hctx_for_each_ctx(hctx, ctx, i) blk_mq_debugfs_register_ctx(hctx, ctx); @@ -712,7 +728,7 @@ void blk_mq_debugfs_register_sched(struct request_queue *q) q->sched_debugfs_dir = debugfs_create_dir("sched", q->debugfs_dir); - debugfs_create_files(q->sched_debugfs_dir, q, e->queue_debugfs_attrs); + debugfs_create_files(q, q->sched_debugfs_dir, q, e->queue_debugfs_attrs); } void blk_mq_debugfs_unregister_sched(struct request_queue *q) @@ -751,7 +767,8 @@ static void blk_mq_debugfs_register_rqos(struct rq_qos *rqos) q->debugfs_dir); rqos->debugfs_dir = debugfs_create_dir(dir_name, q->rqos_debugfs_dir); - debugfs_create_files(rqos->debugfs_dir, rqos, rqos->ops->debugfs_attrs); + debugfs_create_files(q, rqos->debugfs_dir, rqos, + rqos->ops->debugfs_attrs); } void blk_mq_debugfs_register_rq_qos(struct request_queue *q) @@ -788,7 +805,7 @@ void blk_mq_debugfs_register_sched_hctx(struct request_queue *q, hctx->sched_debugfs_dir = debugfs_create_dir("sched", hctx->debugfs_dir); - debugfs_create_files(hctx->sched_debugfs_dir, hctx, + debugfs_create_files(q, hctx->sched_debugfs_dir, hctx, e->hctx_debugfs_attrs); } -- cgit v1.2.3 From 1db61b0afdd7e8aa9289c423fdff002603b520b5 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Tue, 3 Feb 2026 16:19:43 +0800 Subject: blk-mq-sched: unify elevators checking for async requests bfq and mq-deadline consider sync writes as async requests and only reserve tags for sync reads by async_depth, however, kyber doesn't consider sync writes as async requests for now. Consider the case there are lots of dirty pages, and user use fsync to flush dirty pages. In this case sched_tags can be exhausted by sync writes and sync reads can stuck waiting for tag. Hence let kyber follow what mq-deadline and bfq did, and unify async requests checking for all elevators. Signed-off-by: Yu Kuai Reviewed-by: Nilay Shroff Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/bfq-iosched.c | 2 +- block/blk-mq-sched.h | 5 +++++ block/kyber-iosched.c | 2 +- block/mq-deadline.c | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) (limited to 'block') diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 3ebdec40e758..44746f4c0b89 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -697,7 +697,7 @@ static void bfq_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) unsigned int limit, act_idx; /* Sync reads have full depth available */ - if (op_is_sync(opf) && !op_is_write(opf)) + if (blk_mq_is_sync_read(opf)) limit = data->q->nr_requests; else limit = bfqd->async_depths[!!bfqd->wr_busy_queues][op_is_sync(opf)]; diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h index 02c40a72e959..5678e15bd33c 100644 --- a/block/blk-mq-sched.h +++ b/block/blk-mq-sched.h @@ -137,4 +137,9 @@ static inline void blk_mq_set_min_shallow_depth(struct request_queue *q, depth); } +static inline bool blk_mq_is_sync_read(blk_opf_t opf) +{ + return op_is_sync(opf) && !op_is_write(opf); +} + #endif diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index c1b36ffd19ce..2b3f5b8959af 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -556,7 +556,7 @@ static void kyber_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) * We use the scheduler tags as per-hardware queue queueing tokens. * Async requests can be limited at this stage. */ - if (!op_is_sync(opf)) { + if (!blk_mq_is_sync_read(opf)) { struct kyber_queue_data *kqd = data->q->elevator->elevator_data; data->shallow_depth = kqd->async_depth; diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 3e3719093aec..29d00221fbea 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -495,7 +495,7 @@ static void dd_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) struct deadline_data *dd = data->q->elevator->elevator_data; /* Do not throttle synchronous reads. */ - if (op_is_sync(opf) && !op_is_write(opf)) + if (blk_mq_is_sync_read(opf)) return; /* -- cgit v1.2.3 From cf02d7d41b064af3e2c3a3a1ea9042a5b565b0d8 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Tue, 3 Feb 2026 16:19:44 +0800 Subject: blk-mq: factor out a helper blk_mq_limit_depth() There are no functional changes, just make code cleaner. Signed-off-by: Yu Kuai Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-mq.c | 62 +++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 25 deletions(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index cf1daedbb39f..b7b272e856b8 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -498,6 +498,42 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) return rq_list_pop(data->cached_rqs); } +static void blk_mq_limit_depth(struct blk_mq_alloc_data *data) +{ + struct elevator_mq_ops *ops; + + /* If no I/O scheduler has been configured, don't limit requests */ + if (!data->q->elevator) { + blk_mq_tag_busy(data->hctx); + return; + } + + /* + * All requests use scheduler tags when an I/O scheduler is + * enabled for the queue. + */ + data->rq_flags |= RQF_SCHED_TAGS; + + /* + * Flush/passthrough requests are special and go directly to the + * dispatch list, they are not subject to the async_depth limit. + */ + if ((data->cmd_flags & REQ_OP_MASK) == REQ_OP_FLUSH || + blk_op_is_passthrough(data->cmd_flags)) + return; + + WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED); + data->rq_flags |= RQF_USE_SCHED; + + /* + * By default, sync requests have no limit, and async requests are + * limited to async_depth. + */ + ops = &data->q->elevator->type->ops; + if (ops->limit_depth) + ops->limit_depth(data->cmd_flags, data); +} + static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) { struct request_queue *q = data->q; @@ -516,31 +552,7 @@ retry: data->ctx = blk_mq_get_ctx(q); data->hctx = blk_mq_map_queue(data->cmd_flags, data->ctx); - if (q->elevator) { - /* - * All requests use scheduler tags when an I/O scheduler is - * enabled for the queue. - */ - data->rq_flags |= RQF_SCHED_TAGS; - - /* - * Flush/passthrough requests are special and go directly to the - * dispatch list. - */ - if ((data->cmd_flags & REQ_OP_MASK) != REQ_OP_FLUSH && - !blk_op_is_passthrough(data->cmd_flags)) { - struct elevator_mq_ops *ops = &q->elevator->type->ops; - - WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED); - - data->rq_flags |= RQF_USE_SCHED; - if (ops->limit_depth) - ops->limit_depth(data->cmd_flags, data); - } - } else { - blk_mq_tag_busy(data->hctx); - } - + blk_mq_limit_depth(data); if (data->flags & BLK_MQ_REQ_RESERVED) data->rq_flags |= RQF_RESV; -- cgit v1.2.3 From f98afe4f31bb8b07fea318606c08030c2049587e Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Tue, 3 Feb 2026 16:19:45 +0800 Subject: blk-mq: add a new queue sysfs attribute async_depth Add a new field async_depth to request_queue and related APIs, this is currently not used, following patches will convert elevators to use this instead of internal async_depth. Signed-off-by: Yu Kuai Reviewed-by: Nilay Shroff Signed-off-by: Jens Axboe --- block/blk-core.c | 1 + block/blk-mq.c | 6 ++++++ block/blk-sysfs.c | 42 ++++++++++++++++++++++++++++++++++++++++++ block/elevator.c | 1 + 4 files changed, 50 insertions(+) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index d6732dc69dd9..474700ffaa1c 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -463,6 +463,7 @@ struct request_queue *blk_alloc_queue(struct queue_limits *lim, int node_id) fs_reclaim_release(GFP_KERNEL); q->nr_requests = BLKDEV_DEFAULT_RQ; + q->async_depth = BLKDEV_DEFAULT_RQ; return q; diff --git a/block/blk-mq.c b/block/blk-mq.c index b7b272e856b8..0ad3dd3329db 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -4662,6 +4662,7 @@ int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, spin_lock_init(&q->requeue_lock); q->nr_requests = set->queue_depth; + q->async_depth = set->queue_depth; blk_mq_init_cpu_queues(q, set->nr_hw_queues); blk_mq_map_swqueue(q); @@ -5028,6 +5029,11 @@ struct elevator_tags *blk_mq_update_nr_requests(struct request_queue *q, q->elevator->et = et; } + /* + * Preserve relative value, both nr and async_depth are at most 16 bit + * value, no need to worry about overflow. + */ + q->async_depth = max(q->async_depth * nr / q->nr_requests, 1); q->nr_requests = nr; if (q->elevator && q->elevator->type->ops.depth_updated) q->elevator->type->ops.depth_updated(q); diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index a580688c3ad5..003aa684e854 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -127,6 +127,46 @@ unlock: return ret; } +static ssize_t queue_async_depth_show(struct gendisk *disk, char *page) +{ + guard(mutex)(&disk->queue->elevator_lock); + + return queue_var_show(disk->queue->async_depth, page); +} + +static ssize_t +queue_async_depth_store(struct gendisk *disk, const char *page, size_t count) +{ + struct request_queue *q = disk->queue; + unsigned int memflags; + unsigned long nr; + int ret; + + if (!queue_is_mq(q)) + return -EINVAL; + + ret = queue_var_store(&nr, page, count); + if (ret < 0) + return ret; + + if (nr == 0) + return -EINVAL; + + memflags = blk_mq_freeze_queue(q); + scoped_guard(mutex, &q->elevator_lock) { + if (q->elevator) { + q->async_depth = min(q->nr_requests, nr); + if (q->elevator->type->ops.depth_updated) + q->elevator->type->ops.depth_updated(q); + } else { + ret = -EINVAL; + } + } + blk_mq_unfreeze_queue(q, memflags); + + return ret; +} + static ssize_t queue_ra_show(struct gendisk *disk, char *page) { ssize_t ret; @@ -532,6 +572,7 @@ static struct queue_sysfs_entry _prefix##_entry = { \ } QUEUE_RW_ENTRY(queue_requests, "nr_requests"); +QUEUE_RW_ENTRY(queue_async_depth, "async_depth"); QUEUE_RW_ENTRY(queue_ra, "read_ahead_kb"); QUEUE_LIM_RW_ENTRY(queue_max_sectors, "max_sectors_kb"); QUEUE_LIM_RO_ENTRY(queue_max_hw_sectors, "max_hw_sectors_kb"); @@ -719,6 +760,7 @@ static struct attribute *blk_mq_queue_attrs[] = { */ &elv_iosched_entry.attr, &queue_requests_entry.attr, + &queue_async_depth_entry.attr, #ifdef CONFIG_BLK_WBT &queue_wb_lat_entry.attr, #endif diff --git a/block/elevator.c b/block/elevator.c index a2f8b2251dc6..ebe2a1fcf011 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -589,6 +589,7 @@ static int elevator_switch(struct request_queue *q, struct elv_change_ctx *ctx) blk_queue_flag_clear(QUEUE_FLAG_SQ_SCHED, q); q->elevator = NULL; q->nr_requests = q->tag_set->queue_depth; + q->async_depth = q->tag_set->queue_depth; } blk_add_trace_msg(q, "elv switch: %s", ctx->name); -- cgit v1.2.3 From 8cbe62f4d8c37b74947569c7b874848f39f09a22 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Tue, 3 Feb 2026 16:19:46 +0800 Subject: kyber: covert to use request_queue->async_depth Instead of the internal async_depth, remove kqd->async_depth and related helpers. Noted elevator attribute async_depth is now removed, queue attribute with the same name is used instead. Signed-off-by: Yu Kuai Reviewed-by: Nilay Shroff Signed-off-by: Jens Axboe --- block/kyber-iosched.c | 33 +++++---------------------------- 1 file changed, 5 insertions(+), 28 deletions(-) (limited to 'block') diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index 2b3f5b8959af..b84163d1f851 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -47,9 +47,8 @@ enum { * asynchronous requests, we reserve 25% of requests for synchronous * operations. */ - KYBER_ASYNC_PERCENT = 75, + KYBER_DEFAULT_ASYNC_PERCENT = 75, }; - /* * Maximum device-wide depth for each scheduling domain. * @@ -157,9 +156,6 @@ struct kyber_queue_data { */ struct sbitmap_queue domain_tokens[KYBER_NUM_DOMAINS]; - /* Number of allowed async requests. */ - unsigned int async_depth; - struct kyber_cpu_latency __percpu *cpu_latency; /* Timer for stats aggregation and adjusting domain tokens. */ @@ -401,10 +397,7 @@ err: static void kyber_depth_updated(struct request_queue *q) { - struct kyber_queue_data *kqd = q->elevator->elevator_data; - - kqd->async_depth = q->nr_requests * KYBER_ASYNC_PERCENT / 100U; - blk_mq_set_min_shallow_depth(q, kqd->async_depth); + blk_mq_set_min_shallow_depth(q, q->async_depth); } static int kyber_init_sched(struct request_queue *q, struct elevator_queue *eq) @@ -414,6 +407,7 @@ static int kyber_init_sched(struct request_queue *q, struct elevator_queue *eq) blk_queue_flag_clear(QUEUE_FLAG_SQ_SCHED, q); q->elevator = eq; + q->async_depth = q->nr_requests * KYBER_DEFAULT_ASYNC_PERCENT / 100; kyber_depth_updated(q); return 0; @@ -552,15 +546,8 @@ static void rq_clear_domain_token(struct kyber_queue_data *kqd, static void kyber_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) { - /* - * We use the scheduler tags as per-hardware queue queueing tokens. - * Async requests can be limited at this stage. - */ - if (!blk_mq_is_sync_read(opf)) { - struct kyber_queue_data *kqd = data->q->elevator->elevator_data; - - data->shallow_depth = kqd->async_depth; - } + if (!blk_mq_is_sync_read(opf)) + data->shallow_depth = data->q->async_depth; } static bool kyber_bio_merge(struct request_queue *q, struct bio *bio, @@ -956,15 +943,6 @@ KYBER_DEBUGFS_DOMAIN_ATTRS(KYBER_DISCARD, discard) KYBER_DEBUGFS_DOMAIN_ATTRS(KYBER_OTHER, other) #undef KYBER_DEBUGFS_DOMAIN_ATTRS -static int kyber_async_depth_show(void *data, struct seq_file *m) -{ - struct request_queue *q = data; - struct kyber_queue_data *kqd = q->elevator->elevator_data; - - seq_printf(m, "%u\n", kqd->async_depth); - return 0; -} - static int kyber_cur_domain_show(void *data, struct seq_file *m) { struct blk_mq_hw_ctx *hctx = data; @@ -990,7 +968,6 @@ static const struct blk_mq_debugfs_attr kyber_queue_debugfs_attrs[] = { KYBER_QUEUE_DOMAIN_ATTRS(write), KYBER_QUEUE_DOMAIN_ATTRS(discard), KYBER_QUEUE_DOMAIN_ATTRS(other), - {"async_depth", 0400, kyber_async_depth_show}, {}, }; #undef KYBER_QUEUE_DOMAIN_ATTRS -- cgit v1.2.3 From 988bb1b9ededab9aed83df8c1f5be0232b71ded3 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Tue, 3 Feb 2026 16:19:47 +0800 Subject: mq-deadline: covert to use request_queue->async_depth In downstream kernel, we test with mq-deadline with many fio workloads, and we found a performance regression after commit 39823b47bbd4 ("block/mq-deadline: Fix the tag reservation code") with following test: [global] rw=randread direct=1 ramp_time=1 ioengine=libaio iodepth=1024 numjobs=24 bs=1024k group_reporting=1 runtime=60 [job1] filename=/dev/sda Root cause is that mq-deadline now support configuring async_depth, although the default value is nr_request, however the minimal value is 1, hence min_shallow_depth is set to 1, causing wake_batch to be 1. For consequence, sbitmap_queue will be waken up after each IO instead of 8 IO. In this test case, sda is HDD and max_sectors is 128k, hence each submitted 1M io will be splited into 8 sequential 128k requests, however due to there are 24 jobs and total tags are exhausted, the 8 requests are unlikely to be dispatched sequentially, and changing wake_batch to 1 will make this much worse, accounting blktrace D stage, the percentage of sequential io is decreased from 8% to 0.8%. Fix this problem by converting to request_queue->async_depth, where min_shallow_depth is set each time async_depth is updated. Noted elevator attribute async_depth is now removed, queue attribute with the same name is used instead. Fixes: 39823b47bbd4 ("block/mq-deadline: Fix the tag reservation code") Signed-off-by: Yu Kuai Reviewed-by: Nilay Shroff Reviewed-by: Bart Van Assche Signed-off-by: Jens Axboe --- block/mq-deadline.c | 39 +++++---------------------------------- 1 file changed, 5 insertions(+), 34 deletions(-) (limited to 'block') diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 29d00221fbea..95917a88976f 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -98,7 +98,6 @@ struct deadline_data { int fifo_batch; int writes_starved; int front_merges; - u32 async_depth; int prio_aging_expire; spinlock_t lock; @@ -486,32 +485,16 @@ unlock: return rq; } -/* - * Called by __blk_mq_alloc_request(). The shallow_depth value set by this - * function is used by __blk_mq_get_tag(). - */ static void dd_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) { - struct deadline_data *dd = data->q->elevator->elevator_data; - - /* Do not throttle synchronous reads. */ - if (blk_mq_is_sync_read(opf)) - return; - - /* - * Throttle asynchronous requests and writes such that these requests - * do not block the allocation of synchronous requests. - */ - data->shallow_depth = dd->async_depth; + if (!blk_mq_is_sync_read(opf)) + data->shallow_depth = data->q->async_depth; } -/* Called by blk_mq_update_nr_requests(). */ +/* Called by blk_mq_init_sched() and blk_mq_update_nr_requests(). */ static void dd_depth_updated(struct request_queue *q) { - struct deadline_data *dd = q->elevator->elevator_data; - - dd->async_depth = q->nr_requests; - blk_mq_set_min_shallow_depth(q, 1); + blk_mq_set_min_shallow_depth(q, q->async_depth); } static void dd_exit_sched(struct elevator_queue *e) @@ -576,6 +559,7 @@ static int dd_init_sched(struct request_queue *q, struct elevator_queue *eq) blk_queue_flag_set(QUEUE_FLAG_SQ_SCHED, q); q->elevator = eq; + q->async_depth = q->nr_requests; dd_depth_updated(q); return 0; } @@ -763,7 +747,6 @@ SHOW_JIFFIES(deadline_write_expire_show, dd->fifo_expire[DD_WRITE]); SHOW_JIFFIES(deadline_prio_aging_expire_show, dd->prio_aging_expire); SHOW_INT(deadline_writes_starved_show, dd->writes_starved); SHOW_INT(deadline_front_merges_show, dd->front_merges); -SHOW_INT(deadline_async_depth_show, dd->async_depth); SHOW_INT(deadline_fifo_batch_show, dd->fifo_batch); #undef SHOW_INT #undef SHOW_JIFFIES @@ -793,7 +776,6 @@ STORE_JIFFIES(deadline_write_expire_store, &dd->fifo_expire[DD_WRITE], 0, INT_MA STORE_JIFFIES(deadline_prio_aging_expire_store, &dd->prio_aging_expire, 0, INT_MAX); STORE_INT(deadline_writes_starved_store, &dd->writes_starved, INT_MIN, INT_MAX); STORE_INT(deadline_front_merges_store, &dd->front_merges, 0, 1); -STORE_INT(deadline_async_depth_store, &dd->async_depth, 1, INT_MAX); STORE_INT(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX); #undef STORE_FUNCTION #undef STORE_INT @@ -807,7 +789,6 @@ static const struct elv_fs_entry deadline_attrs[] = { DD_ATTR(write_expire), DD_ATTR(writes_starved), DD_ATTR(front_merges), - DD_ATTR(async_depth), DD_ATTR(fifo_batch), DD_ATTR(prio_aging_expire), __ATTR_NULL @@ -894,15 +875,6 @@ static int deadline_starved_show(void *data, struct seq_file *m) return 0; } -static int dd_async_depth_show(void *data, struct seq_file *m) -{ - struct request_queue *q = data; - struct deadline_data *dd = q->elevator->elevator_data; - - seq_printf(m, "%u\n", dd->async_depth); - return 0; -} - static int dd_queued_show(void *data, struct seq_file *m) { struct request_queue *q = data; @@ -1002,7 +974,6 @@ static const struct blk_mq_debugfs_attr deadline_queue_debugfs_attrs[] = { DEADLINE_NEXT_RQ_ATTR(write2), {"batching", 0400, deadline_batching_show}, {"starved", 0400, deadline_starved_show}, - {"async_depth", 0400, dd_async_depth_show}, {"dispatch", 0400, .seq_ops = &deadline_dispatch_seq_ops}, {"owned_by_driver", 0400, dd_owned_by_driver_show}, {"queued", 0400, dd_queued_show}, -- cgit v1.2.3 From 2110858c5178176d0d535b7762b20cb9c0d03146 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Tue, 3 Feb 2026 16:19:48 +0800 Subject: block, bfq: convert to use request_queue->async_depth The default limits is unchanged, and user can configure async_depth now. Signed-off-by: Yu Kuai Reviewed-by: Nilay Shroff Signed-off-by: Jens Axboe --- block/bfq-iosched.c | 43 +++++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 26 deletions(-) (limited to 'block') diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 44746f4c0b89..b180ce583951 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -7112,39 +7112,29 @@ void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg) static void bfq_depth_updated(struct request_queue *q) { struct bfq_data *bfqd = q->elevator->elevator_data; - unsigned int nr_requests = q->nr_requests; + unsigned int async_depth = q->async_depth; /* - * In-word depths if no bfq_queue is being weight-raised: - * leaving 25% of tags only for sync reads. + * By default: + * - sync reads are not limited + * If bfqq is not being weight-raised: + * - sync writes are limited to 75%(async depth default value) + * - async IO are limited to 50% + * If bfqq is being weight-raised: + * - sync writes are limited to ~37% + * - async IO are limited to ~18 * - * In next formulas, right-shift the value - * (1U<sb.shift), instead of computing directly - * (1U<<(bt->sb.shift - something)), to be robust against - * any possible value of bt->sb.shift, without having to - * limit 'something'. + * If request_queue->async_depth is updated by user, all limit are + * updated relatively. */ - /* no more than 50% of tags for async I/O */ - bfqd->async_depths[0][0] = max(nr_requests >> 1, 1U); - /* - * no more than 75% of tags for sync writes (25% extra tags - * w.r.t. async I/O, to prevent async I/O from starving sync - * writes) - */ - bfqd->async_depths[0][1] = max((nr_requests * 3) >> 2, 1U); + bfqd->async_depths[0][1] = async_depth; + bfqd->async_depths[0][0] = max(async_depth * 2 / 3, 1U); + bfqd->async_depths[1][1] = max(async_depth >> 1, 1U); + bfqd->async_depths[1][0] = max(async_depth >> 2, 1U); /* - * In-word depths in case some bfq_queue is being weight- - * raised: leaving ~63% of tags for sync reads. This is the - * highest percentage for which, in our tests, application - * start-up times didn't suffer from any regression due to tag - * shortage. + * Due to cgroup qos, the allowed request for bfqq might be 1 */ - /* no more than ~18% of tags for async I/O */ - bfqd->async_depths[1][0] = max((nr_requests * 3) >> 4, 1U); - /* no more than ~37% of tags for sync writes (~20% extra tags) */ - bfqd->async_depths[1][1] = max((nr_requests * 6) >> 4, 1U); - blk_mq_set_min_shallow_depth(q, 1); } @@ -7365,6 +7355,7 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_queue *eq) blk_queue_flag_set(QUEUE_FLAG_DISABLE_WBT_DEF, q); wbt_disable_default(q->disk); blk_stat_enable_accounting(q); + q->async_depth = (q->nr_requests * 3) >> 2; return 0; -- cgit v1.2.3 From d4d78dd43cecaeaadb24e78927a4d6fda7d1f1d9 Mon Sep 17 00:00:00 2001 From: Yang Xiuwei Date: Tue, 6 Jan 2026 10:42:57 +0800 Subject: block: remove redundant kill_bdev() call in set_blocksize() The second kill_bdev() call in set_blocksize() is redundant as the first call already clears all buffers and pagecache, and locks prevent new pagecache creation between the calls. Signed-off-by: Yang Xiuwei Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/bdev.c | 1 - 1 file changed, 1 deletion(-) (limited to 'block') diff --git a/block/bdev.c b/block/bdev.c index b8fbb9576110..ed022f8c48c7 100644 --- a/block/bdev.c +++ b/block/bdev.c @@ -208,7 +208,6 @@ int set_blocksize(struct file *file, int size) inode->i_blkbits = blksize_bits(size); mapping_set_folio_min_order(inode->i_mapping, get_order(size)); - kill_bdev(bdev); filemap_invalidate_unlock(inode->i_mapping); inode_unlock(inode); } -- cgit v1.2.3 From ee81212f74a57c5d2b56cf504f40d528dac6faaf Mon Sep 17 00:00:00 2001 From: Luke Wang Date: Wed, 4 Feb 2026 11:40:02 +0800 Subject: block: decouple secure erase size limit from discard size limit Secure erase should use max_secure_erase_sectors instead of being limited by max_discard_sectors. Separate the handling of REQ_OP_SECURE_ERASE from REQ_OP_DISCARD to allow each operation to use its own size limit. Signed-off-by: Luke Wang Reviewed-by: Ulf Hansson Signed-off-by: Jens Axboe --- block/blk-merge.c | 21 +++++++++++++++++---- block/blk.h | 6 +++++- 2 files changed, 22 insertions(+), 5 deletions(-) (limited to 'block') diff --git a/block/blk-merge.c b/block/blk-merge.c index b82c6d304658..c18bc440d647 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -158,8 +158,9 @@ static struct bio *bio_submit_split(struct bio *bio, int split_sectors) return bio; } -struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, - unsigned *nsegs) +static struct bio *__bio_split_discard(struct bio *bio, + const struct queue_limits *lim, unsigned *nsegs, + unsigned int max_sectors) { unsigned int max_discard_sectors, granularity; sector_t tmp; @@ -169,8 +170,7 @@ struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, granularity = max(lim->discard_granularity >> 9, 1U); - max_discard_sectors = - min(lim->max_discard_sectors, bio_allowed_max_sectors(lim)); + max_discard_sectors = min(max_sectors, bio_allowed_max_sectors(lim)); max_discard_sectors -= max_discard_sectors % granularity; if (unlikely(!max_discard_sectors)) return bio; @@ -194,6 +194,19 @@ struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, return bio_submit_split(bio, split_sectors); } +struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, + unsigned *nsegs) +{ + unsigned int max_sectors; + + if (bio_op(bio) == REQ_OP_SECURE_ERASE) + max_sectors = lim->max_secure_erase_sectors; + else + max_sectors = lim->max_discard_sectors; + + return __bio_split_discard(bio, lim, nsegs, max_sectors); +} + static inline unsigned int blk_boundary_sectors(const struct queue_limits *lim, bool is_atomic) { diff --git a/block/blk.h b/block/blk.h index 980eef1f5690..401d19ed08a6 100644 --- a/block/blk.h +++ b/block/blk.h @@ -208,10 +208,14 @@ static inline unsigned int blk_queue_get_max_sectors(struct request *rq) struct request_queue *q = rq->q; enum req_op op = req_op(rq); - if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE)) + if (unlikely(op == REQ_OP_DISCARD)) return min(q->limits.max_discard_sectors, UINT_MAX >> SECTOR_SHIFT); + if (unlikely(op == REQ_OP_SECURE_ERASE)) + return min(q->limits.max_secure_erase_sectors, + UINT_MAX >> SECTOR_SHIFT); + if (unlikely(op == REQ_OP_WRITE_ZEROES)) return q->limits.max_write_zeroes_sectors; -- cgit v1.2.3