From fa0bdd45d7e3703826ea75f5fe3359865d75c319 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 26 Jan 2026 06:53:32 +0100 Subject: block: add a BIO_MAX_SIZE constant and use it Currently the only constant for the maximum bio size is BIO_MAX_SECTORS, which is in units of 512-byte sectors, but a lot of user need a byte limit. Add a BIO_MAX_SIZE constant, redefine BIO_MAX_SECTORS in terms of it, and switch all bio-related uses of UINT_MAX for the maximum size to use the symbolic names instead. Signed-off-by: Christoph Hellwig Reviewed-by: Martin K. Petersen Reviewed-by: Darrick J. Wong Reviewed-by: Anuj Gupta Reviewed-by: Johannes Thumshirn Reviewed-by: Damien Le Moal Signed-off-by: Jens Axboe --- block/bio.c | 10 +++++----- block/blk-lib.c | 9 ++++----- block/blk-merge.c | 8 ++++---- 3 files changed, 13 insertions(+), 14 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index 2359c0723b88..ac7703e149c6 100644 --- a/block/bio.c +++ b/block/bio.c @@ -924,7 +924,7 @@ static inline bool bio_full(struct bio *bio, unsigned len) { if (bio->bi_vcnt >= bio->bi_max_vecs) return true; - if (bio->bi_iter.bi_size > UINT_MAX - len) + if (bio->bi_iter.bi_size > BIO_MAX_SIZE - len) return true; return false; } @@ -1030,7 +1030,7 @@ int bio_add_page(struct bio *bio, struct page *page, { if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return 0; - if (bio->bi_iter.bi_size > UINT_MAX - len) + if (bio->bi_iter.bi_size > BIO_MAX_SIZE - len) return 0; if (bio->bi_vcnt > 0) { @@ -1057,7 +1057,7 @@ void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len, { unsigned long nr = off / PAGE_SIZE; - WARN_ON_ONCE(len > UINT_MAX); + WARN_ON_ONCE(len > BIO_MAX_SIZE); __bio_add_page(bio, folio_page(folio, nr), len, off % PAGE_SIZE); } EXPORT_SYMBOL_GPL(bio_add_folio_nofail); @@ -1081,7 +1081,7 @@ bool bio_add_folio(struct bio *bio, struct folio *folio, size_t len, { unsigned long nr = off / PAGE_SIZE; - if (len > UINT_MAX) + if (len > BIO_MAX_SIZE) return false; return bio_add_page(bio, folio_page(folio, nr), len, off % PAGE_SIZE) > 0; } @@ -1238,7 +1238,7 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) extraction_flags |= ITER_ALLOW_P2PDMA; size = iov_iter_extract_pages(iter, &pages, - UINT_MAX - bio->bi_iter.bi_size, + BIO_MAX_SIZE - bio->bi_iter.bi_size, nr_pages, extraction_flags, &offset); if (unlikely(size <= 0)) return size ? size : -EFAULT; diff --git a/block/blk-lib.c b/block/blk-lib.c index 9e2cc58f881f..0be3acdc3eb5 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -32,7 +32,7 @@ static sector_t bio_discard_limit(struct block_device *bdev, sector_t sector) * Align the bio size to the discard granularity to make splitting the bio * at discard granularity boundaries easier in the driver if needed. */ - return round_down(UINT_MAX, discard_granularity) >> SECTOR_SHIFT; + return round_down(BIO_MAX_SIZE, discard_granularity) >> SECTOR_SHIFT; } struct bio *blk_alloc_discard_bio(struct block_device *bdev, @@ -107,8 +107,7 @@ static sector_t bio_write_zeroes_limit(struct block_device *bdev) { sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; - return min(bdev_write_zeroes_sectors(bdev), - (UINT_MAX >> SECTOR_SHIFT) & ~bs_mask); + return min(bdev_write_zeroes_sectors(bdev), BIO_MAX_SECTORS & ~bs_mask); } /* @@ -337,8 +336,8 @@ int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector, int ret = 0; /* make sure that "len << SECTOR_SHIFT" doesn't overflow */ - if (max_sectors > UINT_MAX >> SECTOR_SHIFT) - max_sectors = UINT_MAX >> SECTOR_SHIFT; + if (max_sectors > BIO_MAX_SECTORS) + max_sectors = BIO_MAX_SECTORS; max_sectors &= ~bs_mask; if (max_sectors == 0) diff --git a/block/blk-merge.c b/block/blk-merge.c index b82c6d304658..0eb0aef97197 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -95,13 +95,13 @@ static inline bool req_gap_front_merge(struct request *req, struct bio *bio) } /* - * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size - * is defined as 'unsigned int', meantime it has to be aligned to with the + * The maximum size that a bio can fit has to be aligned down to the * logical block size, which is the minimum accepted unit by hardware. */ static unsigned int bio_allowed_max_sectors(const struct queue_limits *lim) { - return round_down(UINT_MAX, lim->logical_block_size) >> SECTOR_SHIFT; + return round_down(BIO_MAX_SIZE, lim->logical_block_size) >> + SECTOR_SHIFT; } /* @@ -502,7 +502,7 @@ unsigned int blk_recalc_rq_segments(struct request *rq) rq_for_each_bvec(bv, rq, iter) bvec_split_segs(&rq->q->limits, &bv, &nr_phys_segs, &bytes, - UINT_MAX, UINT_MAX); + UINT_MAX, BIO_MAX_SIZE); return nr_phys_segs; } -- cgit v1.2.3 From 4d77007d42fd4f44c2f5a1555603df53e16a1362 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 26 Jan 2026 06:53:33 +0100 Subject: block: refactor get_contig_folio_len Move all of the logic to find the contigous length inside a folio into get_contig_folio_len instead of keeping some of it in the caller. Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Reviewed-by: Johannes Thumshirn Reviewed-by: Darrick J. Wong Tested-by: Anuj Gupta Reviewed-by: Martin K. Petersen Reviewed-by: Anuj Gupta Signed-off-by: Jens Axboe --- block/bio.c | 62 +++++++++++++++++++++++++++---------------------------------- 1 file changed, 27 insertions(+), 35 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index ac7703e149c6..d633e80d821f 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1172,33 +1172,35 @@ void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter) bio_set_flag(bio, BIO_CLONED); } -static unsigned int get_contig_folio_len(unsigned int *num_pages, - struct page **pages, unsigned int i, - struct folio *folio, size_t left, +static unsigned int get_contig_folio_len(struct page **pages, + unsigned int *num_pages, size_t left, size_t offset) { - size_t bytes = left; - size_t contig_sz = min_t(size_t, PAGE_SIZE - offset, bytes); - unsigned int j; + struct folio *folio = page_folio(pages[0]); + size_t contig_sz = min_t(size_t, PAGE_SIZE - offset, left); + unsigned int max_pages, i; + size_t folio_offset, len; + + folio_offset = PAGE_SIZE * folio_page_idx(folio, pages[0]) + offset; + len = min(folio_size(folio) - folio_offset, left); /* - * We might COW a single page in the middle of - * a large folio, so we have to check that all - * pages belong to the same folio. + * We might COW a single page in the middle of a large folio, so we have + * to check that all pages belong to the same folio. */ - bytes -= contig_sz; - for (j = i + 1; j < i + *num_pages; j++) { - size_t next = min_t(size_t, PAGE_SIZE, bytes); + left -= contig_sz; + max_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE); + for (i = 1; i < max_pages; i++) { + size_t next = min_t(size_t, PAGE_SIZE, left); - if (page_folio(pages[j]) != folio || - pages[j] != pages[j - 1] + 1) { + if (page_folio(pages[i]) != folio || + pages[i] != pages[i - 1] + 1) break; - } contig_sz += next; - bytes -= next; + left -= next; } - *num_pages = j - i; + *num_pages = i; return contig_sz; } @@ -1222,8 +1224,8 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; struct page **pages = (struct page **)bv; ssize_t size; - unsigned int num_pages, i = 0; - size_t offset, folio_offset, left, len; + unsigned int i = 0; + size_t offset, left, len; int ret = 0; /* @@ -1244,23 +1246,12 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) return size ? size : -EFAULT; nr_pages = DIV_ROUND_UP(offset + size, PAGE_SIZE); - for (left = size, i = 0; left > 0; left -= len, i += num_pages) { - struct page *page = pages[i]; - struct folio *folio = page_folio(page); + for (left = size; left > 0; left -= len) { unsigned int old_vcnt = bio->bi_vcnt; + unsigned int nr_to_add; - folio_offset = ((size_t)folio_page_idx(folio, page) << - PAGE_SHIFT) + offset; - - len = min(folio_size(folio) - folio_offset, left); - - num_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE); - - if (num_pages > 1) - len = get_contig_folio_len(&num_pages, pages, i, - folio, left, offset); - - if (!bio_add_folio(bio, folio, len, folio_offset)) { + len = get_contig_folio_len(&pages[i], &nr_to_add, left, offset); + if (!bio_add_page(bio, pages[i], len, offset)) { WARN_ON_ONCE(1); ret = -EINVAL; goto out; @@ -1275,8 +1266,9 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) * single pin per page. */ if (offset && bio->bi_vcnt == old_vcnt) - unpin_user_folio(folio, 1); + unpin_user_folio(page_folio(pages[i]), 1); } + i += nr_to_add; offset = 0; } -- cgit v1.2.3 From 12da89e8844ae16e86b75a32b34a4f0b0525f453 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 26 Jan 2026 06:53:34 +0100 Subject: block: open code bio_add_page and fix handling of mismatching P2P ranges bio_add_page fails to add data to the bio when mixing P2P with non-P2P ranges, or ranges that map to different P2P providers. In that case it will trigger that WARN_ON and return an error up the chain instead of simply starting a new bio as intended. Fix this by open coding bio_add_page and handling this case explicitly. While doing so, stop merging physical contiguous data that belongs to multiple folios. While this merge could lead to more efficient bio packing in some case, dropping will allow to remove handling of this corner case in other places and make the code more robust. Signed-off-by: Christoph Hellwig Reviewed-by: Anuj Gupta Reviewed-by: Damien Le Moal Reviewed-by: Johannes Thumshirn Reviewed-by: Darrick J. Wong Tested-by: Anuj Gupta Reviewed-by: Martin K. Petersen Signed-off-by: Jens Axboe --- block/bio.c | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index d633e80d821f..4591f0ba90f5 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1216,7 +1216,7 @@ static unsigned int get_contig_folio_len(struct page **pages, * For a multi-segment *iter, this function only adds pages from the next * non-empty segment of the iov iterator. */ -static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) +static ssize_t __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) { iov_iter_extraction_t extraction_flags = 0; unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; @@ -1226,7 +1226,6 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) ssize_t size; unsigned int i = 0; size_t offset, left, len; - int ret = 0; /* * Move page array up in the allocated memory for the bio vecs as far as @@ -1247,37 +1246,26 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) nr_pages = DIV_ROUND_UP(offset + size, PAGE_SIZE); for (left = size; left > 0; left -= len) { - unsigned int old_vcnt = bio->bi_vcnt; unsigned int nr_to_add; - len = get_contig_folio_len(&pages[i], &nr_to_add, left, offset); - if (!bio_add_page(bio, pages[i], len, offset)) { - WARN_ON_ONCE(1); - ret = -EINVAL; - goto out; - } + if (bio->bi_vcnt > 0) { + struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1]; - if (bio_flagged(bio, BIO_PAGE_PINNED)) { - /* - * We're adding another fragment of a page that already - * was part of the last segment. Undo our pin as the - * page was pinned when an earlier fragment of it was - * added to the bio and __bio_release_pages expects a - * single pin per page. - */ - if (offset && bio->bi_vcnt == old_vcnt) - unpin_user_folio(page_folio(pages[i]), 1); + if (!zone_device_pages_have_same_pgmap(prev->bv_page, + pages[i])) + break; } + + len = get_contig_folio_len(&pages[i], &nr_to_add, left, offset); + __bio_add_page(bio, pages[i], len, offset); i += nr_to_add; offset = 0; } iov_iter_revert(iter, left); -out: while (i < nr_pages) bio_release_page(bio, pages[i++]); - - return ret; + return size - left; } /* @@ -1337,7 +1325,7 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, unsigned len_align_mask) { - int ret = 0; + ssize_t ret; if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return -EIO; @@ -1350,9 +1338,10 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, if (iov_iter_extract_will_pin(iter)) bio_set_flag(bio, BIO_PAGE_PINNED); + do { ret = __bio_iov_iter_get_pages(bio, iter); - } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0)); + } while (ret > 0 && iov_iter_count(iter) && !bio_full(bio, 0)); if (bio->bi_vcnt) return bio_iov_iter_align_down(bio, iter, len_align_mask); -- cgit v1.2.3 From 91b73c458182801a8c9cf6135335e064567d1013 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 26 Jan 2026 06:53:35 +0100 Subject: iov_iter: extract a iov_iter_extract_bvecs helper from bio code Massage __bio_iov_iter_get_pages so that it doesn't need the bio, and move it to lib/iov_iter.c so that it can be used by block code for other things than filling a bio and by other subsystems like netfs. Signed-off-by: Christoph Hellwig Reviewed-by: Martin K. Petersen Signed-off-by: Jens Axboe --- block/bio.c | 120 +++++++++--------------------------------------------------- 1 file changed, 18 insertions(+), 102 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index 4591f0ba90f5..530082c8cf0c 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1172,102 +1172,6 @@ void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter) bio_set_flag(bio, BIO_CLONED); } -static unsigned int get_contig_folio_len(struct page **pages, - unsigned int *num_pages, size_t left, - size_t offset) -{ - struct folio *folio = page_folio(pages[0]); - size_t contig_sz = min_t(size_t, PAGE_SIZE - offset, left); - unsigned int max_pages, i; - size_t folio_offset, len; - - folio_offset = PAGE_SIZE * folio_page_idx(folio, pages[0]) + offset; - len = min(folio_size(folio) - folio_offset, left); - - /* - * We might COW a single page in the middle of a large folio, so we have - * to check that all pages belong to the same folio. - */ - left -= contig_sz; - max_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE); - for (i = 1; i < max_pages; i++) { - size_t next = min_t(size_t, PAGE_SIZE, left); - - if (page_folio(pages[i]) != folio || - pages[i] != pages[i - 1] + 1) - break; - contig_sz += next; - left -= next; - } - - *num_pages = i; - return contig_sz; -} - -#define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *)) - -/** - * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio - * @bio: bio to add pages to - * @iter: iov iterator describing the region to be mapped - * - * Extracts pages from *iter and appends them to @bio's bvec array. The pages - * will have to be cleaned up in the way indicated by the BIO_PAGE_PINNED flag. - * For a multi-segment *iter, this function only adds pages from the next - * non-empty segment of the iov iterator. - */ -static ssize_t __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) -{ - iov_iter_extraction_t extraction_flags = 0; - unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; - unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt; - struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; - struct page **pages = (struct page **)bv; - ssize_t size; - unsigned int i = 0; - size_t offset, left, len; - - /* - * Move page array up in the allocated memory for the bio vecs as far as - * possible so that we can start filling biovecs from the beginning - * without overwriting the temporary page array. - */ - BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2); - pages += entries_left * (PAGE_PTRS_PER_BVEC - 1); - - if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue)) - extraction_flags |= ITER_ALLOW_P2PDMA; - - size = iov_iter_extract_pages(iter, &pages, - BIO_MAX_SIZE - bio->bi_iter.bi_size, - nr_pages, extraction_flags, &offset); - if (unlikely(size <= 0)) - return size ? size : -EFAULT; - - nr_pages = DIV_ROUND_UP(offset + size, PAGE_SIZE); - for (left = size; left > 0; left -= len) { - unsigned int nr_to_add; - - if (bio->bi_vcnt > 0) { - struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1]; - - if (!zone_device_pages_have_same_pgmap(prev->bv_page, - pages[i])) - break; - } - - len = get_contig_folio_len(&pages[i], &nr_to_add, left, offset); - __bio_add_page(bio, pages[i], len, offset); - i += nr_to_add; - offset = 0; - } - - iov_iter_revert(iter, left); - while (i < nr_pages) - bio_release_page(bio, pages[i++]); - return size - left; -} - /* * Aligns the bio size to the len_align_mask, releasing excessive bio vecs that * __bio_iov_iter_get_pages may have inserted, and reverts the trimmed length @@ -1325,7 +1229,7 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, unsigned len_align_mask) { - ssize_t ret; + iov_iter_extraction_t flags = 0; if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return -EIO; @@ -1338,14 +1242,26 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, if (iov_iter_extract_will_pin(iter)) bio_set_flag(bio, BIO_PAGE_PINNED); + if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue)) + flags |= ITER_ALLOW_P2PDMA; do { - ret = __bio_iov_iter_get_pages(bio, iter); - } while (ret > 0 && iov_iter_count(iter) && !bio_full(bio, 0)); + ssize_t ret; + + ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec, + BIO_MAX_SIZE - bio->bi_iter.bi_size, + &bio->bi_vcnt, bio->bi_max_vecs, flags); + if (ret <= 0) { + if (!bio->bi_vcnt) + return ret; + break; + } + bio->bi_iter.bi_size += ret; + } while (iov_iter_count(iter) && !bio_full(bio, 0)); - if (bio->bi_vcnt) - return bio_iov_iter_align_down(bio, iter, len_align_mask); - return ret; + if (is_pci_p2pdma_page(bio->bi_io_vec->bv_page)) + bio->bi_opf |= REQ_NOMERGE; + return bio_iov_iter_align_down(bio, iter, len_align_mask); } static void submit_bio_wait_endio(struct bio *bio) -- cgit v1.2.3 From 301f5356521ed90f72a67797156d75093aac786f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 26 Jan 2026 06:53:36 +0100 Subject: block: remove bio_release_page Merge bio_release_page into the only remaining caller. Signed-off-by: Christoph Hellwig Reviewed-by: Anuj Gupta Reviewed-by: Damien Le Moal Reviewed-by: Johannes Thumshirn Reviewed-by: Darrick J. Wong Tested-by: Anuj Gupta Reviewed-by: Martin K. Petersen Signed-off-by: Jens Axboe --- block/bio.c | 4 +++- block/blk.h | 11 ----------- 2 files changed, 3 insertions(+), 12 deletions(-) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index 530082c8cf0c..285b573ae82f 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1195,7 +1195,9 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, break; } - bio_release_page(bio, bv->bv_page); + if (bio_flagged(bio, BIO_PAGE_PINNED)) + unpin_user_page(bv->bv_page); + bio->bi_vcnt--; nbytes -= bv->bv_len; } while (nbytes); diff --git a/block/blk.h b/block/blk.h index 980eef1f5690..886238cae5f1 100644 --- a/block/blk.h +++ b/block/blk.h @@ -595,17 +595,6 @@ void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors); struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id, struct lock_class_key *lkclass); - -/* - * Clean up a page appropriately, where the page may be pinned, may have a - * ref taken on it or neither. - */ -static inline void bio_release_page(struct bio *bio, struct page *page) -{ - if (bio_flagged(bio, BIO_PAGE_PINNED)) - unpin_user_page(page); -} - struct request_queue *blk_alloc_queue(struct queue_limits *lim, int node_id); int disk_scan_partitions(struct gendisk *disk, blk_mode_t mode); -- cgit v1.2.3 From 8dd5e7c75d7bb2635c7efd219ff20693fc24096a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 26 Jan 2026 06:53:37 +0100 Subject: block: add helpers to bounce buffer an iov_iter into bios Add helpers to implement bounce buffering of data into a bio to implement direct I/O for cases where direct user access is not possible because stable in-flight data is required. These are intended to be used as easily as bio_iov_iter_get_pages for the zero-copy path. The write side is trivial and just copies data into the bounce buffer. The read side is a lot more complex because it needs to perform the copy from the completion context, and without preserving the iov_iter through the call chain. It steals a trick from the integrity data user interface and uses the first vector in the bio for the bounce buffer data that is fed to the block I/O stack, and uses the others to record the user buffer fragments. Signed-off-by: Christoph Hellwig Reviewed-by: Anuj Gupta Reviewed-by: Damien Le Moal Reviewed-by: Johannes Thumshirn Tested-by: Anuj Gupta Reviewed-by: Martin K. Petersen Reviewed-by: Darrick J. Wong Signed-off-by: Jens Axboe --- block/bio.c | 179 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) (limited to 'block') diff --git a/block/bio.c b/block/bio.c index 285b573ae82f..49f7548a31d6 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1266,6 +1266,185 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, return bio_iov_iter_align_down(bio, iter, len_align_mask); } +static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size) +{ + struct folio *folio; + + while (*size > PAGE_SIZE) { + folio = folio_alloc(gfp | __GFP_NORETRY, get_order(*size)); + if (folio) + return folio; + *size = rounddown_pow_of_two(*size - 1); + } + + return folio_alloc(gfp, get_order(*size)); +} + +static void bio_free_folios(struct bio *bio) +{ + struct bio_vec *bv; + int i; + + bio_for_each_bvec_all(bv, bio, i) { + struct folio *folio = page_folio(bv->bv_page); + + if (!is_zero_folio(folio)) + folio_put(folio); + } +} + +static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter) +{ + size_t total_len = iov_iter_count(iter); + + if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) + return -EINVAL; + if (WARN_ON_ONCE(bio->bi_iter.bi_size)) + return -EINVAL; + if (WARN_ON_ONCE(bio->bi_vcnt >= bio->bi_max_vecs)) + return -EINVAL; + + do { + size_t this_len = min(total_len, SZ_1M); + struct folio *folio; + + if (this_len > PAGE_SIZE * 2) + this_len = rounddown_pow_of_two(this_len); + + if (bio->bi_iter.bi_size > BIO_MAX_SIZE - this_len) + break; + + folio = folio_alloc_greedy(GFP_KERNEL, &this_len); + if (!folio) + break; + bio_add_folio_nofail(bio, folio, this_len, 0); + + if (copy_from_iter(folio_address(folio), this_len, iter) != + this_len) { + bio_free_folios(bio); + return -EFAULT; + } + + total_len -= this_len; + } while (total_len && bio->bi_vcnt < bio->bi_max_vecs); + + if (!bio->bi_iter.bi_size) + return -ENOMEM; + return 0; +} + +static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter) +{ + size_t len = min(iov_iter_count(iter), SZ_1M); + struct folio *folio; + + folio = folio_alloc_greedy(GFP_KERNEL, &len); + if (!folio) + return -ENOMEM; + + do { + ssize_t ret; + + ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec + 1, len, + &bio->bi_vcnt, bio->bi_max_vecs - 1, 0); + if (ret <= 0) { + if (!bio->bi_vcnt) + return ret; + break; + } + len -= ret; + bio->bi_iter.bi_size += ret; + } while (len && bio->bi_vcnt < bio->bi_max_vecs - 1); + + /* + * Set the folio directly here. The above loop has already calculated + * the correct bi_size, and we use bi_vcnt for the user buffers. That + * is safe as bi_vcnt is only used by the submitter and not the actual + * I/O path. + */ + bvec_set_folio(&bio->bi_io_vec[0], folio, bio->bi_iter.bi_size, 0); + if (iov_iter_extract_will_pin(iter)) + bio_set_flag(bio, BIO_PAGE_PINNED); + return 0; +} + +/** + * bio_iov_iter_bounce - bounce buffer data from an iter into a bio + * @bio: bio to send + * @iter: iter to read from / write into + * + * Helper for direct I/O implementations that need to bounce buffer because + * we need to checksum the data or perform other operations that require + * consistency. Allocates folios to back the bounce buffer, and for writes + * copies the data into it. Needs to be paired with bio_iov_iter_unbounce() + * called on completion. + */ +int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter) +{ + if (op_is_write(bio_op(bio))) + return bio_iov_iter_bounce_write(bio, iter); + return bio_iov_iter_bounce_read(bio, iter); +} + +static void bvec_unpin(struct bio_vec *bv, bool mark_dirty) +{ + struct folio *folio = page_folio(bv->bv_page); + size_t nr_pages = (bv->bv_offset + bv->bv_len - 1) / PAGE_SIZE - + bv->bv_offset / PAGE_SIZE + 1; + + if (mark_dirty) + folio_mark_dirty_lock(folio); + unpin_user_folio(folio, nr_pages); +} + +static void bio_iov_iter_unbounce_read(struct bio *bio, bool is_error, + bool mark_dirty) +{ + unsigned int len = bio->bi_io_vec[0].bv_len; + + if (likely(!is_error)) { + void *buf = bvec_virt(&bio->bi_io_vec[0]); + struct iov_iter to; + + iov_iter_bvec(&to, ITER_DEST, bio->bi_io_vec + 1, bio->bi_vcnt, + len); + /* copying to pinned pages should always work */ + WARN_ON_ONCE(copy_to_iter(buf, len, &to) != len); + } else { + /* No need to mark folios dirty if never copied to them */ + mark_dirty = false; + } + + if (bio_flagged(bio, BIO_PAGE_PINNED)) { + int i; + + for (i = 0; i < bio->bi_vcnt; i++) + bvec_unpin(&bio->bi_io_vec[1 + i], mark_dirty); + } + + folio_put(page_folio(bio->bi_io_vec[0].bv_page)); +} + +/** + * bio_iov_iter_unbounce - finish a bounce buffer operation + * @bio: completed bio + * @is_error: %true if an I/O error occurred and data should not be copied + * @mark_dirty: If %true, folios will be marked dirty. + * + * Helper for direct I/O implementations that need to bounce buffer because + * we need to checksum the data or perform other operations that require + * consistency. Called to complete a bio set up by bio_iov_iter_bounce(). + * Copies data back for reads, and marks the original folios dirty if + * requested and then frees the bounce buffer. + */ +void bio_iov_iter_unbounce(struct bio *bio, bool is_error, bool mark_dirty) +{ + if (op_is_write(bio_op(bio))) + bio_free_folios(bio); + else + bio_iov_iter_unbounce_read(bio, is_error, mark_dirty); +} + static void submit_bio_wait_endio(struct bio *bio) { complete(bio->bi_private); -- cgit v1.2.3