1 // SPDX-License-Identifier: GPL-2.0
3 * Functions related to generic helpers functions
5 #include <linux/kernel.h>
6 #include <linux/module.h>
8 #include <linux/blkdev.h>
9 #include <linux/scatterlist.h>
13 static sector_t
bio_discard_limit(struct block_device
*bdev
, sector_t sector
)
15 unsigned int discard_granularity
= bdev_discard_granularity(bdev
);
16 sector_t granularity_aligned_sector
;
18 if (bdev_is_partition(bdev
))
19 sector
+= bdev
->bd_start_sect
;
21 granularity_aligned_sector
=
22 round_up(sector
, discard_granularity
>> SECTOR_SHIFT
);
25 * Make sure subsequent bios start aligned to the discard granularity if
26 * it needs to be split.
28 if (granularity_aligned_sector
!= sector
)
29 return granularity_aligned_sector
- sector
;
32 * Align the bio size to the discard granularity to make splitting the bio
33 * at discard granularity boundaries easier in the driver if needed.
35 return round_down(UINT_MAX
, discard_granularity
) >> SECTOR_SHIFT
;
38 int __blkdev_issue_discard(struct block_device
*bdev
, sector_t sector
,
39 sector_t nr_sects
, gfp_t gfp_mask
, struct bio
**biop
)
41 struct bio
*bio
= *biop
;
44 if (bdev_read_only(bdev
))
46 if (!bdev_max_discard_sectors(bdev
))
49 /* In case the discard granularity isn't set by buggy device driver */
50 if (WARN_ON_ONCE(!bdev_discard_granularity(bdev
))) {
51 pr_err_ratelimited("%pg: Error: discard_granularity is 0.\n",
56 bs_mask
= (bdev_logical_block_size(bdev
) >> 9) - 1;
57 if ((sector
| nr_sects
) & bs_mask
)
65 min(nr_sects
, bio_discard_limit(bdev
, sector
));
67 bio
= blk_next_bio(bio
, bdev
, 0, REQ_OP_DISCARD
, gfp_mask
);
68 bio
->bi_iter
.bi_sector
= sector
;
69 bio
->bi_iter
.bi_size
= req_sects
<< 9;
71 nr_sects
-= req_sects
;
74 * We can loop for a long time in here, if someone does
75 * full device discards (like mkfs). Be nice and allow
76 * us to schedule out to avoid softlocking if preempt
85 EXPORT_SYMBOL(__blkdev_issue_discard
);
88 * blkdev_issue_discard - queue a discard
89 * @bdev: blockdev to issue discard for
90 * @sector: start sector
91 * @nr_sects: number of sectors to discard
92 * @gfp_mask: memory allocation flags (for bio_alloc)
95 * Issue a discard request for the sectors in question.
97 int blkdev_issue_discard(struct block_device
*bdev
, sector_t sector
,
98 sector_t nr_sects
, gfp_t gfp_mask
)
100 struct bio
*bio
= NULL
;
101 struct blk_plug plug
;
104 blk_start_plug(&plug
);
105 ret
= __blkdev_issue_discard(bdev
, sector
, nr_sects
, gfp_mask
, &bio
);
107 ret
= submit_bio_wait(bio
);
108 if (ret
== -EOPNOTSUPP
)
112 blk_finish_plug(&plug
);
116 EXPORT_SYMBOL(blkdev_issue_discard
);
118 static int __blkdev_issue_write_zeroes(struct block_device
*bdev
,
119 sector_t sector
, sector_t nr_sects
, gfp_t gfp_mask
,
120 struct bio
**biop
, unsigned flags
)
122 struct bio
*bio
= *biop
;
123 unsigned int max_write_zeroes_sectors
;
125 if (bdev_read_only(bdev
))
128 /* Ensure that max_write_zeroes_sectors doesn't overflow bi_size */
129 max_write_zeroes_sectors
= bdev_write_zeroes_sectors(bdev
);
131 if (max_write_zeroes_sectors
== 0)
135 bio
= blk_next_bio(bio
, bdev
, 0, REQ_OP_WRITE_ZEROES
, gfp_mask
);
136 bio
->bi_iter
.bi_sector
= sector
;
137 if (flags
& BLKDEV_ZERO_NOUNMAP
)
138 bio
->bi_opf
|= REQ_NOUNMAP
;
140 if (nr_sects
> max_write_zeroes_sectors
) {
141 bio
->bi_iter
.bi_size
= max_write_zeroes_sectors
<< 9;
142 nr_sects
-= max_write_zeroes_sectors
;
143 sector
+= max_write_zeroes_sectors
;
145 bio
->bi_iter
.bi_size
= nr_sects
<< 9;
156 * Convert a number of 512B sectors to a number of pages.
157 * The result is limited to a number of pages that can fit into a BIO.
158 * Also make sure that the result is always at least 1 (page) for the cases
159 * where nr_sects is lower than the number of sectors in a page.
161 static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects
)
163 sector_t pages
= DIV_ROUND_UP_SECTOR_T(nr_sects
, PAGE_SIZE
/ 512);
165 return min(pages
, (sector_t
)BIO_MAX_VECS
);
168 static int __blkdev_issue_zero_pages(struct block_device
*bdev
,
169 sector_t sector
, sector_t nr_sects
, gfp_t gfp_mask
,
172 struct bio
*bio
= *biop
;
176 if (bdev_read_only(bdev
))
179 while (nr_sects
!= 0) {
180 bio
= blk_next_bio(bio
, bdev
, __blkdev_sectors_to_bio_pages(nr_sects
),
181 REQ_OP_WRITE
, gfp_mask
);
182 bio
->bi_iter
.bi_sector
= sector
;
184 while (nr_sects
!= 0) {
185 sz
= min((sector_t
) PAGE_SIZE
, nr_sects
<< 9);
186 bi_size
= bio_add_page(bio
, ZERO_PAGE(0), sz
, 0);
187 nr_sects
-= bi_size
>> 9;
188 sector
+= bi_size
>> 9;
200 * __blkdev_issue_zeroout - generate number of zero filed write bios
201 * @bdev: blockdev to issue
202 * @sector: start sector
203 * @nr_sects: number of sectors to write
204 * @gfp_mask: memory allocation flags (for bio_alloc)
205 * @biop: pointer to anchor bio
206 * @flags: controls detailed behavior
209 * Zero-fill a block range, either using hardware offload or by explicitly
210 * writing zeroes to the device.
212 * If a device is using logical block provisioning, the underlying space will
213 * not be released if %flags contains BLKDEV_ZERO_NOUNMAP.
215 * If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return
216 * -EOPNOTSUPP if no explicit hardware offload for zeroing is provided.
218 int __blkdev_issue_zeroout(struct block_device
*bdev
, sector_t sector
,
219 sector_t nr_sects
, gfp_t gfp_mask
, struct bio
**biop
,
225 bs_mask
= (bdev_logical_block_size(bdev
) >> 9) - 1;
226 if ((sector
| nr_sects
) & bs_mask
)
229 ret
= __blkdev_issue_write_zeroes(bdev
, sector
, nr_sects
, gfp_mask
,
231 if (ret
!= -EOPNOTSUPP
|| (flags
& BLKDEV_ZERO_NOFALLBACK
))
234 return __blkdev_issue_zero_pages(bdev
, sector
, nr_sects
, gfp_mask
,
237 EXPORT_SYMBOL(__blkdev_issue_zeroout
);
240 * blkdev_issue_zeroout - zero-fill a block range
241 * @bdev: blockdev to write
242 * @sector: start sector
243 * @nr_sects: number of sectors to write
244 * @gfp_mask: memory allocation flags (for bio_alloc)
245 * @flags: controls detailed behavior
248 * Zero-fill a block range, either using hardware offload or by explicitly
249 * writing zeroes to the device. See __blkdev_issue_zeroout() for the
250 * valid values for %flags.
252 int blkdev_issue_zeroout(struct block_device
*bdev
, sector_t sector
,
253 sector_t nr_sects
, gfp_t gfp_mask
, unsigned flags
)
258 struct blk_plug plug
;
259 bool try_write_zeroes
= !!bdev_write_zeroes_sectors(bdev
);
261 bs_mask
= (bdev_logical_block_size(bdev
) >> 9) - 1;
262 if ((sector
| nr_sects
) & bs_mask
)
267 blk_start_plug(&plug
);
268 if (try_write_zeroes
) {
269 ret
= __blkdev_issue_write_zeroes(bdev
, sector
, nr_sects
,
270 gfp_mask
, &bio
, flags
);
271 } else if (!(flags
& BLKDEV_ZERO_NOFALLBACK
)) {
272 ret
= __blkdev_issue_zero_pages(bdev
, sector
, nr_sects
,
275 /* No zeroing offload support */
278 if (ret
== 0 && bio
) {
279 ret
= submit_bio_wait(bio
);
282 blk_finish_plug(&plug
);
283 if (ret
&& try_write_zeroes
) {
284 if (!(flags
& BLKDEV_ZERO_NOFALLBACK
)) {
285 try_write_zeroes
= false;
288 if (!bdev_write_zeroes_sectors(bdev
)) {
290 * Zeroing offload support was indicated, but the
291 * device reported ILLEGAL REQUEST (for some devices
292 * there is no non-destructive way to verify whether
293 * WRITE ZEROES is actually supported).
301 EXPORT_SYMBOL(blkdev_issue_zeroout
);
303 int blkdev_issue_secure_erase(struct block_device
*bdev
, sector_t sector
,
304 sector_t nr_sects
, gfp_t gfp
)
306 sector_t bs_mask
= (bdev_logical_block_size(bdev
) >> 9) - 1;
307 unsigned int max_sectors
= bdev_max_secure_erase_sectors(bdev
);
308 struct bio
*bio
= NULL
;
309 struct blk_plug plug
;
312 /* make sure that "len << SECTOR_SHIFT" doesn't overflow */
313 if (max_sectors
> UINT_MAX
>> SECTOR_SHIFT
)
314 max_sectors
= UINT_MAX
>> SECTOR_SHIFT
;
315 max_sectors
&= ~bs_mask
;
317 if (max_sectors
== 0)
319 if ((sector
| nr_sects
) & bs_mask
)
321 if (bdev_read_only(bdev
))
324 blk_start_plug(&plug
);
326 unsigned int len
= min_t(sector_t
, nr_sects
, max_sectors
);
328 bio
= blk_next_bio(bio
, bdev
, 0, REQ_OP_SECURE_ERASE
, gfp
);
329 bio
->bi_iter
.bi_sector
= sector
;
330 bio
->bi_iter
.bi_size
= len
<< SECTOR_SHIFT
;
335 ret
= submit_bio_wait(bio
);
341 blk_finish_plug(&plug
);
345 EXPORT_SYMBOL(blkdev_issue_secure_erase
);