block/blk-lib.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Functions related to generic helpers functions
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/module.h>
   7 #include <linux/bio.h>
   8 #include <linux/blkdev.h>
   9 #include <linux/scatterlist.h>
  10
  11 #include "blk.h"
  12
  13 static sector_t bio_discard_limit(struct block_device *bdev, sector_t sector)
  14 {
  15         unsigned int discard_granularity = bdev_discard_granularity(bdev);
  16         sector_t granularity_aligned_sector;
  17
  18         if (bdev_is_partition(bdev))
  19                 sector += bdev->bd_start_sect;
  20
  21         granularity_aligned_sector =
  22                 round_up(sector, discard_granularity >> SECTOR_SHIFT);
  23
  24         /*
  25          * Make sure subsequent bios start aligned to the discard granularity if
  26          * it needs to be split.
  27          */
  28         if (granularity_aligned_sector != sector)
  29                 return granularity_aligned_sector - sector;
  30
  31         /*
  32          * Align the bio size to the discard granularity to make splitting the bio
  33          * at discard granularity boundaries easier in the driver if needed.
  34          */
  35         return round_down(UINT_MAX, discard_granularity) >> SECTOR_SHIFT;
  36 }
  37
  38 int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
  39                 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop)
  40 {
  41         struct bio *bio = *biop;
  42         sector_t bs_mask;
  43
  44         if (bdev_read_only(bdev))
  45                 return -EPERM;
  46         if (!bdev_max_discard_sectors(bdev))
  47                 return -EOPNOTSUPP;
  48
  49         /* In case the discard granularity isn't set by buggy device driver */
  50         if (WARN_ON_ONCE(!bdev_discard_granularity(bdev))) {
  51                 pr_err_ratelimited("%pg: Error: discard_granularity is 0.\n",
  52                                    bdev);
  53                 return -EOPNOTSUPP;
  54         }
  55
  56         bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
  57         if ((sector | nr_sects) & bs_mask)
  58                 return -EINVAL;
  59
  60         if (!nr_sects)
  61                 return -EINVAL;
  62
  63         while (nr_sects) {
  64                 sector_t req_sects =
  65                         min(nr_sects, bio_discard_limit(bdev, sector));
  66
  67                 bio = blk_next_bio(bio, bdev, 0, REQ_OP_DISCARD, gfp_mask);
  68                 bio->bi_iter.bi_sector = sector;
  69                 bio->bi_iter.bi_size = req_sects << 9;
  70                 sector += req_sects;
  71                 nr_sects -= req_sects;
  72
  73                 /*
  74                  * We can loop for a long time in here, if someone does
  75                  * full device discards (like mkfs). Be nice and allow
  76                  * us to schedule out to avoid softlocking if preempt
  77                  * is disabled.
  78                  */
  79                 cond_resched();
  80         }
  81
  82         *biop = bio;
  83         return 0;
  84 }
  85 EXPORT_SYMBOL(__blkdev_issue_discard);
  86
  87 /**
  88  * blkdev_issue_discard - queue a discard
  89  * @bdev:       blockdev to issue discard for
  90  * @sector:     start sector
  91  * @nr_sects:   number of sectors to discard
  92  * @gfp_mask:   memory allocation flags (for bio_alloc)
  93  *
  94  * Description:
  95  *    Issue a discard request for the sectors in question.
  96  */
  97 int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
  98                 sector_t nr_sects, gfp_t gfp_mask)
  99 {
 100         struct bio *bio = NULL;
 101         struct blk_plug plug;
 102         int ret;
 103
 104         blk_start_plug(&plug);
 105         ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, &bio);
 106         if (!ret && bio) {
 107                 ret = submit_bio_wait(bio);
 108                 if (ret == -EOPNOTSUPP)
 109                         ret = 0;
 110                 bio_put(bio);
 111         }
 112         blk_finish_plug(&plug);
 113
 114         return ret;
 115 }
 116 EXPORT_SYMBOL(blkdev_issue_discard);
 117
 118 static int __blkdev_issue_write_zeroes(struct block_device *bdev,
 119                 sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
 120                 struct bio **biop, unsigned flags)
 121 {
 122         struct bio *bio = *biop;
 123         unsigned int max_write_zeroes_sectors;
 124
 125         if (bdev_read_only(bdev))
 126                 return -EPERM;
 127
 128         /* Ensure that max_write_zeroes_sectors doesn't overflow bi_size */
 129         max_write_zeroes_sectors = bdev_write_zeroes_sectors(bdev);
 130
 131         if (max_write_zeroes_sectors == 0)
 132                 return -EOPNOTSUPP;
 133
 134         while (nr_sects) {
 135                 bio = blk_next_bio(bio, bdev, 0, REQ_OP_WRITE_ZEROES, gfp_mask);
 136                 bio->bi_iter.bi_sector = sector;
 137                 if (flags & BLKDEV_ZERO_NOUNMAP)
 138                         bio->bi_opf |= REQ_NOUNMAP;
 139
 140                 if (nr_sects > max_write_zeroes_sectors) {
 141                         bio->bi_iter.bi_size = max_write_zeroes_sectors << 9;
 142                         nr_sects -= max_write_zeroes_sectors;
 143                         sector += max_write_zeroes_sectors;
 144                 } else {
 145                         bio->bi_iter.bi_size = nr_sects << 9;
 146                         nr_sects = 0;
 147                 }
 148                 cond_resched();
 149         }
 150
 151         *biop = bio;
 152         return 0;
 153 }
 154
 155 /*
 156  * Convert a number of 512B sectors to a number of pages.
 157  * The result is limited to a number of pages that can fit into a BIO.
 158  * Also make sure that the result is always at least 1 (page) for the cases
 159  * where nr_sects is lower than the number of sectors in a page.
 160  */
 161 static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects)
 162 {
 163         sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512);
 164
 165         return min(pages, (sector_t)BIO_MAX_VECS);
 166 }
 167
 168 static int __blkdev_issue_zero_pages(struct block_device *bdev,
 169                 sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
 170                 struct bio **biop)
 171 {
 172         struct bio *bio = *biop;
 173         int bi_size = 0;
 174         unsigned int sz;
 175
 176         if (bdev_read_only(bdev))
 177                 return -EPERM;
 178
 179         while (nr_sects != 0) {
 180                 bio = blk_next_bio(bio, bdev, __blkdev_sectors_to_bio_pages(nr_sects),
 181                                    REQ_OP_WRITE, gfp_mask);
 182                 bio->bi_iter.bi_sector = sector;
 183
 184                 while (nr_sects != 0) {
 185                         sz = min((sector_t) PAGE_SIZE, nr_sects << 9);
 186                         bi_size = bio_add_page(bio, ZERO_PAGE(0), sz, 0);
 187                         nr_sects -= bi_size >> 9;
 188                         sector += bi_size >> 9;
 189                         if (bi_size < sz)
 190                                 break;
 191                 }
 192                 cond_resched();
 193         }
 194
 195         *biop = bio;
 196         return 0;
 197 }
 198
 199 /**
 200  * __blkdev_issue_zeroout - generate number of zero filed write bios
 201  * @bdev:       blockdev to issue
 202  * @sector:     start sector
 203  * @nr_sects:   number of sectors to write
 204  * @gfp_mask:   memory allocation flags (for bio_alloc)
 205  * @biop:       pointer to anchor bio
 206  * @flags:      controls detailed behavior
 207  *
 208  * Description:
 209  *  Zero-fill a block range, either using hardware offload or by explicitly
 210  *  writing zeroes to the device.
 211  *
 212  *  If a device is using logical block provisioning, the underlying space will
 213  *  not be released if %flags contains BLKDEV_ZERO_NOUNMAP.
 214  *
 215  *  If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return
 216  *  -EOPNOTSUPP if no explicit hardware offload for zeroing is provided.
 217  */
 218 int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
 219                 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
 220                 unsigned flags)
 221 {
 222         int ret;
 223         sector_t bs_mask;
 224
 225         bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
 226         if ((sector | nr_sects) & bs_mask)
 227                 return -EINVAL;
 228
 229         ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp_mask,
 230                         biop, flags);
 231         if (ret != -EOPNOTSUPP || (flags & BLKDEV_ZERO_NOFALLBACK))
 232                 return ret;
 233
 234         return __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask,
 235                                          biop);
 236 }
 237 EXPORT_SYMBOL(__blkdev_issue_zeroout);
 238
 239 /**
 240  * blkdev_issue_zeroout - zero-fill a block range
 241  * @bdev:       blockdev to write
 242  * @sector:     start sector
 243  * @nr_sects:   number of sectors to write
 244  * @gfp_mask:   memory allocation flags (for bio_alloc)
 245  * @flags:      controls detailed behavior
 246  *
 247  * Description:
 248  *  Zero-fill a block range, either using hardware offload or by explicitly
 249  *  writing zeroes to the device.  See __blkdev_issue_zeroout() for the
 250  *  valid values for %flags.
 251  */
 252 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
 253                 sector_t nr_sects, gfp_t gfp_mask, unsigned flags)
 254 {
 255         int ret = 0;
 256         sector_t bs_mask;
 257         struct bio *bio;
 258         struct blk_plug plug;
 259         bool try_write_zeroes = !!bdev_write_zeroes_sectors(bdev);
 260
 261         bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
 262         if ((sector | nr_sects) & bs_mask)
 263                 return -EINVAL;
 264
 265 retry:
 266         bio = NULL;
 267         blk_start_plug(&plug);
 268         if (try_write_zeroes) {
 269                 ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects,
 270                                                   gfp_mask, &bio, flags);
 271         } else if (!(flags & BLKDEV_ZERO_NOFALLBACK)) {
 272                 ret = __blkdev_issue_zero_pages(bdev, sector, nr_sects,
 273                                                 gfp_mask, &bio);
 274         } else {
 275                 /* No zeroing offload support */
 276                 ret = -EOPNOTSUPP;
 277         }
 278         if (ret == 0 && bio) {
 279                 ret = submit_bio_wait(bio);
 280                 bio_put(bio);
 281         }
 282         blk_finish_plug(&plug);
 283         if (ret && try_write_zeroes) {
 284                 if (!(flags & BLKDEV_ZERO_NOFALLBACK)) {
 285                         try_write_zeroes = false;
 286                         goto retry;
 287                 }
 288                 if (!bdev_write_zeroes_sectors(bdev)) {
 289                         /*
 290                          * Zeroing offload support was indicated, but the
 291                          * device reported ILLEGAL REQUEST (for some devices
 292                          * there is no non-destructive way to verify whether
 293                          * WRITE ZEROES is actually supported).
 294                          */
 295                         ret = -EOPNOTSUPP;
 296                 }
 297         }
 298
 299         return ret;
 300 }
 301 EXPORT_SYMBOL(blkdev_issue_zeroout);
 302
 303 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector,
 304                 sector_t nr_sects, gfp_t gfp)
 305 {
 306         sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
 307         unsigned int max_sectors = bdev_max_secure_erase_sectors(bdev);
 308         struct bio *bio = NULL;
 309         struct blk_plug plug;
 310         int ret = 0;
 311
 312         if (max_sectors == 0)
 313                 return -EOPNOTSUPP;
 314         if ((sector | nr_sects) & bs_mask)
 315                 return -EINVAL;
 316         if (bdev_read_only(bdev))
 317                 return -EPERM;
 318
 319         blk_start_plug(&plug);
 320         for (;;) {
 321                 unsigned int len = min_t(sector_t, nr_sects, max_sectors);
 322
 323                 bio = blk_next_bio(bio, bdev, 0, REQ_OP_SECURE_ERASE, gfp);
 324                 bio->bi_iter.bi_sector = sector;
 325                 bio->bi_iter.bi_size = len;
 326
 327                 sector += len << SECTOR_SHIFT;
 328                 nr_sects -= len << SECTOR_SHIFT;
 329                 if (!nr_sects) {
 330                         ret = submit_bio_wait(bio);
 331                         bio_put(bio);
 332                         break;
 333                 }
 334                 cond_resched();
 335         }
 336         blk_finish_plug(&plug);
 337
 338         return ret;
 339 }
 340 EXPORT_SYMBOL(blkdev_issue_secure_erase);