1 // SPDX-License-Identifier: GPL-2.0
5 * Portions Copyright (C) 2020 Christoph Hellwig
8 #include <linux/module.h>
9 #include <linux/ctype.h>
11 #include <linux/kdev_t.h>
12 #include <linux/kernel.h>
13 #include <linux/blkdev.h>
14 #include <linux/backing-dev.h>
15 #include <linux/init.h>
16 #include <linux/spinlock.h>
17 #include <linux/proc_fs.h>
18 #include <linux/seq_file.h>
19 #include <linux/slab.h>
20 #include <linux/kmod.h>
21 #include <linux/major.h>
22 #include <linux/mutex.h>
23 #include <linux/idr.h>
24 #include <linux/log2.h>
25 #include <linux/pm_runtime.h>
26 #include <linux/badblocks.h>
27 #include <linux/part_stat.h>
28 #include <linux/blktrace_api.h>
30 #include "blk-throttle.h"
32 #include "blk-mq-sched.h"
33 #include "blk-rq-qos.h"
34 #include "blk-cgroup.h"
36 static struct kobject
*block_depr
;
39 * Unique, monotonically increasing sequential number associated with block
40 * devices instances (i.e. incremented each time a device is attached).
41 * Associating uevents with block devices in userspace is difficult and racy:
42 * the uevent netlink socket is lossy, and on slow and overloaded systems has
43 * a very high latency.
44 * Block devices do not have exclusive owners in userspace, any process can set
45 * one up (e.g. loop devices). Moreover, device names can be reused (e.g. loop0
46 * can be reused again and again).
47 * A userspace process setting up a block device and watching for its events
48 * cannot thus reliably tell whether an event relates to the device it just set
49 * up or another earlier instance with the same name.
50 * This sequential number allows userspace processes to solve this problem, and
51 * uniquely associate an uevent to the lifetime to a device.
53 static atomic64_t diskseq
;
55 /* for extended dynamic devt allocation, currently only one major is used */
56 #define NR_EXT_DEVT (1 << MINORBITS)
57 static DEFINE_IDA(ext_devt_ida
);
59 void set_capacity(struct gendisk
*disk
, sector_t sectors
)
61 bdev_set_nr_sectors(disk
->part0
, sectors
);
63 EXPORT_SYMBOL(set_capacity
);
66 * Set disk capacity and notify if the size is not currently zero and will not
67 * be set to zero. Returns true if a uevent was sent, otherwise false.
69 bool set_capacity_and_notify(struct gendisk
*disk
, sector_t size
)
71 sector_t capacity
= get_capacity(disk
);
72 char *envp
[] = { "RESIZE=1", NULL
};
74 set_capacity(disk
, size
);
77 * Only print a message and send a uevent if the gendisk is user visible
78 * and alive. This avoids spamming the log and udev when setting the
79 * initial capacity during probing.
81 if (size
== capacity
||
83 (disk
->flags
& GENHD_FL_HIDDEN
))
86 pr_info("%s: detected capacity change from %lld to %lld\n",
87 disk
->disk_name
, capacity
, size
);
90 * Historically we did not send a uevent for changes to/from an empty
93 if (!capacity
|| !size
)
95 kobject_uevent_env(&disk_to_dev(disk
)->kobj
, KOBJ_CHANGE
, envp
);
98 EXPORT_SYMBOL_GPL(set_capacity_and_notify
);
100 static void part_stat_read_all(struct block_device
*part
,
101 struct disk_stats
*stat
)
105 memset(stat
, 0, sizeof(struct disk_stats
));
106 for_each_possible_cpu(cpu
) {
107 struct disk_stats
*ptr
= per_cpu_ptr(part
->bd_stats
, cpu
);
110 for (group
= 0; group
< NR_STAT_GROUPS
; group
++) {
111 stat
->nsecs
[group
] += ptr
->nsecs
[group
];
112 stat
->sectors
[group
] += ptr
->sectors
[group
];
113 stat
->ios
[group
] += ptr
->ios
[group
];
114 stat
->merges
[group
] += ptr
->merges
[group
];
117 stat
->io_ticks
+= ptr
->io_ticks
;
121 static unsigned int part_in_flight(struct block_device
*part
)
123 unsigned int inflight
= 0;
126 for_each_possible_cpu(cpu
) {
127 inflight
+= part_stat_local_read_cpu(part
, in_flight
[0], cpu
) +
128 part_stat_local_read_cpu(part
, in_flight
[1], cpu
);
130 if ((int)inflight
< 0)
136 static void part_in_flight_rw(struct block_device
*part
,
137 unsigned int inflight
[2])
143 for_each_possible_cpu(cpu
) {
144 inflight
[0] += part_stat_local_read_cpu(part
, in_flight
[0], cpu
);
145 inflight
[1] += part_stat_local_read_cpu(part
, in_flight
[1], cpu
);
147 if ((int)inflight
[0] < 0)
149 if ((int)inflight
[1] < 0)
154 * Can be deleted altogether. Later.
157 #define BLKDEV_MAJOR_HASH_SIZE 255
158 static struct blk_major_name
{
159 struct blk_major_name
*next
;
162 #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
163 void (*probe
)(dev_t devt
);
165 } *major_names
[BLKDEV_MAJOR_HASH_SIZE
];
166 static DEFINE_MUTEX(major_names_lock
);
167 static DEFINE_SPINLOCK(major_names_spinlock
);
169 /* index in the above - for now: assume no multimajor ranges */
170 static inline int major_to_index(unsigned major
)
172 return major
% BLKDEV_MAJOR_HASH_SIZE
;
175 #ifdef CONFIG_PROC_FS
176 void blkdev_show(struct seq_file
*seqf
, off_t offset
)
178 struct blk_major_name
*dp
;
180 spin_lock(&major_names_spinlock
);
181 for (dp
= major_names
[major_to_index(offset
)]; dp
; dp
= dp
->next
)
182 if (dp
->major
== offset
)
183 seq_printf(seqf
, "%3d %s\n", dp
->major
, dp
->name
);
184 spin_unlock(&major_names_spinlock
);
186 #endif /* CONFIG_PROC_FS */
189 * __register_blkdev - register a new block device
191 * @major: the requested major device number [1..BLKDEV_MAJOR_MAX-1]. If
192 * @major = 0, try to allocate any unused major number.
193 * @name: the name of the new block device as a zero terminated string
194 * @probe: pre-devtmpfs / pre-udev callback used to create disks when their
195 * pre-created device node is accessed. When a probe call uses
196 * add_disk() and it fails the driver must cleanup resources. This
197 * interface may soon be removed.
199 * The @name must be unique within the system.
201 * The return value depends on the @major input parameter:
203 * - if a major device number was requested in range [1..BLKDEV_MAJOR_MAX-1]
204 * then the function returns zero on success, or a negative error code
205 * - if any unused major number was requested with @major = 0 parameter
206 * then the return value is the allocated major number in range
207 * [1..BLKDEV_MAJOR_MAX-1] or a negative error code otherwise
209 * See Documentation/admin-guide/devices.txt for the list of allocated
212 * Use register_blkdev instead for any new code.
214 int __register_blkdev(unsigned int major
, const char *name
,
215 void (*probe
)(dev_t devt
))
217 struct blk_major_name
**n
, *p
;
220 mutex_lock(&major_names_lock
);
224 for (index
= ARRAY_SIZE(major_names
)-1; index
> 0; index
--) {
225 if (major_names
[index
] == NULL
)
230 printk("%s: failed to get major for %s\n",
239 if (major
>= BLKDEV_MAJOR_MAX
) {
240 pr_err("%s: major requested (%u) is greater than the maximum (%u) for %s\n",
241 __func__
, major
, BLKDEV_MAJOR_MAX
-1, name
);
247 p
= kmalloc(sizeof(struct blk_major_name
), GFP_KERNEL
);
254 #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
257 strscpy(p
->name
, name
, sizeof(p
->name
));
259 index
= major_to_index(major
);
261 spin_lock(&major_names_spinlock
);
262 for (n
= &major_names
[index
]; *n
; n
= &(*n
)->next
) {
263 if ((*n
)->major
== major
)
270 spin_unlock(&major_names_spinlock
);
273 printk("register_blkdev: cannot get major %u for %s\n",
278 mutex_unlock(&major_names_lock
);
281 EXPORT_SYMBOL(__register_blkdev
);
283 void unregister_blkdev(unsigned int major
, const char *name
)
285 struct blk_major_name
**n
;
286 struct blk_major_name
*p
= NULL
;
287 int index
= major_to_index(major
);
289 mutex_lock(&major_names_lock
);
290 spin_lock(&major_names_spinlock
);
291 for (n
= &major_names
[index
]; *n
; n
= &(*n
)->next
)
292 if ((*n
)->major
== major
)
294 if (!*n
|| strcmp((*n
)->name
, name
)) {
300 spin_unlock(&major_names_spinlock
);
301 mutex_unlock(&major_names_lock
);
305 EXPORT_SYMBOL(unregister_blkdev
);
307 int blk_alloc_ext_minor(void)
311 idx
= ida_alloc_range(&ext_devt_ida
, 0, NR_EXT_DEVT
- 1, GFP_KERNEL
);
317 void blk_free_ext_minor(unsigned int minor
)
319 ida_free(&ext_devt_ida
, minor
);
322 void disk_uevent(struct gendisk
*disk
, enum kobject_action action
)
324 struct block_device
*part
;
328 xa_for_each(&disk
->part_tbl
, idx
, part
) {
329 if (bdev_is_partition(part
) && !bdev_nr_sectors(part
))
331 if (!kobject_get_unless_zero(&part
->bd_device
.kobj
))
335 kobject_uevent(bdev_kobj(part
), action
);
336 put_device(&part
->bd_device
);
341 EXPORT_SYMBOL_GPL(disk_uevent
);
343 int disk_scan_partitions(struct gendisk
*disk
, blk_mode_t mode
)
345 struct bdev_handle
*handle
;
348 if (disk
->flags
& (GENHD_FL_NO_PART
| GENHD_FL_HIDDEN
))
350 if (test_bit(GD_SUPPRESS_PART_SCAN
, &disk
->state
))
352 if (disk
->open_partitions
)
356 * If the device is opened exclusively by current thread already, it's
357 * safe to scan partitons, otherwise, use bd_prepare_to_claim() to
358 * synchronize with other exclusive openers and other partition
361 if (!(mode
& BLK_OPEN_EXCL
)) {
362 ret
= bd_prepare_to_claim(disk
->part0
, disk_scan_partitions
,
368 set_bit(GD_NEED_PART_SCAN
, &disk
->state
);
369 handle
= bdev_open_by_dev(disk_devt(disk
), mode
& ~BLK_OPEN_EXCL
, NULL
,
372 ret
= PTR_ERR(handle
);
374 bdev_release(handle
);
377 * If blkdev_get_by_dev() failed early, GD_NEED_PART_SCAN is still set,
378 * and this will cause that re-assemble partitioned raid device will
379 * creat partition for underlying disk.
381 clear_bit(GD_NEED_PART_SCAN
, &disk
->state
);
382 if (!(mode
& BLK_OPEN_EXCL
))
383 bd_abort_claiming(disk
->part0
, disk_scan_partitions
);
388 * device_add_disk - add disk information to kernel list
389 * @parent: parent device for the disk
390 * @disk: per-device partitioning information
391 * @groups: Additional per-device sysfs groups
393 * This function registers the partitioning information in @disk
396 int __must_check
device_add_disk(struct device
*parent
, struct gendisk
*disk
,
397 const struct attribute_group
**groups
)
400 struct device
*ddev
= disk_to_dev(disk
);
403 /* Only makes sense for bio-based to set ->poll_bio */
404 if (queue_is_mq(disk
->queue
) && disk
->fops
->poll_bio
)
408 * The disk queue should now be all set with enough information about
409 * the device for the elevator code to pick an adequate default
410 * elevator if one is needed, that is, for devices requesting queue
413 elevator_init_mq(disk
->queue
);
415 /* Mark bdev as having a submit_bio, if needed */
416 disk
->part0
->bd_has_submit_bio
= disk
->fops
->submit_bio
!= NULL
;
419 * If the driver provides an explicit major number it also must provide
420 * the number of minors numbers supported, and those will be used to
422 * Otherwise just allocate the device numbers for both the whole device
423 * and all partitions from the extended dev_t space.
427 if (WARN_ON(!disk
->minors
))
428 goto out_exit_elevator
;
430 if (disk
->minors
> DISK_MAX_PARTS
) {
431 pr_err("block: can't allocate more than %d partitions\n",
433 disk
->minors
= DISK_MAX_PARTS
;
435 if (disk
->first_minor
+ disk
->minors
> MINORMASK
+ 1)
436 goto out_exit_elevator
;
438 if (WARN_ON(disk
->minors
))
439 goto out_exit_elevator
;
441 ret
= blk_alloc_ext_minor();
443 goto out_exit_elevator
;
444 disk
->major
= BLOCK_EXT_MAJOR
;
445 disk
->first_minor
= ret
;
448 /* delay uevents, until we scanned partition table */
449 dev_set_uevent_suppress(ddev
, 1);
451 ddev
->parent
= parent
;
452 ddev
->groups
= groups
;
453 dev_set_name(ddev
, "%s", disk
->disk_name
);
454 if (!(disk
->flags
& GENHD_FL_HIDDEN
))
455 ddev
->devt
= MKDEV(disk
->major
, disk
->first_minor
);
456 ret
= device_add(ddev
);
458 goto out_free_ext_minor
;
460 ret
= disk_alloc_events(disk
);
464 ret
= sysfs_create_link(block_depr
, &ddev
->kobj
,
465 kobject_name(&ddev
->kobj
));
470 * avoid probable deadlock caused by allocating memory with
471 * GFP_KERNEL in runtime_resume callback of its all ancestor
474 pm_runtime_set_memalloc_noio(ddev
, true);
476 disk
->part0
->bd_holder_dir
=
477 kobject_create_and_add("holders", &ddev
->kobj
);
478 if (!disk
->part0
->bd_holder_dir
) {
480 goto out_del_block_link
;
482 disk
->slave_dir
= kobject_create_and_add("slaves", &ddev
->kobj
);
483 if (!disk
->slave_dir
) {
485 goto out_put_holder_dir
;
488 ret
= blk_register_queue(disk
);
490 goto out_put_slave_dir
;
492 if (!(disk
->flags
& GENHD_FL_HIDDEN
)) {
493 ret
= bdi_register(disk
->bdi
, "%u:%u",
494 disk
->major
, disk
->first_minor
);
496 goto out_unregister_queue
;
497 bdi_set_owner(disk
->bdi
, ddev
);
498 ret
= sysfs_create_link(&ddev
->kobj
,
499 &disk
->bdi
->dev
->kobj
, "bdi");
501 goto out_unregister_bdi
;
503 /* Make sure the first partition scan will be proceed */
504 if (get_capacity(disk
) && !(disk
->flags
& GENHD_FL_NO_PART
) &&
505 !test_bit(GD_SUPPRESS_PART_SCAN
, &disk
->state
))
506 set_bit(GD_NEED_PART_SCAN
, &disk
->state
);
508 bdev_add(disk
->part0
, ddev
->devt
);
509 if (get_capacity(disk
))
510 disk_scan_partitions(disk
, BLK_OPEN_READ
);
513 * Announce the disk and partitions after all partitions are
514 * created. (for hidden disks uevents remain suppressed forever)
516 dev_set_uevent_suppress(ddev
, 0);
517 disk_uevent(disk
, KOBJ_ADD
);
520 * Even if the block_device for a hidden gendisk is not
521 * registered, it needs to have a valid bd_dev so that the
522 * freeing of the dynamic major works.
524 disk
->part0
->bd_dev
= MKDEV(disk
->major
, disk
->first_minor
);
527 disk_update_readahead(disk
);
528 disk_add_events(disk
);
529 set_bit(GD_ADDED
, &disk
->state
);
533 if (!(disk
->flags
& GENHD_FL_HIDDEN
))
534 bdi_unregister(disk
->bdi
);
535 out_unregister_queue
:
536 blk_unregister_queue(disk
);
537 rq_qos_exit(disk
->queue
);
539 kobject_put(disk
->slave_dir
);
540 disk
->slave_dir
= NULL
;
542 kobject_put(disk
->part0
->bd_holder_dir
);
544 sysfs_remove_link(block_depr
, dev_name(ddev
));
548 if (disk
->major
== BLOCK_EXT_MAJOR
)
549 blk_free_ext_minor(disk
->first_minor
);
551 if (disk
->queue
->elevator
)
552 elevator_exit(disk
->queue
);
555 EXPORT_SYMBOL(device_add_disk
);
557 static void blk_report_disk_dead(struct gendisk
*disk
, bool surprise
)
559 struct block_device
*bdev
;
563 * On surprise disk removal, bdev_mark_dead() may call into file
564 * systems below. Make it clear that we're expecting to not hold
567 lockdep_assert_not_held(&disk
->open_mutex
);
570 xa_for_each(&disk
->part_tbl
, idx
, bdev
) {
571 if (!kobject_get_unless_zero(&bdev
->bd_device
.kobj
))
575 bdev_mark_dead(bdev
, surprise
);
577 put_device(&bdev
->bd_device
);
583 static void __blk_mark_disk_dead(struct gendisk
*disk
)
588 if (test_and_set_bit(GD_DEAD
, &disk
->state
))
591 if (test_bit(GD_OWNS_QUEUE
, &disk
->state
))
592 blk_queue_flag_set(QUEUE_FLAG_DYING
, disk
->queue
);
595 * Stop buffered writers from dirtying pages that can't be written out.
597 set_capacity(disk
, 0);
600 * Prevent new I/O from crossing bio_queue_enter().
602 blk_queue_start_drain(disk
->queue
);
606 * blk_mark_disk_dead - mark a disk as dead
607 * @disk: disk to mark as dead
609 * Mark as disk as dead (e.g. surprise removed) and don't accept any new I/O
612 void blk_mark_disk_dead(struct gendisk
*disk
)
614 __blk_mark_disk_dead(disk
);
615 blk_report_disk_dead(disk
, true);
617 EXPORT_SYMBOL_GPL(blk_mark_disk_dead
);
620 * del_gendisk - remove the gendisk
621 * @disk: the struct gendisk to remove
623 * Removes the gendisk and all its associated resources. This deletes the
624 * partitions associated with the gendisk, and unregisters the associated
627 * This is the counter to the respective __device_add_disk() call.
629 * The final removal of the struct gendisk happens when its refcount reaches 0
630 * with put_disk(), which should be called after del_gendisk(), if
631 * __device_add_disk() was used.
633 * Drivers exist which depend on the release of the gendisk to be synchronous,
634 * it should not be deferred.
638 void del_gendisk(struct gendisk
*disk
)
640 struct request_queue
*q
= disk
->queue
;
641 struct block_device
*part
;
646 if (WARN_ON_ONCE(!disk_live(disk
) && !(disk
->flags
& GENHD_FL_HIDDEN
)))
649 disk_del_events(disk
);
652 * Prevent new openers by unlinked the bdev inode.
654 mutex_lock(&disk
->open_mutex
);
655 xa_for_each(&disk
->part_tbl
, idx
, part
)
656 remove_inode_hash(part
->bd_inode
);
657 mutex_unlock(&disk
->open_mutex
);
660 * Tell the file system to write back all dirty data and shut down if
661 * it hasn't been notified earlier.
663 if (!test_bit(GD_DEAD
, &disk
->state
))
664 blk_report_disk_dead(disk
, false);
665 __blk_mark_disk_dead(disk
);
668 * Drop all partitions now that the disk is marked dead.
670 mutex_lock(&disk
->open_mutex
);
671 xa_for_each_start(&disk
->part_tbl
, idx
, part
, 1)
672 drop_partition(part
);
673 mutex_unlock(&disk
->open_mutex
);
675 if (!(disk
->flags
& GENHD_FL_HIDDEN
)) {
676 sysfs_remove_link(&disk_to_dev(disk
)->kobj
, "bdi");
679 * Unregister bdi before releasing device numbers (as they can
680 * get reused and we'd get clashes in sysfs).
682 bdi_unregister(disk
->bdi
);
685 blk_unregister_queue(disk
);
687 kobject_put(disk
->part0
->bd_holder_dir
);
688 kobject_put(disk
->slave_dir
);
689 disk
->slave_dir
= NULL
;
691 part_stat_set_all(disk
->part0
, 0);
692 disk
->part0
->bd_stamp
= 0;
693 sysfs_remove_link(block_depr
, dev_name(disk_to_dev(disk
)));
694 pm_runtime_set_memalloc_noio(disk_to_dev(disk
), false);
695 device_del(disk_to_dev(disk
));
697 blk_mq_freeze_queue_wait(q
);
699 blk_throtl_cancel_bios(disk
);
702 blk_flush_integrity();
705 blk_mq_cancel_work_sync(q
);
707 blk_mq_quiesce_queue(q
);
709 mutex_lock(&q
->sysfs_lock
);
711 mutex_unlock(&q
->sysfs_lock
);
714 blk_mq_unquiesce_queue(q
);
717 * If the disk does not own the queue, allow using passthrough requests
718 * again. Else leave the queue frozen to fail all I/O.
720 if (!test_bit(GD_OWNS_QUEUE
, &disk
->state
)) {
721 blk_queue_flag_clear(QUEUE_FLAG_INIT_DONE
, q
);
722 __blk_mq_unfreeze_queue(q
, true);
725 blk_mq_exit_queue(q
);
728 EXPORT_SYMBOL(del_gendisk
);
731 * invalidate_disk - invalidate the disk
732 * @disk: the struct gendisk to invalidate
734 * A helper to invalidates the disk. It will clean the disk's associated
735 * buffer/page caches and reset its internal states so that the disk
736 * can be reused by the drivers.
740 void invalidate_disk(struct gendisk
*disk
)
742 struct block_device
*bdev
= disk
->part0
;
744 invalidate_bdev(bdev
);
745 bdev
->bd_inode
->i_mapping
->wb_err
= 0;
746 set_capacity(disk
, 0);
748 EXPORT_SYMBOL(invalidate_disk
);
750 /* sysfs access to bad-blocks list. */
751 static ssize_t
disk_badblocks_show(struct device
*dev
,
752 struct device_attribute
*attr
,
755 struct gendisk
*disk
= dev_to_disk(dev
);
758 return sprintf(page
, "\n");
760 return badblocks_show(disk
->bb
, page
, 0);
763 static ssize_t
disk_badblocks_store(struct device
*dev
,
764 struct device_attribute
*attr
,
765 const char *page
, size_t len
)
767 struct gendisk
*disk
= dev_to_disk(dev
);
772 return badblocks_store(disk
->bb
, page
, len
, 0);
775 #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
776 void blk_request_module(dev_t devt
)
778 unsigned int major
= MAJOR(devt
);
779 struct blk_major_name
**n
;
781 mutex_lock(&major_names_lock
);
782 for (n
= &major_names
[major_to_index(major
)]; *n
; n
= &(*n
)->next
) {
783 if ((*n
)->major
== major
&& (*n
)->probe
) {
785 mutex_unlock(&major_names_lock
);
789 mutex_unlock(&major_names_lock
);
791 if (request_module("block-major-%d-%d", MAJOR(devt
), MINOR(devt
)) > 0)
792 /* Make old-style 2.4 aliases work */
793 request_module("block-major-%d", MAJOR(devt
));
795 #endif /* CONFIG_BLOCK_LEGACY_AUTOLOAD */
797 #ifdef CONFIG_PROC_FS
799 static void *disk_seqf_start(struct seq_file
*seqf
, loff_t
*pos
)
802 struct class_dev_iter
*iter
;
805 iter
= kmalloc(sizeof(*iter
), GFP_KERNEL
);
807 return ERR_PTR(-ENOMEM
);
809 seqf
->private = iter
;
810 class_dev_iter_init(iter
, &block_class
, NULL
, &disk_type
);
812 dev
= class_dev_iter_next(iter
);
817 return dev_to_disk(dev
);
820 static void *disk_seqf_next(struct seq_file
*seqf
, void *v
, loff_t
*pos
)
825 dev
= class_dev_iter_next(seqf
->private);
827 return dev_to_disk(dev
);
832 static void disk_seqf_stop(struct seq_file
*seqf
, void *v
)
834 struct class_dev_iter
*iter
= seqf
->private;
836 /* stop is called even after start failed :-( */
838 class_dev_iter_exit(iter
);
840 seqf
->private = NULL
;
844 static void *show_partition_start(struct seq_file
*seqf
, loff_t
*pos
)
848 p
= disk_seqf_start(seqf
, pos
);
849 if (!IS_ERR_OR_NULL(p
) && !*pos
)
850 seq_puts(seqf
, "major minor #blocks name\n\n");
854 static int show_partition(struct seq_file
*seqf
, void *v
)
856 struct gendisk
*sgp
= v
;
857 struct block_device
*part
;
860 if (!get_capacity(sgp
) || (sgp
->flags
& GENHD_FL_HIDDEN
))
864 xa_for_each(&sgp
->part_tbl
, idx
, part
) {
865 if (!bdev_nr_sectors(part
))
867 seq_printf(seqf
, "%4d %7d %10llu %pg\n",
868 MAJOR(part
->bd_dev
), MINOR(part
->bd_dev
),
869 bdev_nr_sectors(part
) >> 1, part
);
875 static const struct seq_operations partitions_op
= {
876 .start
= show_partition_start
,
877 .next
= disk_seqf_next
,
878 .stop
= disk_seqf_stop
,
879 .show
= show_partition
883 static int __init
genhd_device_init(void)
887 error
= class_register(&block_class
);
892 register_blkdev(BLOCK_EXT_MAJOR
, "blkext");
894 /* create top-level block dir */
895 block_depr
= kobject_create_and_add("block", NULL
);
899 subsys_initcall(genhd_device_init
);
901 static ssize_t
disk_range_show(struct device
*dev
,
902 struct device_attribute
*attr
, char *buf
)
904 struct gendisk
*disk
= dev_to_disk(dev
);
906 return sprintf(buf
, "%d\n", disk
->minors
);
909 static ssize_t
disk_ext_range_show(struct device
*dev
,
910 struct device_attribute
*attr
, char *buf
)
912 struct gendisk
*disk
= dev_to_disk(dev
);
914 return sprintf(buf
, "%d\n",
915 (disk
->flags
& GENHD_FL_NO_PART
) ? 1 : DISK_MAX_PARTS
);
918 static ssize_t
disk_removable_show(struct device
*dev
,
919 struct device_attribute
*attr
, char *buf
)
921 struct gendisk
*disk
= dev_to_disk(dev
);
923 return sprintf(buf
, "%d\n",
924 (disk
->flags
& GENHD_FL_REMOVABLE
? 1 : 0));
927 static ssize_t
disk_hidden_show(struct device
*dev
,
928 struct device_attribute
*attr
, char *buf
)
930 struct gendisk
*disk
= dev_to_disk(dev
);
932 return sprintf(buf
, "%d\n",
933 (disk
->flags
& GENHD_FL_HIDDEN
? 1 : 0));
936 static ssize_t
disk_ro_show(struct device
*dev
,
937 struct device_attribute
*attr
, char *buf
)
939 struct gendisk
*disk
= dev_to_disk(dev
);
941 return sprintf(buf
, "%d\n", get_disk_ro(disk
) ? 1 : 0);
944 ssize_t
part_size_show(struct device
*dev
,
945 struct device_attribute
*attr
, char *buf
)
947 return sprintf(buf
, "%llu\n", bdev_nr_sectors(dev_to_bdev(dev
)));
950 ssize_t
part_stat_show(struct device
*dev
,
951 struct device_attribute
*attr
, char *buf
)
953 struct block_device
*bdev
= dev_to_bdev(dev
);
954 struct request_queue
*q
= bdev_get_queue(bdev
);
955 struct disk_stats stat
;
956 unsigned int inflight
;
959 inflight
= blk_mq_in_flight(q
, bdev
);
961 inflight
= part_in_flight(bdev
);
965 update_io_ticks(bdev
, jiffies
, true);
968 part_stat_read_all(bdev
, &stat
);
970 "%8lu %8lu %8llu %8u "
971 "%8lu %8lu %8llu %8u "
973 "%8lu %8lu %8llu %8u "
977 stat
.merges
[STAT_READ
],
978 (unsigned long long)stat
.sectors
[STAT_READ
],
979 (unsigned int)div_u64(stat
.nsecs
[STAT_READ
], NSEC_PER_MSEC
),
980 stat
.ios
[STAT_WRITE
],
981 stat
.merges
[STAT_WRITE
],
982 (unsigned long long)stat
.sectors
[STAT_WRITE
],
983 (unsigned int)div_u64(stat
.nsecs
[STAT_WRITE
], NSEC_PER_MSEC
),
985 jiffies_to_msecs(stat
.io_ticks
),
986 (unsigned int)div_u64(stat
.nsecs
[STAT_READ
] +
987 stat
.nsecs
[STAT_WRITE
] +
988 stat
.nsecs
[STAT_DISCARD
] +
989 stat
.nsecs
[STAT_FLUSH
],
991 stat
.ios
[STAT_DISCARD
],
992 stat
.merges
[STAT_DISCARD
],
993 (unsigned long long)stat
.sectors
[STAT_DISCARD
],
994 (unsigned int)div_u64(stat
.nsecs
[STAT_DISCARD
], NSEC_PER_MSEC
),
995 stat
.ios
[STAT_FLUSH
],
996 (unsigned int)div_u64(stat
.nsecs
[STAT_FLUSH
], NSEC_PER_MSEC
));
999 ssize_t
part_inflight_show(struct device
*dev
, struct device_attribute
*attr
,
1002 struct block_device
*bdev
= dev_to_bdev(dev
);
1003 struct request_queue
*q
= bdev_get_queue(bdev
);
1004 unsigned int inflight
[2];
1007 blk_mq_in_flight_rw(q
, bdev
, inflight
);
1009 part_in_flight_rw(bdev
, inflight
);
1011 return sprintf(buf
, "%8u %8u\n", inflight
[0], inflight
[1]);
1014 static ssize_t
disk_capability_show(struct device
*dev
,
1015 struct device_attribute
*attr
, char *buf
)
1017 dev_warn_once(dev
, "the capability attribute has been deprecated.\n");
1018 return sprintf(buf
, "0\n");
1021 static ssize_t
disk_alignment_offset_show(struct device
*dev
,
1022 struct device_attribute
*attr
,
1025 struct gendisk
*disk
= dev_to_disk(dev
);
1027 return sprintf(buf
, "%d\n", bdev_alignment_offset(disk
->part0
));
1030 static ssize_t
disk_discard_alignment_show(struct device
*dev
,
1031 struct device_attribute
*attr
,
1034 struct gendisk
*disk
= dev_to_disk(dev
);
1036 return sprintf(buf
, "%d\n", bdev_alignment_offset(disk
->part0
));
1039 static ssize_t
diskseq_show(struct device
*dev
,
1040 struct device_attribute
*attr
, char *buf
)
1042 struct gendisk
*disk
= dev_to_disk(dev
);
1044 return sprintf(buf
, "%llu\n", disk
->diskseq
);
1047 static DEVICE_ATTR(range
, 0444, disk_range_show
, NULL
);
1048 static DEVICE_ATTR(ext_range
, 0444, disk_ext_range_show
, NULL
);
1049 static DEVICE_ATTR(removable
, 0444, disk_removable_show
, NULL
);
1050 static DEVICE_ATTR(hidden
, 0444, disk_hidden_show
, NULL
);
1051 static DEVICE_ATTR(ro
, 0444, disk_ro_show
, NULL
);
1052 static DEVICE_ATTR(size
, 0444, part_size_show
, NULL
);
1053 static DEVICE_ATTR(alignment_offset
, 0444, disk_alignment_offset_show
, NULL
);
1054 static DEVICE_ATTR(discard_alignment
, 0444, disk_discard_alignment_show
, NULL
);
1055 static DEVICE_ATTR(capability
, 0444, disk_capability_show
, NULL
);
1056 static DEVICE_ATTR(stat
, 0444, part_stat_show
, NULL
);
1057 static DEVICE_ATTR(inflight
, 0444, part_inflight_show
, NULL
);
1058 static DEVICE_ATTR(badblocks
, 0644, disk_badblocks_show
, disk_badblocks_store
);
1059 static DEVICE_ATTR(diskseq
, 0444, diskseq_show
, NULL
);
1061 #ifdef CONFIG_FAIL_MAKE_REQUEST
1062 ssize_t
part_fail_show(struct device
*dev
,
1063 struct device_attribute
*attr
, char *buf
)
1065 return sprintf(buf
, "%d\n", dev_to_bdev(dev
)->bd_make_it_fail
);
1068 ssize_t
part_fail_store(struct device
*dev
,
1069 struct device_attribute
*attr
,
1070 const char *buf
, size_t count
)
1074 if (count
> 0 && sscanf(buf
, "%d", &i
) > 0)
1075 dev_to_bdev(dev
)->bd_make_it_fail
= i
;
1080 static struct device_attribute dev_attr_fail
=
1081 __ATTR(make
-it
-fail
, 0644, part_fail_show
, part_fail_store
);
1082 #endif /* CONFIG_FAIL_MAKE_REQUEST */
1084 #ifdef CONFIG_FAIL_IO_TIMEOUT
1085 static struct device_attribute dev_attr_fail_timeout
=
1086 __ATTR(io
-timeout
-fail
, 0644, part_timeout_show
, part_timeout_store
);
1089 static struct attribute
*disk_attrs
[] = {
1090 &dev_attr_range
.attr
,
1091 &dev_attr_ext_range
.attr
,
1092 &dev_attr_removable
.attr
,
1093 &dev_attr_hidden
.attr
,
1095 &dev_attr_size
.attr
,
1096 &dev_attr_alignment_offset
.attr
,
1097 &dev_attr_discard_alignment
.attr
,
1098 &dev_attr_capability
.attr
,
1099 &dev_attr_stat
.attr
,
1100 &dev_attr_inflight
.attr
,
1101 &dev_attr_badblocks
.attr
,
1102 &dev_attr_events
.attr
,
1103 &dev_attr_events_async
.attr
,
1104 &dev_attr_events_poll_msecs
.attr
,
1105 &dev_attr_diskseq
.attr
,
1106 #ifdef CONFIG_FAIL_MAKE_REQUEST
1107 &dev_attr_fail
.attr
,
1109 #ifdef CONFIG_FAIL_IO_TIMEOUT
1110 &dev_attr_fail_timeout
.attr
,
1115 static umode_t
disk_visible(struct kobject
*kobj
, struct attribute
*a
, int n
)
1117 struct device
*dev
= container_of(kobj
, typeof(*dev
), kobj
);
1118 struct gendisk
*disk
= dev_to_disk(dev
);
1120 if (a
== &dev_attr_badblocks
.attr
&& !disk
->bb
)
1125 static struct attribute_group disk_attr_group
= {
1126 .attrs
= disk_attrs
,
1127 .is_visible
= disk_visible
,
1130 static const struct attribute_group
*disk_attr_groups
[] = {
1132 #ifdef CONFIG_BLK_DEV_IO_TRACE
1133 &blk_trace_attr_group
,
1135 #ifdef CONFIG_BLK_DEV_INTEGRITY
1136 &blk_integrity_attr_group
,
1142 * disk_release - releases all allocated resources of the gendisk
1143 * @dev: the device representing this disk
1145 * This function releases all allocated resources of the gendisk.
1147 * Drivers which used __device_add_disk() have a gendisk with a request_queue
1148 * assigned. Since the request_queue sits on top of the gendisk for these
1149 * drivers we also call blk_put_queue() for them, and we expect the
1150 * request_queue refcount to reach 0 at this point, and so the request_queue
1151 * will also be freed prior to the disk.
1153 * Context: can sleep
1155 static void disk_release(struct device
*dev
)
1157 struct gendisk
*disk
= dev_to_disk(dev
);
1160 WARN_ON_ONCE(disk_live(disk
));
1162 blk_trace_remove(disk
->queue
);
1165 * To undo the all initialization from blk_mq_init_allocated_queue in
1166 * case of a probe failure where add_disk is never called we have to
1167 * call blk_mq_exit_queue here. We can't do this for the more common
1168 * teardown case (yet) as the tagset can be gone by the time the disk
1169 * is released once it was added.
1171 if (queue_is_mq(disk
->queue
) &&
1172 test_bit(GD_OWNS_QUEUE
, &disk
->state
) &&
1173 !test_bit(GD_ADDED
, &disk
->state
))
1174 blk_mq_exit_queue(disk
->queue
);
1176 blkcg_exit_disk(disk
);
1178 bioset_exit(&disk
->bio_split
);
1180 disk_release_events(disk
);
1181 kfree(disk
->random
);
1182 disk_free_zone_bitmaps(disk
);
1183 xa_destroy(&disk
->part_tbl
);
1185 disk
->queue
->disk
= NULL
;
1186 blk_put_queue(disk
->queue
);
1188 if (test_bit(GD_ADDED
, &disk
->state
) && disk
->fops
->free_disk
)
1189 disk
->fops
->free_disk(disk
);
1191 iput(disk
->part0
->bd_inode
); /* frees the disk */
1194 static int block_uevent(const struct device
*dev
, struct kobj_uevent_env
*env
)
1196 const struct gendisk
*disk
= dev_to_disk(dev
);
1198 return add_uevent_var(env
, "DISKSEQ=%llu", disk
->diskseq
);
1201 struct class block_class
= {
1203 .dev_uevent
= block_uevent
,
1206 static char *block_devnode(const struct device
*dev
, umode_t
*mode
,
1207 kuid_t
*uid
, kgid_t
*gid
)
1209 struct gendisk
*disk
= dev_to_disk(dev
);
1211 if (disk
->fops
->devnode
)
1212 return disk
->fops
->devnode(disk
, mode
);
1216 const struct device_type disk_type
= {
1218 .groups
= disk_attr_groups
,
1219 .release
= disk_release
,
1220 .devnode
= block_devnode
,
1223 #ifdef CONFIG_PROC_FS
1225 * aggregate disk stat collector. Uses the same stats that the sysfs
1226 * entries do, above, but makes them available through one seq_file.
1228 * The output looks suspiciously like /proc/partitions with a bunch of
1231 static int diskstats_show(struct seq_file
*seqf
, void *v
)
1233 struct gendisk
*gp
= v
;
1234 struct block_device
*hd
;
1235 unsigned int inflight
;
1236 struct disk_stats stat
;
1240 if (&disk_to_dev(gp)->kobj.entry == block_class.devices.next)
1241 seq_puts(seqf, "major minor name"
1242 " rio rmerge rsect ruse wio wmerge "
1243 "wsect wuse running use aveq"
1248 xa_for_each(&gp
->part_tbl
, idx
, hd
) {
1249 if (bdev_is_partition(hd
) && !bdev_nr_sectors(hd
))
1251 if (queue_is_mq(gp
->queue
))
1252 inflight
= blk_mq_in_flight(gp
->queue
, hd
);
1254 inflight
= part_in_flight(hd
);
1258 update_io_ticks(hd
, jiffies
, true);
1261 part_stat_read_all(hd
, &stat
);
1262 seq_printf(seqf
, "%4d %7d %pg "
1269 MAJOR(hd
->bd_dev
), MINOR(hd
->bd_dev
), hd
,
1270 stat
.ios
[STAT_READ
],
1271 stat
.merges
[STAT_READ
],
1272 stat
.sectors
[STAT_READ
],
1273 (unsigned int)div_u64(stat
.nsecs
[STAT_READ
],
1275 stat
.ios
[STAT_WRITE
],
1276 stat
.merges
[STAT_WRITE
],
1277 stat
.sectors
[STAT_WRITE
],
1278 (unsigned int)div_u64(stat
.nsecs
[STAT_WRITE
],
1281 jiffies_to_msecs(stat
.io_ticks
),
1282 (unsigned int)div_u64(stat
.nsecs
[STAT_READ
] +
1283 stat
.nsecs
[STAT_WRITE
] +
1284 stat
.nsecs
[STAT_DISCARD
] +
1285 stat
.nsecs
[STAT_FLUSH
],
1287 stat
.ios
[STAT_DISCARD
],
1288 stat
.merges
[STAT_DISCARD
],
1289 stat
.sectors
[STAT_DISCARD
],
1290 (unsigned int)div_u64(stat
.nsecs
[STAT_DISCARD
],
1292 stat
.ios
[STAT_FLUSH
],
1293 (unsigned int)div_u64(stat
.nsecs
[STAT_FLUSH
],
1302 static const struct seq_operations diskstats_op
= {
1303 .start
= disk_seqf_start
,
1304 .next
= disk_seqf_next
,
1305 .stop
= disk_seqf_stop
,
1306 .show
= diskstats_show
1309 static int __init
proc_genhd_init(void)
1311 proc_create_seq("diskstats", 0, NULL
, &diskstats_op
);
1312 proc_create_seq("partitions", 0, NULL
, &partitions_op
);
1315 module_init(proc_genhd_init
);
1316 #endif /* CONFIG_PROC_FS */
1318 dev_t
part_devt(struct gendisk
*disk
, u8 partno
)
1320 struct block_device
*part
;
1324 part
= xa_load(&disk
->part_tbl
, partno
);
1326 devt
= part
->bd_dev
;
1332 struct gendisk
*__alloc_disk_node(struct request_queue
*q
, int node_id
,
1333 struct lock_class_key
*lkclass
)
1335 struct gendisk
*disk
;
1337 disk
= kzalloc_node(sizeof(struct gendisk
), GFP_KERNEL
, node_id
);
1341 if (bioset_init(&disk
->bio_split
, BIO_POOL_SIZE
, 0, 0))
1344 disk
->bdi
= bdi_alloc(node_id
);
1346 goto out_free_bioset
;
1348 /* bdev_alloc() might need the queue, set before the first call */
1351 disk
->part0
= bdev_alloc(disk
, 0);
1355 disk
->node_id
= node_id
;
1356 mutex_init(&disk
->open_mutex
);
1357 xa_init(&disk
->part_tbl
);
1358 if (xa_insert(&disk
->part_tbl
, 0, disk
->part0
, GFP_KERNEL
))
1359 goto out_destroy_part_tbl
;
1361 if (blkcg_init_disk(disk
))
1362 goto out_erase_part0
;
1364 rand_initialize_disk(disk
);
1365 disk_to_dev(disk
)->class = &block_class
;
1366 disk_to_dev(disk
)->type
= &disk_type
;
1367 device_initialize(disk_to_dev(disk
));
1370 lockdep_init_map(&disk
->lockdep_map
, "(bio completion)", lkclass
, 0);
1371 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
1372 INIT_LIST_HEAD(&disk
->slave_bdevs
);
1377 xa_erase(&disk
->part_tbl
, 0);
1378 out_destroy_part_tbl
:
1379 xa_destroy(&disk
->part_tbl
);
1380 disk
->part0
->bd_disk
= NULL
;
1381 iput(disk
->part0
->bd_inode
);
1385 bioset_exit(&disk
->bio_split
);
1391 struct gendisk
*__blk_alloc_disk(int node
, struct lock_class_key
*lkclass
)
1393 struct request_queue
*q
;
1394 struct gendisk
*disk
;
1396 q
= blk_alloc_queue(node
);
1400 disk
= __alloc_disk_node(q
, node
, lkclass
);
1405 set_bit(GD_OWNS_QUEUE
, &disk
->state
);
1408 EXPORT_SYMBOL(__blk_alloc_disk
);
1411 * put_disk - decrements the gendisk refcount
1412 * @disk: the struct gendisk to decrement the refcount for
1414 * This decrements the refcount for the struct gendisk. When this reaches 0
1415 * we'll have disk_release() called.
1417 * Note: for blk-mq disk put_disk must be called before freeing the tag_set
1418 * when handling probe errors (that is before add_disk() is called).
1420 * Context: Any context, but the last reference must not be dropped from
1423 void put_disk(struct gendisk
*disk
)
1426 put_device(disk_to_dev(disk
));
1428 EXPORT_SYMBOL(put_disk
);
1430 static void set_disk_ro_uevent(struct gendisk
*gd
, int ro
)
1432 char event
[] = "DISK_RO=1";
1433 char *envp
[] = { event
, NULL
};
1437 kobject_uevent_env(&disk_to_dev(gd
)->kobj
, KOBJ_CHANGE
, envp
);
1441 * set_disk_ro - set a gendisk read-only
1442 * @disk: gendisk to operate on
1443 * @read_only: %true to set the disk read-only, %false set the disk read/write
1445 * This function is used to indicate whether a given disk device should have its
1446 * read-only flag set. set_disk_ro() is typically used by device drivers to
1447 * indicate whether the underlying physical device is write-protected.
1449 void set_disk_ro(struct gendisk
*disk
, bool read_only
)
1452 if (test_and_set_bit(GD_READ_ONLY
, &disk
->state
))
1455 if (!test_and_clear_bit(GD_READ_ONLY
, &disk
->state
))
1458 set_disk_ro_uevent(disk
, read_only
);
1460 EXPORT_SYMBOL(set_disk_ro
);
1462 void inc_diskseq(struct gendisk
*disk
)
1464 disk
->diskseq
= atomic64_inc_return(&diskseq
);