1 // SPDX-License-Identifier: GPL-2.0
3 * CPU subsystem support
6 #include <linux/kernel.h>
7 #include <linux/module.h>
8 #include <linux/init.h>
9 #include <linux/sched.h>
10 #include <linux/cpu.h>
11 #include <linux/topology.h>
12 #include <linux/device.h>
13 #include <linux/node.h>
14 #include <linux/gfp.h>
15 #include <linux/slab.h>
16 #include <linux/percpu.h>
17 #include <linux/acpi.h>
19 #include <linux/cpufeature.h>
20 #include <linux/tick.h>
21 #include <linux/pm_qos.h>
22 #include <linux/delay.h>
23 #include <linux/sched/isolation.h>
27 static DEFINE_PER_CPU(struct device
*, cpu_sys_devices
);
29 static int cpu_subsys_match(struct device
*dev
, struct device_driver
*drv
)
31 /* ACPI style match is the only one that may succeed. */
32 if (acpi_driver_match_device(dev
, drv
))
38 #ifdef CONFIG_HOTPLUG_CPU
39 static void change_cpu_under_node(struct cpu
*cpu
,
40 unsigned int from_nid
, unsigned int to_nid
)
42 int cpuid
= cpu
->dev
.id
;
43 unregister_cpu_under_node(cpuid
, from_nid
);
44 register_cpu_under_node(cpuid
, to_nid
);
45 cpu
->node_id
= to_nid
;
48 static int cpu_subsys_online(struct device
*dev
)
50 struct cpu
*cpu
= container_of(dev
, struct cpu
, dev
);
56 from_nid
= cpu_to_node(cpuid
);
57 if (from_nid
== NUMA_NO_NODE
)
61 ret
= cpu_device_up(dev
);
64 * If -EBUSY is returned, it is likely that hotplug is temporarily
65 * disabled when cpu_hotplug_disable() was called. This condition is
66 * transient. So we retry after waiting for an exponentially
67 * increasing delay up to a total of at least 620ms as some PCI
68 * device initialization can take quite a while.
74 msleep(10 * (1 << retries
));
79 * When hot adding memory to memoryless node and enabling a cpu
80 * on the node, node number of the cpu may internally change.
82 to_nid
= cpu_to_node(cpuid
);
83 if (from_nid
!= to_nid
)
84 change_cpu_under_node(cpu
, from_nid
, to_nid
);
89 static int cpu_subsys_offline(struct device
*dev
)
91 return cpu_device_down(dev
);
94 void unregister_cpu(struct cpu
*cpu
)
96 int logical_cpu
= cpu
->dev
.id
;
98 unregister_cpu_under_node(logical_cpu
, cpu_to_node(logical_cpu
));
100 device_unregister(&cpu
->dev
);
101 per_cpu(cpu_sys_devices
, logical_cpu
) = NULL
;
105 #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
106 static ssize_t
cpu_probe_store(struct device
*dev
,
107 struct device_attribute
*attr
,
114 ret
= lock_device_hotplug_sysfs();
118 cnt
= arch_cpu_probe(buf
, count
);
120 unlock_device_hotplug();
124 static ssize_t
cpu_release_store(struct device
*dev
,
125 struct device_attribute
*attr
,
132 ret
= lock_device_hotplug_sysfs();
136 cnt
= arch_cpu_release(buf
, count
);
138 unlock_device_hotplug();
142 static DEVICE_ATTR(probe
, S_IWUSR
, NULL
, cpu_probe_store
);
143 static DEVICE_ATTR(release
, S_IWUSR
, NULL
, cpu_release_store
);
144 #endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */
145 #endif /* CONFIG_HOTPLUG_CPU */
148 #include <linux/kexec.h>
150 static ssize_t
crash_notes_show(struct device
*dev
,
151 struct device_attribute
*attr
,
154 struct cpu
*cpu
= container_of(dev
, struct cpu
, dev
);
155 unsigned long long addr
;
158 cpunum
= cpu
->dev
.id
;
161 * Might be reading other cpu's data based on which cpu read thread
162 * has been scheduled. But cpu data (memory) is allocated once during
163 * boot up and this data does not change there after. Hence this
164 * operation should be safe. No locking required.
166 addr
= per_cpu_ptr_to_phys(per_cpu_ptr(crash_notes
, cpunum
));
168 return sysfs_emit(buf
, "%llx\n", addr
);
170 static DEVICE_ATTR_ADMIN_RO(crash_notes
);
172 static ssize_t
crash_notes_size_show(struct device
*dev
,
173 struct device_attribute
*attr
,
176 return sysfs_emit(buf
, "%zu\n", sizeof(note_buf_t
));
178 static DEVICE_ATTR_ADMIN_RO(crash_notes_size
);
180 static struct attribute
*crash_note_cpu_attrs
[] = {
181 &dev_attr_crash_notes
.attr
,
182 &dev_attr_crash_notes_size
.attr
,
186 static const struct attribute_group crash_note_cpu_attr_group
= {
187 .attrs
= crash_note_cpu_attrs
,
191 static const struct attribute_group
*common_cpu_attr_groups
[] = {
193 &crash_note_cpu_attr_group
,
198 static const struct attribute_group
*hotplugable_cpu_attr_groups
[] = {
200 &crash_note_cpu_attr_group
,
206 * Print cpu online, possible, present, and system maps
210 struct device_attribute attr
;
211 const struct cpumask
*const map
;
214 static ssize_t
show_cpus_attr(struct device
*dev
,
215 struct device_attribute
*attr
,
218 struct cpu_attr
*ca
= container_of(attr
, struct cpu_attr
, attr
);
220 return cpumap_print_to_pagebuf(true, buf
, ca
->map
);
223 #define _CPU_ATTR(name, map) \
224 { __ATTR(name, 0444, show_cpus_attr, NULL), map }
226 /* Keep in sync with cpu_subsys_attrs */
227 static struct cpu_attr cpu_attrs
[] = {
228 _CPU_ATTR(online
, &__cpu_online_mask
),
229 _CPU_ATTR(possible
, &__cpu_possible_mask
),
230 _CPU_ATTR(present
, &__cpu_present_mask
),
234 * Print values for NR_CPUS and offlined cpus
236 static ssize_t
print_cpus_kernel_max(struct device
*dev
,
237 struct device_attribute
*attr
, char *buf
)
239 return sysfs_emit(buf
, "%d\n", NR_CPUS
- 1);
241 static DEVICE_ATTR(kernel_max
, 0444, print_cpus_kernel_max
, NULL
);
243 /* arch-optional setting to enable display of offline cpus >= nr_cpu_ids */
244 unsigned int total_cpus
;
246 static ssize_t
print_cpus_offline(struct device
*dev
,
247 struct device_attribute
*attr
, char *buf
)
250 cpumask_var_t offline
;
252 /* display offline cpus < nr_cpu_ids */
253 if (!alloc_cpumask_var(&offline
, GFP_KERNEL
))
255 cpumask_andnot(offline
, cpu_possible_mask
, cpu_online_mask
);
256 len
+= sysfs_emit_at(buf
, len
, "%*pbl", cpumask_pr_args(offline
));
257 free_cpumask_var(offline
);
259 /* display offline cpus >= nr_cpu_ids */
260 if (total_cpus
&& nr_cpu_ids
< total_cpus
) {
261 len
+= sysfs_emit_at(buf
, len
, ",");
263 if (nr_cpu_ids
== total_cpus
-1)
264 len
+= sysfs_emit_at(buf
, len
, "%u", nr_cpu_ids
);
266 len
+= sysfs_emit_at(buf
, len
, "%u-%d",
267 nr_cpu_ids
, total_cpus
- 1);
270 len
+= sysfs_emit_at(buf
, len
, "\n");
274 static DEVICE_ATTR(offline
, 0444, print_cpus_offline
, NULL
);
276 static ssize_t
print_cpus_isolated(struct device
*dev
,
277 struct device_attribute
*attr
, char *buf
)
280 cpumask_var_t isolated
;
282 if (!alloc_cpumask_var(&isolated
, GFP_KERNEL
))
285 cpumask_andnot(isolated
, cpu_possible_mask
,
286 housekeeping_cpumask(HK_TYPE_DOMAIN
));
287 len
= sysfs_emit(buf
, "%*pbl\n", cpumask_pr_args(isolated
));
289 free_cpumask_var(isolated
);
293 static DEVICE_ATTR(isolated
, 0444, print_cpus_isolated
, NULL
);
295 #ifdef CONFIG_NO_HZ_FULL
296 static ssize_t
print_cpus_nohz_full(struct device
*dev
,
297 struct device_attribute
*attr
, char *buf
)
299 return sysfs_emit(buf
, "%*pbl\n", cpumask_pr_args(tick_nohz_full_mask
));
301 static DEVICE_ATTR(nohz_full
, 0444, print_cpus_nohz_full
, NULL
);
304 #ifdef CONFIG_CRASH_HOTPLUG
305 static ssize_t
crash_hotplug_show(struct device
*dev
,
306 struct device_attribute
*attr
,
309 return sysfs_emit(buf
, "%d\n", crash_hotplug_cpu_support());
311 static DEVICE_ATTR_ADMIN_RO(crash_hotplug
);
314 static void cpu_device_release(struct device
*dev
)
317 * This is an empty function to prevent the driver core from spitting a
318 * warning at us. Yes, I know this is directly opposite of what the
319 * documentation for the driver core and kobjects say, and the author
320 * of this code has already been publically ridiculed for doing
321 * something as foolish as this. However, at this point in time, it is
322 * the only way to handle the issue of statically allocated cpu
323 * devices. The different architectures will have their cpu device
324 * code reworked to properly handle this in the near future, so this
325 * function will then be changed to correctly free up the memory held
328 * Never copy this way of doing things, or you too will be made fun of
329 * on the linux-kernel list, you have been warned.
333 #ifdef CONFIG_GENERIC_CPU_AUTOPROBE
334 static ssize_t
print_cpu_modalias(struct device
*dev
,
335 struct device_attribute
*attr
,
341 len
+= sysfs_emit_at(buf
, len
,
342 "cpu:type:" CPU_FEATURE_TYPEFMT
":feature:",
343 CPU_FEATURE_TYPEVAL
);
345 for (i
= 0; i
< MAX_CPU_FEATURES
; i
++)
346 if (cpu_have_feature(i
)) {
347 if (len
+ sizeof(",XXXX\n") >= PAGE_SIZE
) {
348 WARN(1, "CPU features overflow page\n");
351 len
+= sysfs_emit_at(buf
, len
, ",%04X", i
);
353 len
+= sysfs_emit_at(buf
, len
, "\n");
357 static int cpu_uevent(const struct device
*dev
, struct kobj_uevent_env
*env
)
359 char *buf
= kzalloc(PAGE_SIZE
, GFP_KERNEL
);
361 print_cpu_modalias(NULL
, NULL
, buf
);
362 add_uevent_var(env
, "MODALIAS=%s", buf
);
369 struct bus_type cpu_subsys
= {
372 .match
= cpu_subsys_match
,
373 #ifdef CONFIG_HOTPLUG_CPU
374 .online
= cpu_subsys_online
,
375 .offline
= cpu_subsys_offline
,
377 #ifdef CONFIG_GENERIC_CPU_AUTOPROBE
378 .uevent
= cpu_uevent
,
381 EXPORT_SYMBOL_GPL(cpu_subsys
);
384 * register_cpu - Setup a sysfs device for a CPU.
385 * @cpu - cpu->hotpluggable field set to 1 will generate a control file in
386 * sysfs for this CPU.
387 * @num - CPU number to use when creating the device.
389 * Initialize and register the CPU device.
391 int register_cpu(struct cpu
*cpu
, int num
)
395 cpu
->node_id
= cpu_to_node(num
);
396 memset(&cpu
->dev
, 0x00, sizeof(struct device
));
398 cpu
->dev
.bus
= &cpu_subsys
;
399 cpu
->dev
.release
= cpu_device_release
;
400 cpu
->dev
.offline_disabled
= !cpu
->hotpluggable
;
401 cpu
->dev
.offline
= !cpu_online(num
);
402 cpu
->dev
.of_node
= of_get_cpu_node(num
, NULL
);
403 cpu
->dev
.groups
= common_cpu_attr_groups
;
404 if (cpu
->hotpluggable
)
405 cpu
->dev
.groups
= hotplugable_cpu_attr_groups
;
406 error
= device_register(&cpu
->dev
);
408 put_device(&cpu
->dev
);
412 per_cpu(cpu_sys_devices
, num
) = &cpu
->dev
;
413 register_cpu_under_node(num
, cpu_to_node(num
));
414 dev_pm_qos_expose_latency_limit(&cpu
->dev
,
415 PM_QOS_RESUME_LATENCY_NO_CONSTRAINT
);
420 struct device
*get_cpu_device(unsigned int cpu
)
422 if (cpu
< nr_cpu_ids
&& cpu_possible(cpu
))
423 return per_cpu(cpu_sys_devices
, cpu
);
427 EXPORT_SYMBOL_GPL(get_cpu_device
);
429 static void device_create_release(struct device
*dev
)
435 static struct device
*
436 __cpu_device_create(struct device
*parent
, void *drvdata
,
437 const struct attribute_group
**groups
,
438 const char *fmt
, va_list args
)
440 struct device
*dev
= NULL
;
441 int retval
= -ENOMEM
;
443 dev
= kzalloc(sizeof(*dev
), GFP_KERNEL
);
447 device_initialize(dev
);
448 dev
->parent
= parent
;
449 dev
->groups
= groups
;
450 dev
->release
= device_create_release
;
451 device_set_pm_not_required(dev
);
452 dev_set_drvdata(dev
, drvdata
);
454 retval
= kobject_set_name_vargs(&dev
->kobj
, fmt
, args
);
458 retval
= device_add(dev
);
466 return ERR_PTR(retval
);
469 struct device
*cpu_device_create(struct device
*parent
, void *drvdata
,
470 const struct attribute_group
**groups
,
471 const char *fmt
, ...)
476 va_start(vargs
, fmt
);
477 dev
= __cpu_device_create(parent
, drvdata
, groups
, fmt
, vargs
);
481 EXPORT_SYMBOL_GPL(cpu_device_create
);
483 #ifdef CONFIG_GENERIC_CPU_AUTOPROBE
484 static DEVICE_ATTR(modalias
, 0444, print_cpu_modalias
, NULL
);
487 static struct attribute
*cpu_root_attrs
[] = {
488 #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
489 &dev_attr_probe
.attr
,
490 &dev_attr_release
.attr
,
492 &cpu_attrs
[0].attr
.attr
,
493 &cpu_attrs
[1].attr
.attr
,
494 &cpu_attrs
[2].attr
.attr
,
495 &dev_attr_kernel_max
.attr
,
496 &dev_attr_offline
.attr
,
497 &dev_attr_isolated
.attr
,
498 #ifdef CONFIG_NO_HZ_FULL
499 &dev_attr_nohz_full
.attr
,
501 #ifdef CONFIG_CRASH_HOTPLUG
502 &dev_attr_crash_hotplug
.attr
,
504 #ifdef CONFIG_GENERIC_CPU_AUTOPROBE
505 &dev_attr_modalias
.attr
,
510 static const struct attribute_group cpu_root_attr_group
= {
511 .attrs
= cpu_root_attrs
,
514 static const struct attribute_group
*cpu_root_attr_groups
[] = {
515 &cpu_root_attr_group
,
519 bool cpu_is_hotpluggable(unsigned int cpu
)
521 struct device
*dev
= get_cpu_device(cpu
);
522 return dev
&& container_of(dev
, struct cpu
, dev
)->hotpluggable
523 && tick_nohz_cpu_hotpluggable(cpu
);
525 EXPORT_SYMBOL_GPL(cpu_is_hotpluggable
);
527 #ifdef CONFIG_GENERIC_CPU_DEVICES
528 static DEFINE_PER_CPU(struct cpu
, cpu_devices
);
531 static void __init
cpu_dev_register_generic(void)
533 #ifdef CONFIG_GENERIC_CPU_DEVICES
536 for_each_possible_cpu(i
) {
537 if (register_cpu(&per_cpu(cpu_devices
, i
), i
))
538 panic("Failed to register CPU device");
543 #ifdef CONFIG_GENERIC_CPU_VULNERABILITIES
544 static ssize_t
cpu_show_not_affected(struct device
*dev
,
545 struct device_attribute
*attr
, char *buf
)
547 return sysfs_emit(buf
, "Not affected\n");
550 #define CPU_SHOW_VULN_FALLBACK(func) \
551 ssize_t cpu_show_##func(struct device *, \
552 struct device_attribute *, char *) \
553 __attribute__((weak, alias("cpu_show_not_affected")))
555 CPU_SHOW_VULN_FALLBACK(meltdown
);
556 CPU_SHOW_VULN_FALLBACK(spectre_v1
);
557 CPU_SHOW_VULN_FALLBACK(spectre_v2
);
558 CPU_SHOW_VULN_FALLBACK(spec_store_bypass
);
559 CPU_SHOW_VULN_FALLBACK(l1tf
);
560 CPU_SHOW_VULN_FALLBACK(mds
);
561 CPU_SHOW_VULN_FALLBACK(tsx_async_abort
);
562 CPU_SHOW_VULN_FALLBACK(itlb_multihit
);
563 CPU_SHOW_VULN_FALLBACK(srbds
);
564 CPU_SHOW_VULN_FALLBACK(mmio_stale_data
);
565 CPU_SHOW_VULN_FALLBACK(retbleed
);
566 CPU_SHOW_VULN_FALLBACK(spec_rstack_overflow
);
567 CPU_SHOW_VULN_FALLBACK(gds
);
569 static DEVICE_ATTR(meltdown
, 0444, cpu_show_meltdown
, NULL
);
570 static DEVICE_ATTR(spectre_v1
, 0444, cpu_show_spectre_v1
, NULL
);
571 static DEVICE_ATTR(spectre_v2
, 0444, cpu_show_spectre_v2
, NULL
);
572 static DEVICE_ATTR(spec_store_bypass
, 0444, cpu_show_spec_store_bypass
, NULL
);
573 static DEVICE_ATTR(l1tf
, 0444, cpu_show_l1tf
, NULL
);
574 static DEVICE_ATTR(mds
, 0444, cpu_show_mds
, NULL
);
575 static DEVICE_ATTR(tsx_async_abort
, 0444, cpu_show_tsx_async_abort
, NULL
);
576 static DEVICE_ATTR(itlb_multihit
, 0444, cpu_show_itlb_multihit
, NULL
);
577 static DEVICE_ATTR(srbds
, 0444, cpu_show_srbds
, NULL
);
578 static DEVICE_ATTR(mmio_stale_data
, 0444, cpu_show_mmio_stale_data
, NULL
);
579 static DEVICE_ATTR(retbleed
, 0444, cpu_show_retbleed
, NULL
);
580 static DEVICE_ATTR(spec_rstack_overflow
, 0444, cpu_show_spec_rstack_overflow
, NULL
);
581 static DEVICE_ATTR(gather_data_sampling
, 0444, cpu_show_gds
, NULL
);
583 static struct attribute
*cpu_root_vulnerabilities_attrs
[] = {
584 &dev_attr_meltdown
.attr
,
585 &dev_attr_spectre_v1
.attr
,
586 &dev_attr_spectre_v2
.attr
,
587 &dev_attr_spec_store_bypass
.attr
,
590 &dev_attr_tsx_async_abort
.attr
,
591 &dev_attr_itlb_multihit
.attr
,
592 &dev_attr_srbds
.attr
,
593 &dev_attr_mmio_stale_data
.attr
,
594 &dev_attr_retbleed
.attr
,
595 &dev_attr_spec_rstack_overflow
.attr
,
596 &dev_attr_gather_data_sampling
.attr
,
600 static const struct attribute_group cpu_root_vulnerabilities_group
= {
601 .name
= "vulnerabilities",
602 .attrs
= cpu_root_vulnerabilities_attrs
,
605 static void __init
cpu_register_vulnerabilities(void)
607 struct device
*dev
= bus_get_dev_root(&cpu_subsys
);
610 if (sysfs_create_group(&dev
->kobj
, &cpu_root_vulnerabilities_group
))
611 pr_err("Unable to register CPU vulnerabilities\n");
617 static inline void cpu_register_vulnerabilities(void) { }
620 void __init
cpu_dev_init(void)
622 if (subsys_system_register(&cpu_subsys
, cpu_root_attr_groups
))
623 panic("Failed to register CPU subsystem");
625 cpu_dev_register_generic();
626 cpu_register_vulnerabilities();