2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
4 * This code is licenced under the GPL.
6 #include <linux/sched/mm.h>
7 #include <linux/proc_fs.h>
9 #include <linux/init.h>
10 #include <linux/notifier.h>
11 #include <linux/sched/signal.h>
12 #include <linux/sched/hotplug.h>
13 #include <linux/sched/isolation.h>
14 #include <linux/sched/task.h>
15 #include <linux/sched/smt.h>
16 #include <linux/unistd.h>
17 #include <linux/cpu.h>
18 #include <linux/oom.h>
19 #include <linux/rcupdate.h>
20 #include <linux/delay.h>
21 #include <linux/export.h>
22 #include <linux/bug.h>
23 #include <linux/kthread.h>
24 #include <linux/stop_machine.h>
25 #include <linux/mutex.h>
26 #include <linux/gfp.h>
27 #include <linux/suspend.h>
28 #include <linux/lockdep.h>
29 #include <linux/tick.h>
30 #include <linux/irq.h>
31 #include <linux/nmi.h>
32 #include <linux/smpboot.h>
33 #include <linux/relay.h>
34 #include <linux/slab.h>
35 #include <linux/scs.h>
36 #include <linux/percpu-rwsem.h>
37 #include <linux/cpuset.h>
38 #include <linux/random.h>
39 #include <linux/cc_platform.h>
41 #include <trace/events/power.h>
42 #define CREATE_TRACE_POINTS
43 #include <trace/events/cpuhp.h>
48 * struct cpuhp_cpu_state - Per cpu hotplug state storage
49 * @state: The current cpu state
50 * @target: The target state
51 * @fail: Current CPU hotplug callback state
52 * @thread: Pointer to the hotplug thread
53 * @should_run: Thread should execute
54 * @rollback: Perform a rollback
55 * @single: Single callback invocation
56 * @bringup: Single callback bringup or teardown selector
58 * @node: Remote CPU node; for multi-instance, do a
59 * single entry callback for install/remove
60 * @last: For multi-instance rollback, remember how far we got
61 * @cb_state: The state for a single callback (install/uninstall)
62 * @result: Result of the operation
63 * @ap_sync_state: State for AP synchronization
64 * @done_up: Signal completion to the issuer of the task for cpu-up
65 * @done_down: Signal completion to the issuer of the task for cpu-down
67 struct cpuhp_cpu_state
{
68 enum cpuhp_state state
;
69 enum cpuhp_state target
;
70 enum cpuhp_state fail
;
72 struct task_struct
*thread
;
77 struct hlist_node
*node
;
78 struct hlist_node
*last
;
79 enum cpuhp_state cb_state
;
81 atomic_t ap_sync_state
;
82 struct completion done_up
;
83 struct completion done_down
;
87 static DEFINE_PER_CPU(struct cpuhp_cpu_state
, cpuhp_state
) = {
88 .fail
= CPUHP_INVALID
,
92 cpumask_t cpus_booted_once_mask
;
95 #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
96 static struct lockdep_map cpuhp_state_up_map
=
97 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map
);
98 static struct lockdep_map cpuhp_state_down_map
=
99 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map
);
102 static inline void cpuhp_lock_acquire(bool bringup
)
104 lock_map_acquire(bringup
? &cpuhp_state_up_map
: &cpuhp_state_down_map
);
107 static inline void cpuhp_lock_release(bool bringup
)
109 lock_map_release(bringup
? &cpuhp_state_up_map
: &cpuhp_state_down_map
);
113 static inline void cpuhp_lock_acquire(bool bringup
) { }
114 static inline void cpuhp_lock_release(bool bringup
) { }
119 * struct cpuhp_step - Hotplug state machine step
120 * @name: Name of the step
121 * @startup: Startup function of the step
122 * @teardown: Teardown function of the step
123 * @cant_stop: Bringup/teardown can't be stopped at this step
124 * @multi_instance: State has multiple instances which get added afterwards
129 int (*single
)(unsigned int cpu
);
130 int (*multi
)(unsigned int cpu
,
131 struct hlist_node
*node
);
134 int (*single
)(unsigned int cpu
);
135 int (*multi
)(unsigned int cpu
,
136 struct hlist_node
*node
);
139 struct hlist_head list
;
145 static DEFINE_MUTEX(cpuhp_state_mutex
);
146 static struct cpuhp_step cpuhp_hp_states
[];
148 static struct cpuhp_step
*cpuhp_get_step(enum cpuhp_state state
)
150 return cpuhp_hp_states
+ state
;
153 static bool cpuhp_step_empty(bool bringup
, struct cpuhp_step
*step
)
155 return bringup
? !step
->startup
.single
: !step
->teardown
.single
;
159 * cpuhp_invoke_callback - Invoke the callbacks for a given state
160 * @cpu: The cpu for which the callback should be invoked
161 * @state: The state to do callbacks for
162 * @bringup: True if the bringup callback should be invoked
163 * @node: For multi-instance, do a single entry callback for install/remove
164 * @lastp: For multi-instance rollback, remember how far we got
166 * Called from cpu hotplug and from the state register machinery.
168 * Return: %0 on success or a negative errno code
170 static int cpuhp_invoke_callback(unsigned int cpu
, enum cpuhp_state state
,
171 bool bringup
, struct hlist_node
*node
,
172 struct hlist_node
**lastp
)
174 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
175 struct cpuhp_step
*step
= cpuhp_get_step(state
);
176 int (*cbm
)(unsigned int cpu
, struct hlist_node
*node
);
177 int (*cb
)(unsigned int cpu
);
180 if (st
->fail
== state
) {
181 st
->fail
= CPUHP_INVALID
;
185 if (cpuhp_step_empty(bringup
, step
)) {
190 if (!step
->multi_instance
) {
191 WARN_ON_ONCE(lastp
&& *lastp
);
192 cb
= bringup
? step
->startup
.single
: step
->teardown
.single
;
194 trace_cpuhp_enter(cpu
, st
->target
, state
, cb
);
196 trace_cpuhp_exit(cpu
, st
->state
, state
, ret
);
199 cbm
= bringup
? step
->startup
.multi
: step
->teardown
.multi
;
201 /* Single invocation for instance add/remove */
203 WARN_ON_ONCE(lastp
&& *lastp
);
204 trace_cpuhp_multi_enter(cpu
, st
->target
, state
, cbm
, node
);
205 ret
= cbm(cpu
, node
);
206 trace_cpuhp_exit(cpu
, st
->state
, state
, ret
);
210 /* State transition. Invoke on all instances */
212 hlist_for_each(node
, &step
->list
) {
213 if (lastp
&& node
== *lastp
)
216 trace_cpuhp_multi_enter(cpu
, st
->target
, state
, cbm
, node
);
217 ret
= cbm(cpu
, node
);
218 trace_cpuhp_exit(cpu
, st
->state
, state
, ret
);
232 /* Rollback the instances if one failed */
233 cbm
= !bringup
? step
->startup
.multi
: step
->teardown
.multi
;
237 hlist_for_each(node
, &step
->list
) {
241 trace_cpuhp_multi_enter(cpu
, st
->target
, state
, cbm
, node
);
242 ret
= cbm(cpu
, node
);
243 trace_cpuhp_exit(cpu
, st
->state
, state
, ret
);
245 * Rollback must not fail,
253 static bool cpuhp_is_ap_state(enum cpuhp_state state
)
256 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
257 * purposes as that state is handled explicitly in cpu_down.
259 return state
> CPUHP_BRINGUP_CPU
&& state
!= CPUHP_TEARDOWN_CPU
;
262 static inline void wait_for_ap_thread(struct cpuhp_cpu_state
*st
, bool bringup
)
264 struct completion
*done
= bringup
? &st
->done_up
: &st
->done_down
;
265 wait_for_completion(done
);
268 static inline void complete_ap_thread(struct cpuhp_cpu_state
*st
, bool bringup
)
270 struct completion
*done
= bringup
? &st
->done_up
: &st
->done_down
;
275 * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
277 static bool cpuhp_is_atomic_state(enum cpuhp_state state
)
279 return CPUHP_AP_IDLE_DEAD
<= state
&& state
< CPUHP_AP_ONLINE
;
282 /* Synchronization state management */
283 enum cpuhp_sync_state
{
286 SYNC_STATE_SHOULD_DIE
,
288 SYNC_STATE_SHOULD_ONLINE
,
292 #ifdef CONFIG_HOTPLUG_CORE_SYNC
294 * cpuhp_ap_update_sync_state - Update synchronization state during bringup/teardown
295 * @state: The synchronization state to set
297 * No synchronization point. Just update of the synchronization state, but implies
298 * a full barrier so that the AP changes are visible before the control CPU proceeds.
300 static inline void cpuhp_ap_update_sync_state(enum cpuhp_sync_state state
)
302 atomic_t
*st
= this_cpu_ptr(&cpuhp_state
.ap_sync_state
);
304 (void)atomic_xchg(st
, state
);
307 void __weak
arch_cpuhp_sync_state_poll(void) { cpu_relax(); }
309 static bool cpuhp_wait_for_sync_state(unsigned int cpu
, enum cpuhp_sync_state state
,
310 enum cpuhp_sync_state next_state
)
312 atomic_t
*st
= per_cpu_ptr(&cpuhp_state
.ap_sync_state
, cpu
);
313 ktime_t now
, end
, start
= ktime_get();
316 end
= start
+ 10ULL * NSEC_PER_SEC
;
318 sync
= atomic_read(st
);
321 if (!atomic_try_cmpxchg(st
, &sync
, next_state
))
328 /* Timeout. Leave the state unchanged */
330 } else if (now
- start
< NSEC_PER_MSEC
) {
331 /* Poll for one millisecond */
332 arch_cpuhp_sync_state_poll();
334 usleep_range_state(USEC_PER_MSEC
, 2 * USEC_PER_MSEC
, TASK_UNINTERRUPTIBLE
);
336 sync
= atomic_read(st
);
340 #else /* CONFIG_HOTPLUG_CORE_SYNC */
341 static inline void cpuhp_ap_update_sync_state(enum cpuhp_sync_state state
) { }
342 #endif /* !CONFIG_HOTPLUG_CORE_SYNC */
344 #ifdef CONFIG_HOTPLUG_CORE_SYNC_DEAD
346 * cpuhp_ap_report_dead - Update synchronization state to DEAD
348 * No synchronization point. Just update of the synchronization state.
350 void cpuhp_ap_report_dead(void)
352 cpuhp_ap_update_sync_state(SYNC_STATE_DEAD
);
355 void __weak
arch_cpuhp_cleanup_dead_cpu(unsigned int cpu
) { }
358 * Late CPU shutdown synchronization point. Cannot use cpuhp_state::done_down
359 * because the AP cannot issue complete() at this stage.
361 static void cpuhp_bp_sync_dead(unsigned int cpu
)
363 atomic_t
*st
= per_cpu_ptr(&cpuhp_state
.ap_sync_state
, cpu
);
364 int sync
= atomic_read(st
);
367 /* CPU can have reported dead already. Don't overwrite that! */
368 if (sync
== SYNC_STATE_DEAD
)
370 } while (!atomic_try_cmpxchg(st
, &sync
, SYNC_STATE_SHOULD_DIE
));
372 if (cpuhp_wait_for_sync_state(cpu
, SYNC_STATE_DEAD
, SYNC_STATE_DEAD
)) {
373 /* CPU reached dead state. Invoke the cleanup function */
374 arch_cpuhp_cleanup_dead_cpu(cpu
);
378 /* No further action possible. Emit message and give up. */
379 pr_err("CPU%u failed to report dead state\n", cpu
);
381 #else /* CONFIG_HOTPLUG_CORE_SYNC_DEAD */
382 static inline void cpuhp_bp_sync_dead(unsigned int cpu
) { }
383 #endif /* !CONFIG_HOTPLUG_CORE_SYNC_DEAD */
385 #ifdef CONFIG_HOTPLUG_CORE_SYNC_FULL
387 * cpuhp_ap_sync_alive - Synchronize AP with the control CPU once it is alive
389 * Updates the AP synchronization state to SYNC_STATE_ALIVE and waits
390 * for the BP to release it.
392 void cpuhp_ap_sync_alive(void)
394 atomic_t
*st
= this_cpu_ptr(&cpuhp_state
.ap_sync_state
);
396 cpuhp_ap_update_sync_state(SYNC_STATE_ALIVE
);
398 /* Wait for the control CPU to release it. */
399 while (atomic_read(st
) != SYNC_STATE_SHOULD_ONLINE
)
403 static bool cpuhp_can_boot_ap(unsigned int cpu
)
405 atomic_t
*st
= per_cpu_ptr(&cpuhp_state
.ap_sync_state
, cpu
);
406 int sync
= atomic_read(st
);
410 case SYNC_STATE_DEAD
:
411 /* CPU is properly dead */
413 case SYNC_STATE_KICKED
:
414 /* CPU did not come up in previous attempt */
416 case SYNC_STATE_ALIVE
:
417 /* CPU is stuck cpuhp_ap_sync_alive(). */
420 /* CPU failed to report online or dead and is in limbo state. */
424 /* Prepare for booting */
425 if (!atomic_try_cmpxchg(st
, &sync
, SYNC_STATE_KICKED
))
431 void __weak
arch_cpuhp_cleanup_kick_cpu(unsigned int cpu
) { }
434 * Early CPU bringup synchronization point. Cannot use cpuhp_state::done_up
435 * because the AP cannot issue complete() so early in the bringup.
437 static int cpuhp_bp_sync_alive(unsigned int cpu
)
441 if (!IS_ENABLED(CONFIG_HOTPLUG_CORE_SYNC_FULL
))
444 if (!cpuhp_wait_for_sync_state(cpu
, SYNC_STATE_ALIVE
, SYNC_STATE_SHOULD_ONLINE
)) {
445 pr_err("CPU%u failed to report alive state\n", cpu
);
449 /* Let the architecture cleanup the kick alive mechanics. */
450 arch_cpuhp_cleanup_kick_cpu(cpu
);
453 #else /* CONFIG_HOTPLUG_CORE_SYNC_FULL */
454 static inline int cpuhp_bp_sync_alive(unsigned int cpu
) { return 0; }
455 static inline bool cpuhp_can_boot_ap(unsigned int cpu
) { return true; }
456 #endif /* !CONFIG_HOTPLUG_CORE_SYNC_FULL */
458 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
459 static DEFINE_MUTEX(cpu_add_remove_lock
);
460 bool cpuhp_tasks_frozen
;
461 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen
);
464 * The following two APIs (cpu_maps_update_begin/done) must be used when
465 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
467 void cpu_maps_update_begin(void)
469 mutex_lock(&cpu_add_remove_lock
);
472 void cpu_maps_update_done(void)
474 mutex_unlock(&cpu_add_remove_lock
);
478 * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
479 * Should always be manipulated under cpu_add_remove_lock
481 static int cpu_hotplug_disabled
;
483 #ifdef CONFIG_HOTPLUG_CPU
485 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock
);
487 void cpus_read_lock(void)
489 percpu_down_read(&cpu_hotplug_lock
);
491 EXPORT_SYMBOL_GPL(cpus_read_lock
);
493 int cpus_read_trylock(void)
495 return percpu_down_read_trylock(&cpu_hotplug_lock
);
497 EXPORT_SYMBOL_GPL(cpus_read_trylock
);
499 void cpus_read_unlock(void)
501 percpu_up_read(&cpu_hotplug_lock
);
503 EXPORT_SYMBOL_GPL(cpus_read_unlock
);
505 void cpus_write_lock(void)
507 percpu_down_write(&cpu_hotplug_lock
);
510 void cpus_write_unlock(void)
512 percpu_up_write(&cpu_hotplug_lock
);
515 void lockdep_assert_cpus_held(void)
518 * We can't have hotplug operations before userspace starts running,
519 * and some init codepaths will knowingly not take the hotplug lock.
520 * This is all valid, so mute lockdep until it makes sense to report
523 if (system_state
< SYSTEM_RUNNING
)
526 percpu_rwsem_assert_held(&cpu_hotplug_lock
);
529 #ifdef CONFIG_LOCKDEP
530 int lockdep_is_cpus_held(void)
532 return percpu_rwsem_is_held(&cpu_hotplug_lock
);
536 static void lockdep_acquire_cpus_lock(void)
538 rwsem_acquire(&cpu_hotplug_lock
.dep_map
, 0, 0, _THIS_IP_
);
541 static void lockdep_release_cpus_lock(void)
543 rwsem_release(&cpu_hotplug_lock
.dep_map
, _THIS_IP_
);
547 * Wait for currently running CPU hotplug operations to complete (if any) and
548 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
549 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
550 * hotplug path before performing hotplug operations. So acquiring that lock
551 * guarantees mutual exclusion from any currently running hotplug operations.
553 void cpu_hotplug_disable(void)
555 cpu_maps_update_begin();
556 cpu_hotplug_disabled
++;
557 cpu_maps_update_done();
559 EXPORT_SYMBOL_GPL(cpu_hotplug_disable
);
561 static void __cpu_hotplug_enable(void)
563 if (WARN_ONCE(!cpu_hotplug_disabled
, "Unbalanced cpu hotplug enable\n"))
565 cpu_hotplug_disabled
--;
568 void cpu_hotplug_enable(void)
570 cpu_maps_update_begin();
571 __cpu_hotplug_enable();
572 cpu_maps_update_done();
574 EXPORT_SYMBOL_GPL(cpu_hotplug_enable
);
578 static void lockdep_acquire_cpus_lock(void)
582 static void lockdep_release_cpus_lock(void)
586 #endif /* CONFIG_HOTPLUG_CPU */
589 * Architectures that need SMT-specific errata handling during SMT hotplug
590 * should override this.
592 void __weak
arch_smt_update(void) { }
594 #ifdef CONFIG_HOTPLUG_SMT
596 enum cpuhp_smt_control cpu_smt_control __read_mostly
= CPU_SMT_ENABLED
;
597 static unsigned int cpu_smt_max_threads __ro_after_init
;
598 unsigned int cpu_smt_num_threads __read_mostly
= UINT_MAX
;
600 void __init
cpu_smt_disable(bool force
)
602 if (!cpu_smt_possible())
606 pr_info("SMT: Force disabled\n");
607 cpu_smt_control
= CPU_SMT_FORCE_DISABLED
;
609 pr_info("SMT: disabled\n");
610 cpu_smt_control
= CPU_SMT_DISABLED
;
612 cpu_smt_num_threads
= 1;
616 * The decision whether SMT is supported can only be done after the full
617 * CPU identification. Called from architecture code.
619 void __init
cpu_smt_set_num_threads(unsigned int num_threads
,
620 unsigned int max_threads
)
622 WARN_ON(!num_threads
|| (num_threads
> max_threads
));
624 if (max_threads
== 1)
625 cpu_smt_control
= CPU_SMT_NOT_SUPPORTED
;
627 cpu_smt_max_threads
= max_threads
;
630 * If SMT has been disabled via the kernel command line or SMT is
631 * not supported, set cpu_smt_num_threads to 1 for consistency.
632 * If enabled, take the architecture requested number of threads
633 * to bring up into account.
635 if (cpu_smt_control
!= CPU_SMT_ENABLED
)
636 cpu_smt_num_threads
= 1;
637 else if (num_threads
< cpu_smt_num_threads
)
638 cpu_smt_num_threads
= num_threads
;
641 static int __init
smt_cmdline_disable(char *str
)
643 cpu_smt_disable(str
&& !strcmp(str
, "force"));
646 early_param("nosmt", smt_cmdline_disable
);
649 * For Archicture supporting partial SMT states check if the thread is allowed.
650 * Otherwise this has already been checked through cpu_smt_max_threads when
651 * setting the SMT level.
653 static inline bool cpu_smt_thread_allowed(unsigned int cpu
)
655 #ifdef CONFIG_SMT_NUM_THREADS_DYNAMIC
656 return topology_smt_thread_allowed(cpu
);
662 static inline bool cpu_smt_allowed(unsigned int cpu
)
664 if (cpu_smt_control
== CPU_SMT_ENABLED
&& cpu_smt_thread_allowed(cpu
))
667 if (topology_is_primary_thread(cpu
))
671 * On x86 it's required to boot all logical CPUs at least once so
672 * that the init code can get a chance to set CR4.MCE on each
673 * CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any
674 * core will shutdown the machine.
676 return !cpumask_test_cpu(cpu
, &cpus_booted_once_mask
);
679 /* Returns true if SMT is supported and not forcefully (irreversibly) disabled */
680 bool cpu_smt_possible(void)
682 return cpu_smt_control
!= CPU_SMT_FORCE_DISABLED
&&
683 cpu_smt_control
!= CPU_SMT_NOT_SUPPORTED
;
685 EXPORT_SYMBOL_GPL(cpu_smt_possible
);
688 static inline bool cpu_smt_allowed(unsigned int cpu
) { return true; }
691 static inline enum cpuhp_state
692 cpuhp_set_state(int cpu
, struct cpuhp_cpu_state
*st
, enum cpuhp_state target
)
694 enum cpuhp_state prev_state
= st
->state
;
695 bool bringup
= st
->state
< target
;
697 st
->rollback
= false;
702 st
->bringup
= bringup
;
703 if (cpu_dying(cpu
) != !bringup
)
704 set_cpu_dying(cpu
, !bringup
);
710 cpuhp_reset_state(int cpu
, struct cpuhp_cpu_state
*st
,
711 enum cpuhp_state prev_state
)
713 bool bringup
= !st
->bringup
;
715 st
->target
= prev_state
;
718 * Already rolling back. No need invert the bringup value or to change
727 * If we have st->last we need to undo partial multi_instance of this
728 * state first. Otherwise start undo at the previous state.
737 st
->bringup
= bringup
;
738 if (cpu_dying(cpu
) != !bringup
)
739 set_cpu_dying(cpu
, !bringup
);
742 /* Regular hotplug invocation of the AP hotplug thread */
743 static void __cpuhp_kick_ap(struct cpuhp_cpu_state
*st
)
745 if (!st
->single
&& st
->state
== st
->target
)
750 * Make sure the above stores are visible before should_run becomes
751 * true. Paired with the mb() above in cpuhp_thread_fun()
754 st
->should_run
= true;
755 wake_up_process(st
->thread
);
756 wait_for_ap_thread(st
, st
->bringup
);
759 static int cpuhp_kick_ap(int cpu
, struct cpuhp_cpu_state
*st
,
760 enum cpuhp_state target
)
762 enum cpuhp_state prev_state
;
765 prev_state
= cpuhp_set_state(cpu
, st
, target
);
767 if ((ret
= st
->result
)) {
768 cpuhp_reset_state(cpu
, st
, prev_state
);
775 static int bringup_wait_for_ap_online(unsigned int cpu
)
777 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
779 /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
780 wait_for_ap_thread(st
, true);
781 if (WARN_ON_ONCE((!cpu_online(cpu
))))
784 /* Unpark the hotplug thread of the target cpu */
785 kthread_unpark(st
->thread
);
788 * SMT soft disabling on X86 requires to bring the CPU out of the
789 * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit. The
790 * CPU marked itself as booted_once in notify_cpu_starting() so the
791 * cpu_smt_allowed() check will now return false if this is not the
794 if (!cpu_smt_allowed(cpu
))
799 #ifdef CONFIG_HOTPLUG_SPLIT_STARTUP
800 static int cpuhp_kick_ap_alive(unsigned int cpu
)
802 if (!cpuhp_can_boot_ap(cpu
))
805 return arch_cpuhp_kick_ap_alive(cpu
, idle_thread_get(cpu
));
808 static int cpuhp_bringup_ap(unsigned int cpu
)
810 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
814 * Some architectures have to walk the irq descriptors to
815 * setup the vector space for the cpu which comes online.
816 * Prevent irq alloc/free across the bringup.
820 ret
= cpuhp_bp_sync_alive(cpu
);
824 ret
= bringup_wait_for_ap_online(cpu
);
830 if (st
->target
<= CPUHP_AP_ONLINE_IDLE
)
833 return cpuhp_kick_ap(cpu
, st
, st
->target
);
840 static int bringup_cpu(unsigned int cpu
)
842 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
843 struct task_struct
*idle
= idle_thread_get(cpu
);
846 if (!cpuhp_can_boot_ap(cpu
))
850 * Some architectures have to walk the irq descriptors to
851 * setup the vector space for the cpu which comes online.
853 * Prevent irq alloc/free across the bringup by acquiring the
854 * sparse irq lock. Hold it until the upcoming CPU completes the
855 * startup in cpuhp_online_idle() which allows to avoid
856 * intermediate synchronization points in the architecture code.
860 ret
= __cpu_up(cpu
, idle
);
864 ret
= cpuhp_bp_sync_alive(cpu
);
868 ret
= bringup_wait_for_ap_online(cpu
);
874 if (st
->target
<= CPUHP_AP_ONLINE_IDLE
)
877 return cpuhp_kick_ap(cpu
, st
, st
->target
);
885 static int finish_cpu(unsigned int cpu
)
887 struct task_struct
*idle
= idle_thread_get(cpu
);
888 struct mm_struct
*mm
= idle
->active_mm
;
891 * idle_task_exit() will have switched to &init_mm, now
892 * clean up any remaining active_mm state.
895 idle
->active_mm
= &init_mm
;
901 * Hotplug state machine related functions
905 * Get the next state to run. Empty ones will be skipped. Returns true if a
908 * st->state will be modified ahead of time, to match state_to_run, as if it
911 static bool cpuhp_next_state(bool bringup
,
912 enum cpuhp_state
*state_to_run
,
913 struct cpuhp_cpu_state
*st
,
914 enum cpuhp_state target
)
918 if (st
->state
>= target
)
921 *state_to_run
= ++st
->state
;
923 if (st
->state
<= target
)
926 *state_to_run
= st
->state
--;
929 if (!cpuhp_step_empty(bringup
, cpuhp_get_step(*state_to_run
)))
936 static int __cpuhp_invoke_callback_range(bool bringup
,
938 struct cpuhp_cpu_state
*st
,
939 enum cpuhp_state target
,
942 enum cpuhp_state state
;
945 while (cpuhp_next_state(bringup
, &state
, st
, target
)) {
948 err
= cpuhp_invoke_callback(cpu
, state
, bringup
, NULL
, NULL
);
953 pr_warn("CPU %u %s state %s (%d) failed (%d)\n",
954 cpu
, bringup
? "UP" : "DOWN",
955 cpuhp_get_step(st
->state
)->name
,
967 static inline int cpuhp_invoke_callback_range(bool bringup
,
969 struct cpuhp_cpu_state
*st
,
970 enum cpuhp_state target
)
972 return __cpuhp_invoke_callback_range(bringup
, cpu
, st
, target
, false);
975 static inline void cpuhp_invoke_callback_range_nofail(bool bringup
,
977 struct cpuhp_cpu_state
*st
,
978 enum cpuhp_state target
)
980 __cpuhp_invoke_callback_range(bringup
, cpu
, st
, target
, true);
983 static inline bool can_rollback_cpu(struct cpuhp_cpu_state
*st
)
985 if (IS_ENABLED(CONFIG_HOTPLUG_CPU
))
988 * When CPU hotplug is disabled, then taking the CPU down is not
989 * possible because takedown_cpu() and the architecture and
990 * subsystem specific mechanisms are not available. So the CPU
991 * which would be completely unplugged again needs to stay around
992 * in the current state.
994 return st
->state
<= CPUHP_BRINGUP_CPU
;
997 static int cpuhp_up_callbacks(unsigned int cpu
, struct cpuhp_cpu_state
*st
,
998 enum cpuhp_state target
)
1000 enum cpuhp_state prev_state
= st
->state
;
1003 ret
= cpuhp_invoke_callback_range(true, cpu
, st
, target
);
1005 pr_debug("CPU UP failed (%d) CPU %u state %s (%d)\n",
1006 ret
, cpu
, cpuhp_get_step(st
->state
)->name
,
1009 cpuhp_reset_state(cpu
, st
, prev_state
);
1010 if (can_rollback_cpu(st
))
1011 WARN_ON(cpuhp_invoke_callback_range(false, cpu
, st
,
1018 * The cpu hotplug threads manage the bringup and teardown of the cpus
1020 static int cpuhp_should_run(unsigned int cpu
)
1022 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
1024 return st
->should_run
;
1028 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
1029 * callbacks when a state gets [un]installed at runtime.
1031 * Each invocation of this function by the smpboot thread does a single AP
1034 * It has 3 modes of operation:
1035 * - single: runs st->cb_state
1036 * - up: runs ++st->state, while st->state < st->target
1037 * - down: runs st->state--, while st->state > st->target
1039 * When complete or on error, should_run is cleared and the completion is fired.
1041 static void cpuhp_thread_fun(unsigned int cpu
)
1043 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
1044 bool bringup
= st
->bringup
;
1045 enum cpuhp_state state
;
1047 if (WARN_ON_ONCE(!st
->should_run
))
1051 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
1052 * that if we see ->should_run we also see the rest of the state.
1057 * The BP holds the hotplug lock, but we're now running on the AP,
1058 * ensure that anybody asserting the lock is held, will actually find
1061 lockdep_acquire_cpus_lock();
1062 cpuhp_lock_acquire(bringup
);
1065 state
= st
->cb_state
;
1066 st
->should_run
= false;
1068 st
->should_run
= cpuhp_next_state(bringup
, &state
, st
, st
->target
);
1069 if (!st
->should_run
)
1073 WARN_ON_ONCE(!cpuhp_is_ap_state(state
));
1075 if (cpuhp_is_atomic_state(state
)) {
1076 local_irq_disable();
1077 st
->result
= cpuhp_invoke_callback(cpu
, state
, bringup
, st
->node
, &st
->last
);
1081 * STARTING/DYING must not fail!
1083 WARN_ON_ONCE(st
->result
);
1085 st
->result
= cpuhp_invoke_callback(cpu
, state
, bringup
, st
->node
, &st
->last
);
1090 * If we fail on a rollback, we're up a creek without no
1091 * paddle, no way forward, no way back. We loose, thanks for
1094 WARN_ON_ONCE(st
->rollback
);
1095 st
->should_run
= false;
1099 cpuhp_lock_release(bringup
);
1100 lockdep_release_cpus_lock();
1102 if (!st
->should_run
)
1103 complete_ap_thread(st
, bringup
);
1106 /* Invoke a single callback on a remote cpu */
1108 cpuhp_invoke_ap_callback(int cpu
, enum cpuhp_state state
, bool bringup
,
1109 struct hlist_node
*node
)
1111 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1114 if (!cpu_online(cpu
))
1117 cpuhp_lock_acquire(false);
1118 cpuhp_lock_release(false);
1120 cpuhp_lock_acquire(true);
1121 cpuhp_lock_release(true);
1124 * If we are up and running, use the hotplug thread. For early calls
1125 * we invoke the thread function directly.
1128 return cpuhp_invoke_callback(cpu
, state
, bringup
, node
, NULL
);
1130 st
->rollback
= false;
1134 st
->bringup
= bringup
;
1135 st
->cb_state
= state
;
1138 __cpuhp_kick_ap(st
);
1141 * If we failed and did a partial, do a rollback.
1143 if ((ret
= st
->result
) && st
->last
) {
1144 st
->rollback
= true;
1145 st
->bringup
= !bringup
;
1147 __cpuhp_kick_ap(st
);
1151 * Clean up the leftovers so the next hotplug operation wont use stale
1154 st
->node
= st
->last
= NULL
;
1158 static int cpuhp_kick_ap_work(unsigned int cpu
)
1160 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1161 enum cpuhp_state prev_state
= st
->state
;
1164 cpuhp_lock_acquire(false);
1165 cpuhp_lock_release(false);
1167 cpuhp_lock_acquire(true);
1168 cpuhp_lock_release(true);
1170 trace_cpuhp_enter(cpu
, st
->target
, prev_state
, cpuhp_kick_ap_work
);
1171 ret
= cpuhp_kick_ap(cpu
, st
, st
->target
);
1172 trace_cpuhp_exit(cpu
, st
->state
, prev_state
, ret
);
1177 static struct smp_hotplug_thread cpuhp_threads
= {
1178 .store
= &cpuhp_state
.thread
,
1179 .thread_should_run
= cpuhp_should_run
,
1180 .thread_fn
= cpuhp_thread_fun
,
1181 .thread_comm
= "cpuhp/%u",
1182 .selfparking
= true,
1185 static __init
void cpuhp_init_state(void)
1187 struct cpuhp_cpu_state
*st
;
1190 for_each_possible_cpu(cpu
) {
1191 st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1192 init_completion(&st
->done_up
);
1193 init_completion(&st
->done_down
);
1197 void __init
cpuhp_threads_init(void)
1200 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads
));
1201 kthread_unpark(this_cpu_read(cpuhp_state
.thread
));
1206 * Serialize hotplug trainwrecks outside of the cpu_hotplug_lock
1209 * The operation is still serialized against concurrent CPU hotplug via
1210 * cpu_add_remove_lock, i.e. CPU map protection. But it is _not_
1211 * serialized against other hotplug related activity like adding or
1212 * removing of state callbacks and state instances, which invoke either the
1213 * startup or the teardown callback of the affected state.
1215 * This is required for subsystems which are unfixable vs. CPU hotplug and
1216 * evade lock inversion problems by scheduling work which has to be
1217 * completed _before_ cpu_up()/_cpu_down() returns.
1219 * Don't even think about adding anything to this for any new code or even
1220 * drivers. It's only purpose is to keep existing lock order trainwrecks
1223 * For cpu_down() there might be valid reasons to finish cleanups which are
1224 * not required to be done under cpu_hotplug_lock, but that's a different
1225 * story and would be not invoked via this.
1227 static void cpu_up_down_serialize_trainwrecks(bool tasks_frozen
)
1230 * cpusets delegate hotplug operations to a worker to "solve" the
1231 * lock order problems. Wait for the worker, but only if tasks are
1232 * _not_ frozen (suspend, hibernate) as that would wait forever.
1234 * The wait is required because otherwise the hotplug operation
1235 * returns with inconsistent state, which could even be observed in
1236 * user space when a new CPU is brought up. The CPU plug uevent
1237 * would be delivered and user space reacting on it would fail to
1238 * move tasks to the newly plugged CPU up to the point where the
1239 * work has finished because up to that point the newly plugged CPU
1240 * is not assignable in cpusets/cgroups. On unplug that's not
1241 * necessarily a visible issue, but it is still inconsistent state,
1242 * which is the real problem which needs to be "fixed". This can't
1243 * prevent the transient state between scheduling the work and
1244 * returning from waiting for it.
1247 cpuset_wait_for_hotplug();
1250 #ifdef CONFIG_HOTPLUG_CPU
1251 #ifndef arch_clear_mm_cpumask_cpu
1252 #define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
1256 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
1259 * This function walks all processes, finds a valid mm struct for each one and
1260 * then clears a corresponding bit in mm's cpumask. While this all sounds
1261 * trivial, there are various non-obvious corner cases, which this function
1262 * tries to solve in a safe manner.
1264 * Also note that the function uses a somewhat relaxed locking scheme, so it may
1265 * be called only for an already offlined CPU.
1267 void clear_tasks_mm_cpumask(int cpu
)
1269 struct task_struct
*p
;
1272 * This function is called after the cpu is taken down and marked
1273 * offline, so its not like new tasks will ever get this cpu set in
1274 * their mm mask. -- Peter Zijlstra
1275 * Thus, we may use rcu_read_lock() here, instead of grabbing
1276 * full-fledged tasklist_lock.
1278 WARN_ON(cpu_online(cpu
));
1280 for_each_process(p
) {
1281 struct task_struct
*t
;
1284 * Main thread might exit, but other threads may still have
1285 * a valid mm. Find one.
1287 t
= find_lock_task_mm(p
);
1290 arch_clear_mm_cpumask_cpu(cpu
, t
->mm
);
1296 /* Take this CPU down. */
1297 static int take_cpu_down(void *_param
)
1299 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
1300 enum cpuhp_state target
= max((int)st
->target
, CPUHP_AP_OFFLINE
);
1301 int err
, cpu
= smp_processor_id();
1303 /* Ensure this CPU doesn't handle any more interrupts. */
1304 err
= __cpu_disable();
1309 * Must be called from CPUHP_TEARDOWN_CPU, which means, as we are going
1310 * down, that the current state is CPUHP_TEARDOWN_CPU - 1.
1312 WARN_ON(st
->state
!= (CPUHP_TEARDOWN_CPU
- 1));
1315 * Invoke the former CPU_DYING callbacks. DYING must not fail!
1317 cpuhp_invoke_callback_range_nofail(false, cpu
, st
, target
);
1319 /* Give up timekeeping duties */
1320 tick_handover_do_timer();
1321 /* Remove CPU from timer broadcasting */
1322 tick_offline_cpu(cpu
);
1323 /* Park the stopper thread */
1324 stop_machine_park(cpu
);
1328 static int takedown_cpu(unsigned int cpu
)
1330 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1333 /* Park the smpboot threads */
1334 kthread_park(st
->thread
);
1337 * Prevent irq alloc/free while the dying cpu reorganizes the
1338 * interrupt affinities.
1343 * So now all preempt/rcu users must observe !cpu_active().
1345 err
= stop_machine_cpuslocked(take_cpu_down
, NULL
, cpumask_of(cpu
));
1347 /* CPU refused to die */
1348 irq_unlock_sparse();
1349 /* Unpark the hotplug thread so we can rollback there */
1350 kthread_unpark(st
->thread
);
1353 BUG_ON(cpu_online(cpu
));
1356 * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
1357 * all runnable tasks from the CPU, there's only the idle task left now
1358 * that the migration thread is done doing the stop_machine thing.
1360 * Wait for the stop thread to go away.
1362 wait_for_ap_thread(st
, false);
1363 BUG_ON(st
->state
!= CPUHP_AP_IDLE_DEAD
);
1365 /* Interrupts are moved away from the dying cpu, reenable alloc/free */
1366 irq_unlock_sparse();
1368 hotplug_cpu__broadcast_tick_pull(cpu
);
1369 /* This actually kills the CPU. */
1372 cpuhp_bp_sync_dead(cpu
);
1374 tick_cleanup_dead_cpu(cpu
);
1375 rcutree_migrate_callbacks(cpu
);
1379 static void cpuhp_complete_idle_dead(void *arg
)
1381 struct cpuhp_cpu_state
*st
= arg
;
1383 complete_ap_thread(st
, false);
1386 void cpuhp_report_idle_dead(void)
1388 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
1390 BUG_ON(st
->state
!= CPUHP_AP_OFFLINE
);
1391 rcu_report_dead(smp_processor_id());
1392 st
->state
= CPUHP_AP_IDLE_DEAD
;
1394 * We cannot call complete after rcu_report_dead() so we delegate it
1397 smp_call_function_single(cpumask_first(cpu_online_mask
),
1398 cpuhp_complete_idle_dead
, st
, 0);
1401 static int cpuhp_down_callbacks(unsigned int cpu
, struct cpuhp_cpu_state
*st
,
1402 enum cpuhp_state target
)
1404 enum cpuhp_state prev_state
= st
->state
;
1407 ret
= cpuhp_invoke_callback_range(false, cpu
, st
, target
);
1409 pr_debug("CPU DOWN failed (%d) CPU %u state %s (%d)\n",
1410 ret
, cpu
, cpuhp_get_step(st
->state
)->name
,
1413 cpuhp_reset_state(cpu
, st
, prev_state
);
1415 if (st
->state
< prev_state
)
1416 WARN_ON(cpuhp_invoke_callback_range(true, cpu
, st
,
1423 /* Requires cpu_add_remove_lock to be held */
1424 static int __ref
_cpu_down(unsigned int cpu
, int tasks_frozen
,
1425 enum cpuhp_state target
)
1427 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1428 int prev_state
, ret
= 0;
1430 if (num_online_cpus() == 1)
1433 if (!cpu_present(cpu
))
1438 cpuhp_tasks_frozen
= tasks_frozen
;
1440 prev_state
= cpuhp_set_state(cpu
, st
, target
);
1442 * If the current CPU state is in the range of the AP hotplug thread,
1443 * then we need to kick the thread.
1445 if (st
->state
> CPUHP_TEARDOWN_CPU
) {
1446 st
->target
= max((int)target
, CPUHP_TEARDOWN_CPU
);
1447 ret
= cpuhp_kick_ap_work(cpu
);
1449 * The AP side has done the error rollback already. Just
1450 * return the error code..
1456 * We might have stopped still in the range of the AP hotplug
1457 * thread. Nothing to do anymore.
1459 if (st
->state
> CPUHP_TEARDOWN_CPU
)
1462 st
->target
= target
;
1465 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1466 * to do the further cleanups.
1468 ret
= cpuhp_down_callbacks(cpu
, st
, target
);
1469 if (ret
&& st
->state
< prev_state
) {
1470 if (st
->state
== CPUHP_TEARDOWN_CPU
) {
1471 cpuhp_reset_state(cpu
, st
, prev_state
);
1472 __cpuhp_kick_ap(st
);
1474 WARN(1, "DEAD callback error for CPU%d", cpu
);
1479 cpus_write_unlock();
1481 * Do post unplug cleanup. This is still protected against
1482 * concurrent CPU hotplug via cpu_add_remove_lock.
1484 lockup_detector_cleanup();
1486 cpu_up_down_serialize_trainwrecks(tasks_frozen
);
1490 struct cpu_down_work
{
1492 enum cpuhp_state target
;
1495 static long __cpu_down_maps_locked(void *arg
)
1497 struct cpu_down_work
*work
= arg
;
1499 return _cpu_down(work
->cpu
, 0, work
->target
);
1502 static int cpu_down_maps_locked(unsigned int cpu
, enum cpuhp_state target
)
1504 struct cpu_down_work work
= { .cpu
= cpu
, .target
= target
, };
1507 * If the platform does not support hotplug, report it explicitly to
1508 * differentiate it from a transient offlining failure.
1510 if (cc_platform_has(CC_ATTR_HOTPLUG_DISABLED
))
1512 if (cpu_hotplug_disabled
)
1516 * Ensure that the control task does not run on the to be offlined
1517 * CPU to prevent a deadlock against cfs_b->period_timer.
1519 cpu
= cpumask_any_but(cpu_online_mask
, cpu
);
1520 if (cpu
>= nr_cpu_ids
)
1522 return work_on_cpu(cpu
, __cpu_down_maps_locked
, &work
);
1525 static int cpu_down(unsigned int cpu
, enum cpuhp_state target
)
1529 cpu_maps_update_begin();
1530 err
= cpu_down_maps_locked(cpu
, target
);
1531 cpu_maps_update_done();
1536 * cpu_device_down - Bring down a cpu device
1537 * @dev: Pointer to the cpu device to offline
1539 * This function is meant to be used by device core cpu subsystem only.
1541 * Other subsystems should use remove_cpu() instead.
1543 * Return: %0 on success or a negative errno code
1545 int cpu_device_down(struct device
*dev
)
1547 return cpu_down(dev
->id
, CPUHP_OFFLINE
);
1550 int remove_cpu(unsigned int cpu
)
1554 lock_device_hotplug();
1555 ret
= device_offline(get_cpu_device(cpu
));
1556 unlock_device_hotplug();
1560 EXPORT_SYMBOL_GPL(remove_cpu
);
1562 void smp_shutdown_nonboot_cpus(unsigned int primary_cpu
)
1567 cpu_maps_update_begin();
1570 * Make certain the cpu I'm about to reboot on is online.
1572 * This is inline to what migrate_to_reboot_cpu() already do.
1574 if (!cpu_online(primary_cpu
))
1575 primary_cpu
= cpumask_first(cpu_online_mask
);
1577 for_each_online_cpu(cpu
) {
1578 if (cpu
== primary_cpu
)
1581 error
= cpu_down_maps_locked(cpu
, CPUHP_OFFLINE
);
1583 pr_err("Failed to offline CPU%d - error=%d",
1590 * Ensure all but the reboot CPU are offline.
1592 BUG_ON(num_online_cpus() > 1);
1595 * Make sure the CPUs won't be enabled by someone else after this
1596 * point. Kexec will reboot to a new kernel shortly resetting
1597 * everything along the way.
1599 cpu_hotplug_disabled
++;
1601 cpu_maps_update_done();
1605 #define takedown_cpu NULL
1606 #endif /*CONFIG_HOTPLUG_CPU*/
1609 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1610 * @cpu: cpu that just started
1612 * It must be called by the arch code on the new cpu, before the new cpu
1613 * enables interrupts and before the "boot" cpu returns from __cpu_up().
1615 void notify_cpu_starting(unsigned int cpu
)
1617 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1618 enum cpuhp_state target
= min((int)st
->target
, CPUHP_AP_ONLINE
);
1620 rcu_cpu_starting(cpu
); /* Enables RCU usage on this CPU. */
1621 cpumask_set_cpu(cpu
, &cpus_booted_once_mask
);
1624 * STARTING must not fail!
1626 cpuhp_invoke_callback_range_nofail(true, cpu
, st
, target
);
1630 * Called from the idle task. Wake up the controlling task which brings the
1631 * hotplug thread of the upcoming CPU up and then delegates the rest of the
1632 * online bringup to the hotplug thread.
1634 void cpuhp_online_idle(enum cpuhp_state state
)
1636 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
1638 /* Happens for the boot cpu */
1639 if (state
!= CPUHP_AP_ONLINE_IDLE
)
1642 cpuhp_ap_update_sync_state(SYNC_STATE_ONLINE
);
1645 * Unpark the stopper thread before we start the idle loop (and start
1646 * scheduling); this ensures the stopper task is always available.
1648 stop_machine_unpark(smp_processor_id());
1650 st
->state
= CPUHP_AP_ONLINE_IDLE
;
1651 complete_ap_thread(st
, true);
1654 /* Requires cpu_add_remove_lock to be held */
1655 static int _cpu_up(unsigned int cpu
, int tasks_frozen
, enum cpuhp_state target
)
1657 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1658 struct task_struct
*idle
;
1663 if (!cpu_present(cpu
)) {
1669 * The caller of cpu_up() might have raced with another
1670 * caller. Nothing to do.
1672 if (st
->state
>= target
)
1675 if (st
->state
== CPUHP_OFFLINE
) {
1676 /* Let it fail before we try to bring the cpu up */
1677 idle
= idle_thread_get(cpu
);
1679 ret
= PTR_ERR(idle
);
1684 * Reset stale stack state from the last time this CPU was online.
1686 scs_task_reset(idle
);
1687 kasan_unpoison_task_stack(idle
);
1690 cpuhp_tasks_frozen
= tasks_frozen
;
1692 cpuhp_set_state(cpu
, st
, target
);
1694 * If the current CPU state is in the range of the AP hotplug thread,
1695 * then we need to kick the thread once more.
1697 if (st
->state
> CPUHP_BRINGUP_CPU
) {
1698 ret
= cpuhp_kick_ap_work(cpu
);
1700 * The AP side has done the error rollback already. Just
1701 * return the error code..
1708 * Try to reach the target state. We max out on the BP at
1709 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1710 * responsible for bringing it up to the target state.
1712 target
= min((int)target
, CPUHP_BRINGUP_CPU
);
1713 ret
= cpuhp_up_callbacks(cpu
, st
, target
);
1715 cpus_write_unlock();
1717 cpu_up_down_serialize_trainwrecks(tasks_frozen
);
1721 static int cpu_up(unsigned int cpu
, enum cpuhp_state target
)
1725 if (!cpu_possible(cpu
)) {
1726 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1728 #if defined(CONFIG_IA64)
1729 pr_err("please check additional_cpus= boot parameter\n");
1734 err
= try_online_node(cpu_to_node(cpu
));
1738 cpu_maps_update_begin();
1740 if (cpu_hotplug_disabled
) {
1744 if (!cpu_smt_allowed(cpu
)) {
1749 err
= _cpu_up(cpu
, 0, target
);
1751 cpu_maps_update_done();
1756 * cpu_device_up - Bring up a cpu device
1757 * @dev: Pointer to the cpu device to online
1759 * This function is meant to be used by device core cpu subsystem only.
1761 * Other subsystems should use add_cpu() instead.
1763 * Return: %0 on success or a negative errno code
1765 int cpu_device_up(struct device
*dev
)
1767 return cpu_up(dev
->id
, CPUHP_ONLINE
);
1770 int add_cpu(unsigned int cpu
)
1774 lock_device_hotplug();
1775 ret
= device_online(get_cpu_device(cpu
));
1776 unlock_device_hotplug();
1780 EXPORT_SYMBOL_GPL(add_cpu
);
1783 * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1784 * @sleep_cpu: The cpu we hibernated on and should be brought up.
1786 * On some architectures like arm64, we can hibernate on any CPU, but on
1787 * wake up the CPU we hibernated on might be offline as a side effect of
1788 * using maxcpus= for example.
1790 * Return: %0 on success or a negative errno code
1792 int bringup_hibernate_cpu(unsigned int sleep_cpu
)
1796 if (!cpu_online(sleep_cpu
)) {
1797 pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
1798 ret
= cpu_up(sleep_cpu
, CPUHP_ONLINE
);
1800 pr_err("Failed to bring hibernate-CPU up!\n");
1807 static void __init
cpuhp_bringup_mask(const struct cpumask
*mask
, unsigned int ncpus
,
1808 enum cpuhp_state target
)
1812 for_each_cpu(cpu
, mask
) {
1813 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1815 if (cpu_up(cpu
, target
) && can_rollback_cpu(st
)) {
1817 * If this failed then cpu_up() might have only
1818 * rolled back to CPUHP_BP_KICK_AP for the final
1819 * online. Clean it up. NOOP if already rolled back.
1821 WARN_ON(cpuhp_invoke_callback_range(false, cpu
, st
, CPUHP_OFFLINE
));
1829 #ifdef CONFIG_HOTPLUG_PARALLEL
1830 static bool __cpuhp_parallel_bringup __ro_after_init
= true;
1832 static int __init
parallel_bringup_parse_param(char *arg
)
1834 return kstrtobool(arg
, &__cpuhp_parallel_bringup
);
1836 early_param("cpuhp.parallel", parallel_bringup_parse_param
);
1838 static inline bool cpuhp_smt_aware(void)
1840 return cpu_smt_max_threads
> 1;
1843 static inline const struct cpumask
*cpuhp_get_primary_thread_mask(void)
1845 return cpu_primary_thread_mask
;
1849 * On architectures which have enabled parallel bringup this invokes all BP
1850 * prepare states for each of the to be onlined APs first. The last state
1851 * sends the startup IPI to the APs. The APs proceed through the low level
1852 * bringup code in parallel and then wait for the control CPU to release
1853 * them one by one for the final onlining procedure.
1855 * This avoids waiting for each AP to respond to the startup IPI in
1856 * CPUHP_BRINGUP_CPU.
1858 static bool __init
cpuhp_bringup_cpus_parallel(unsigned int ncpus
)
1860 const struct cpumask
*mask
= cpu_present_mask
;
1862 if (__cpuhp_parallel_bringup
)
1863 __cpuhp_parallel_bringup
= arch_cpuhp_init_parallel_bringup();
1864 if (!__cpuhp_parallel_bringup
)
1867 if (cpuhp_smt_aware()) {
1868 const struct cpumask
*pmask
= cpuhp_get_primary_thread_mask();
1869 static struct cpumask tmp_mask __initdata
;
1872 * X86 requires to prevent that SMT siblings stopped while
1873 * the primary thread does a microcode update for various
1874 * reasons. Bring the primary threads up first.
1876 cpumask_and(&tmp_mask
, mask
, pmask
);
1877 cpuhp_bringup_mask(&tmp_mask
, ncpus
, CPUHP_BP_KICK_AP
);
1878 cpuhp_bringup_mask(&tmp_mask
, ncpus
, CPUHP_ONLINE
);
1879 /* Account for the online CPUs */
1880 ncpus
-= num_online_cpus();
1883 /* Create the mask for secondary CPUs */
1884 cpumask_andnot(&tmp_mask
, mask
, pmask
);
1888 /* Bring the not-yet started CPUs up */
1889 cpuhp_bringup_mask(mask
, ncpus
, CPUHP_BP_KICK_AP
);
1890 cpuhp_bringup_mask(mask
, ncpus
, CPUHP_ONLINE
);
1894 static inline bool cpuhp_bringup_cpus_parallel(unsigned int ncpus
) { return false; }
1895 #endif /* CONFIG_HOTPLUG_PARALLEL */
1897 void __init
bringup_nonboot_cpus(unsigned int setup_max_cpus
)
1899 /* Try parallel bringup optimization if enabled */
1900 if (cpuhp_bringup_cpus_parallel(setup_max_cpus
))
1903 /* Full per CPU serialized bringup */
1904 cpuhp_bringup_mask(cpu_present_mask
, setup_max_cpus
, CPUHP_ONLINE
);
1907 #ifdef CONFIG_PM_SLEEP_SMP
1908 static cpumask_var_t frozen_cpus
;
1910 int freeze_secondary_cpus(int primary
)
1914 cpu_maps_update_begin();
1915 if (primary
== -1) {
1916 primary
= cpumask_first(cpu_online_mask
);
1917 if (!housekeeping_cpu(primary
, HK_TYPE_TIMER
))
1918 primary
= housekeeping_any_cpu(HK_TYPE_TIMER
);
1920 if (!cpu_online(primary
))
1921 primary
= cpumask_first(cpu_online_mask
);
1925 * We take down all of the non-boot CPUs in one shot to avoid races
1926 * with the userspace trying to use the CPU hotplug at the same time
1928 cpumask_clear(frozen_cpus
);
1930 pr_info("Disabling non-boot CPUs ...\n");
1931 for_each_online_cpu(cpu
) {
1935 if (pm_wakeup_pending()) {
1936 pr_info("Wakeup pending. Abort CPU freeze\n");
1941 trace_suspend_resume(TPS("CPU_OFF"), cpu
, true);
1942 error
= _cpu_down(cpu
, 1, CPUHP_OFFLINE
);
1943 trace_suspend_resume(TPS("CPU_OFF"), cpu
, false);
1945 cpumask_set_cpu(cpu
, frozen_cpus
);
1947 pr_err("Error taking CPU%d down: %d\n", cpu
, error
);
1953 BUG_ON(num_online_cpus() > 1);
1955 pr_err("Non-boot CPUs are not disabled\n");
1958 * Make sure the CPUs won't be enabled by someone else. We need to do
1959 * this even in case of failure as all freeze_secondary_cpus() users are
1960 * supposed to do thaw_secondary_cpus() on the failure path.
1962 cpu_hotplug_disabled
++;
1964 cpu_maps_update_done();
1968 void __weak
arch_thaw_secondary_cpus_begin(void)
1972 void __weak
arch_thaw_secondary_cpus_end(void)
1976 void thaw_secondary_cpus(void)
1980 /* Allow everyone to use the CPU hotplug again */
1981 cpu_maps_update_begin();
1982 __cpu_hotplug_enable();
1983 if (cpumask_empty(frozen_cpus
))
1986 pr_info("Enabling non-boot CPUs ...\n");
1988 arch_thaw_secondary_cpus_begin();
1990 for_each_cpu(cpu
, frozen_cpus
) {
1991 trace_suspend_resume(TPS("CPU_ON"), cpu
, true);
1992 error
= _cpu_up(cpu
, 1, CPUHP_ONLINE
);
1993 trace_suspend_resume(TPS("CPU_ON"), cpu
, false);
1995 pr_info("CPU%d is up\n", cpu
);
1998 pr_warn("Error taking CPU%d up: %d\n", cpu
, error
);
2001 arch_thaw_secondary_cpus_end();
2003 cpumask_clear(frozen_cpus
);
2005 cpu_maps_update_done();
2008 static int __init
alloc_frozen_cpus(void)
2010 if (!alloc_cpumask_var(&frozen_cpus
, GFP_KERNEL
|__GFP_ZERO
))
2014 core_initcall(alloc_frozen_cpus
);
2017 * When callbacks for CPU hotplug notifications are being executed, we must
2018 * ensure that the state of the system with respect to the tasks being frozen
2019 * or not, as reported by the notification, remains unchanged *throughout the
2020 * duration* of the execution of the callbacks.
2021 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
2023 * This synchronization is implemented by mutually excluding regular CPU
2024 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
2025 * Hibernate notifications.
2028 cpu_hotplug_pm_callback(struct notifier_block
*nb
,
2029 unsigned long action
, void *ptr
)
2033 case PM_SUSPEND_PREPARE
:
2034 case PM_HIBERNATION_PREPARE
:
2035 cpu_hotplug_disable();
2038 case PM_POST_SUSPEND
:
2039 case PM_POST_HIBERNATION
:
2040 cpu_hotplug_enable();
2051 static int __init
cpu_hotplug_pm_sync_init(void)
2054 * cpu_hotplug_pm_callback has higher priority than x86
2055 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
2056 * to disable cpu hotplug to avoid cpu hotplug race.
2058 pm_notifier(cpu_hotplug_pm_callback
, 0);
2061 core_initcall(cpu_hotplug_pm_sync_init
);
2063 #endif /* CONFIG_PM_SLEEP_SMP */
2067 #endif /* CONFIG_SMP */
2069 /* Boot processor state steps */
2070 static struct cpuhp_step cpuhp_hp_states
[] = {
2073 .startup
.single
= NULL
,
2074 .teardown
.single
= NULL
,
2077 [CPUHP_CREATE_THREADS
]= {
2078 .name
= "threads:prepare",
2079 .startup
.single
= smpboot_create_threads
,
2080 .teardown
.single
= NULL
,
2083 [CPUHP_PERF_PREPARE
] = {
2084 .name
= "perf:prepare",
2085 .startup
.single
= perf_event_init_cpu
,
2086 .teardown
.single
= perf_event_exit_cpu
,
2088 [CPUHP_RANDOM_PREPARE
] = {
2089 .name
= "random:prepare",
2090 .startup
.single
= random_prepare_cpu
,
2091 .teardown
.single
= NULL
,
2093 [CPUHP_WORKQUEUE_PREP
] = {
2094 .name
= "workqueue:prepare",
2095 .startup
.single
= workqueue_prepare_cpu
,
2096 .teardown
.single
= NULL
,
2098 [CPUHP_HRTIMERS_PREPARE
] = {
2099 .name
= "hrtimers:prepare",
2100 .startup
.single
= hrtimers_prepare_cpu
,
2101 .teardown
.single
= hrtimers_dead_cpu
,
2103 [CPUHP_SMPCFD_PREPARE
] = {
2104 .name
= "smpcfd:prepare",
2105 .startup
.single
= smpcfd_prepare_cpu
,
2106 .teardown
.single
= smpcfd_dead_cpu
,
2108 [CPUHP_RELAY_PREPARE
] = {
2109 .name
= "relay:prepare",
2110 .startup
.single
= relay_prepare_cpu
,
2111 .teardown
.single
= NULL
,
2113 [CPUHP_SLAB_PREPARE
] = {
2114 .name
= "slab:prepare",
2115 .startup
.single
= slab_prepare_cpu
,
2116 .teardown
.single
= slab_dead_cpu
,
2118 [CPUHP_RCUTREE_PREP
] = {
2119 .name
= "RCU/tree:prepare",
2120 .startup
.single
= rcutree_prepare_cpu
,
2121 .teardown
.single
= rcutree_dead_cpu
,
2124 * On the tear-down path, timers_dead_cpu() must be invoked
2125 * before blk_mq_queue_reinit_notify() from notify_dead(),
2126 * otherwise a RCU stall occurs.
2128 [CPUHP_TIMERS_PREPARE
] = {
2129 .name
= "timers:prepare",
2130 .startup
.single
= timers_prepare_cpu
,
2131 .teardown
.single
= timers_dead_cpu
,
2134 #ifdef CONFIG_HOTPLUG_SPLIT_STARTUP
2136 * Kicks the AP alive. AP will wait in cpuhp_ap_sync_alive() until
2137 * the next step will release it.
2139 [CPUHP_BP_KICK_AP
] = {
2140 .name
= "cpu:kick_ap",
2141 .startup
.single
= cpuhp_kick_ap_alive
,
2145 * Waits for the AP to reach cpuhp_ap_sync_alive() and then
2146 * releases it for the complete bringup.
2148 [CPUHP_BRINGUP_CPU
] = {
2149 .name
= "cpu:bringup",
2150 .startup
.single
= cpuhp_bringup_ap
,
2151 .teardown
.single
= finish_cpu
,
2156 * All-in-one CPU bringup state which includes the kick alive.
2158 [CPUHP_BRINGUP_CPU
] = {
2159 .name
= "cpu:bringup",
2160 .startup
.single
= bringup_cpu
,
2161 .teardown
.single
= finish_cpu
,
2165 /* Final state before CPU kills itself */
2166 [CPUHP_AP_IDLE_DEAD
] = {
2167 .name
= "idle:dead",
2170 * Last state before CPU enters the idle loop to die. Transient state
2171 * for synchronization.
2173 [CPUHP_AP_OFFLINE
] = {
2174 .name
= "ap:offline",
2177 /* First state is scheduler control. Interrupts are disabled */
2178 [CPUHP_AP_SCHED_STARTING
] = {
2179 .name
= "sched:starting",
2180 .startup
.single
= sched_cpu_starting
,
2181 .teardown
.single
= sched_cpu_dying
,
2183 [CPUHP_AP_RCUTREE_DYING
] = {
2184 .name
= "RCU/tree:dying",
2185 .startup
.single
= NULL
,
2186 .teardown
.single
= rcutree_dying_cpu
,
2188 [CPUHP_AP_SMPCFD_DYING
] = {
2189 .name
= "smpcfd:dying",
2190 .startup
.single
= NULL
,
2191 .teardown
.single
= smpcfd_dying_cpu
,
2193 /* Entry state on starting. Interrupts enabled from here on. Transient
2194 * state for synchronsization */
2195 [CPUHP_AP_ONLINE
] = {
2196 .name
= "ap:online",
2199 * Handled on control processor until the plugged processor manages
2202 [CPUHP_TEARDOWN_CPU
] = {
2203 .name
= "cpu:teardown",
2204 .startup
.single
= NULL
,
2205 .teardown
.single
= takedown_cpu
,
2209 [CPUHP_AP_SCHED_WAIT_EMPTY
] = {
2210 .name
= "sched:waitempty",
2211 .startup
.single
= NULL
,
2212 .teardown
.single
= sched_cpu_wait_empty
,
2215 /* Handle smpboot threads park/unpark */
2216 [CPUHP_AP_SMPBOOT_THREADS
] = {
2217 .name
= "smpboot/threads:online",
2218 .startup
.single
= smpboot_unpark_threads
,
2219 .teardown
.single
= smpboot_park_threads
,
2221 [CPUHP_AP_IRQ_AFFINITY_ONLINE
] = {
2222 .name
= "irq/affinity:online",
2223 .startup
.single
= irq_affinity_online_cpu
,
2224 .teardown
.single
= NULL
,
2226 [CPUHP_AP_PERF_ONLINE
] = {
2227 .name
= "perf:online",
2228 .startup
.single
= perf_event_init_cpu
,
2229 .teardown
.single
= perf_event_exit_cpu
,
2231 [CPUHP_AP_WATCHDOG_ONLINE
] = {
2232 .name
= "lockup_detector:online",
2233 .startup
.single
= lockup_detector_online_cpu
,
2234 .teardown
.single
= lockup_detector_offline_cpu
,
2236 [CPUHP_AP_WORKQUEUE_ONLINE
] = {
2237 .name
= "workqueue:online",
2238 .startup
.single
= workqueue_online_cpu
,
2239 .teardown
.single
= workqueue_offline_cpu
,
2241 [CPUHP_AP_RANDOM_ONLINE
] = {
2242 .name
= "random:online",
2243 .startup
.single
= random_online_cpu
,
2244 .teardown
.single
= NULL
,
2246 [CPUHP_AP_RCUTREE_ONLINE
] = {
2247 .name
= "RCU/tree:online",
2248 .startup
.single
= rcutree_online_cpu
,
2249 .teardown
.single
= rcutree_offline_cpu
,
2253 * The dynamically registered state space is here
2257 /* Last state is scheduler control setting the cpu active */
2258 [CPUHP_AP_ACTIVE
] = {
2259 .name
= "sched:active",
2260 .startup
.single
= sched_cpu_activate
,
2261 .teardown
.single
= sched_cpu_deactivate
,
2265 /* CPU is fully up and running. */
2268 .startup
.single
= NULL
,
2269 .teardown
.single
= NULL
,
2273 /* Sanity check for callbacks */
2274 static int cpuhp_cb_check(enum cpuhp_state state
)
2276 if (state
<= CPUHP_OFFLINE
|| state
>= CPUHP_ONLINE
)
2282 * Returns a free for dynamic slot assignment of the Online state. The states
2283 * are protected by the cpuhp_slot_states mutex and an empty slot is identified
2284 * by having no name assigned.
2286 static int cpuhp_reserve_state(enum cpuhp_state state
)
2288 enum cpuhp_state i
, end
;
2289 struct cpuhp_step
*step
;
2292 case CPUHP_AP_ONLINE_DYN
:
2293 step
= cpuhp_hp_states
+ CPUHP_AP_ONLINE_DYN
;
2294 end
= CPUHP_AP_ONLINE_DYN_END
;
2296 case CPUHP_BP_PREPARE_DYN
:
2297 step
= cpuhp_hp_states
+ CPUHP_BP_PREPARE_DYN
;
2298 end
= CPUHP_BP_PREPARE_DYN_END
;
2304 for (i
= state
; i
<= end
; i
++, step
++) {
2308 WARN(1, "No more dynamic states available for CPU hotplug\n");
2312 static int cpuhp_store_callbacks(enum cpuhp_state state
, const char *name
,
2313 int (*startup
)(unsigned int cpu
),
2314 int (*teardown
)(unsigned int cpu
),
2315 bool multi_instance
)
2317 /* (Un)Install the callbacks for further cpu hotplug operations */
2318 struct cpuhp_step
*sp
;
2322 * If name is NULL, then the state gets removed.
2324 * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
2325 * the first allocation from these dynamic ranges, so the removal
2326 * would trigger a new allocation and clear the wrong (already
2327 * empty) state, leaving the callbacks of the to be cleared state
2328 * dangling, which causes wreckage on the next hotplug operation.
2330 if (name
&& (state
== CPUHP_AP_ONLINE_DYN
||
2331 state
== CPUHP_BP_PREPARE_DYN
)) {
2332 ret
= cpuhp_reserve_state(state
);
2337 sp
= cpuhp_get_step(state
);
2338 if (name
&& sp
->name
)
2341 sp
->startup
.single
= startup
;
2342 sp
->teardown
.single
= teardown
;
2344 sp
->multi_instance
= multi_instance
;
2345 INIT_HLIST_HEAD(&sp
->list
);
2349 static void *cpuhp_get_teardown_cb(enum cpuhp_state state
)
2351 return cpuhp_get_step(state
)->teardown
.single
;
2355 * Call the startup/teardown function for a step either on the AP or
2356 * on the current CPU.
2358 static int cpuhp_issue_call(int cpu
, enum cpuhp_state state
, bool bringup
,
2359 struct hlist_node
*node
)
2361 struct cpuhp_step
*sp
= cpuhp_get_step(state
);
2365 * If there's nothing to do, we done.
2366 * Relies on the union for multi_instance.
2368 if (cpuhp_step_empty(bringup
, sp
))
2371 * The non AP bound callbacks can fail on bringup. On teardown
2372 * e.g. module removal we crash for now.
2375 if (cpuhp_is_ap_state(state
))
2376 ret
= cpuhp_invoke_ap_callback(cpu
, state
, bringup
, node
);
2378 ret
= cpuhp_invoke_callback(cpu
, state
, bringup
, node
, NULL
);
2380 ret
= cpuhp_invoke_callback(cpu
, state
, bringup
, node
, NULL
);
2382 BUG_ON(ret
&& !bringup
);
2387 * Called from __cpuhp_setup_state on a recoverable failure.
2389 * Note: The teardown callbacks for rollback are not allowed to fail!
2391 static void cpuhp_rollback_install(int failedcpu
, enum cpuhp_state state
,
2392 struct hlist_node
*node
)
2396 /* Roll back the already executed steps on the other cpus */
2397 for_each_present_cpu(cpu
) {
2398 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
2399 int cpustate
= st
->state
;
2401 if (cpu
>= failedcpu
)
2404 /* Did we invoke the startup call on that cpu ? */
2405 if (cpustate
>= state
)
2406 cpuhp_issue_call(cpu
, state
, false, node
);
2410 int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state
,
2411 struct hlist_node
*node
,
2414 struct cpuhp_step
*sp
;
2418 lockdep_assert_cpus_held();
2420 sp
= cpuhp_get_step(state
);
2421 if (sp
->multi_instance
== false)
2424 mutex_lock(&cpuhp_state_mutex
);
2426 if (!invoke
|| !sp
->startup
.multi
)
2430 * Try to call the startup callback for each present cpu
2431 * depending on the hotplug state of the cpu.
2433 for_each_present_cpu(cpu
) {
2434 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
2435 int cpustate
= st
->state
;
2437 if (cpustate
< state
)
2440 ret
= cpuhp_issue_call(cpu
, state
, true, node
);
2442 if (sp
->teardown
.multi
)
2443 cpuhp_rollback_install(cpu
, state
, node
);
2449 hlist_add_head(node
, &sp
->list
);
2451 mutex_unlock(&cpuhp_state_mutex
);
2455 int __cpuhp_state_add_instance(enum cpuhp_state state
, struct hlist_node
*node
,
2461 ret
= __cpuhp_state_add_instance_cpuslocked(state
, node
, invoke
);
2465 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance
);
2468 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
2469 * @state: The state to setup
2470 * @name: Name of the step
2471 * @invoke: If true, the startup function is invoked for cpus where
2472 * cpu state >= @state
2473 * @startup: startup callback function
2474 * @teardown: teardown callback function
2475 * @multi_instance: State is set up for multiple instances which get
2478 * The caller needs to hold cpus read locked while calling this function.
2481 * Positive state number if @state is CPUHP_AP_ONLINE_DYN;
2482 * 0 for all other states
2483 * On failure: proper (negative) error code
2485 int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state
,
2486 const char *name
, bool invoke
,
2487 int (*startup
)(unsigned int cpu
),
2488 int (*teardown
)(unsigned int cpu
),
2489 bool multi_instance
)
2494 lockdep_assert_cpus_held();
2496 if (cpuhp_cb_check(state
) || !name
)
2499 mutex_lock(&cpuhp_state_mutex
);
2501 ret
= cpuhp_store_callbacks(state
, name
, startup
, teardown
,
2504 dynstate
= state
== CPUHP_AP_ONLINE_DYN
;
2505 if (ret
> 0 && dynstate
) {
2510 if (ret
|| !invoke
|| !startup
)
2514 * Try to call the startup callback for each present cpu
2515 * depending on the hotplug state of the cpu.
2517 for_each_present_cpu(cpu
) {
2518 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
2519 int cpustate
= st
->state
;
2521 if (cpustate
< state
)
2524 ret
= cpuhp_issue_call(cpu
, state
, true, NULL
);
2527 cpuhp_rollback_install(cpu
, state
, NULL
);
2528 cpuhp_store_callbacks(state
, NULL
, NULL
, NULL
, false);
2533 mutex_unlock(&cpuhp_state_mutex
);
2535 * If the requested state is CPUHP_AP_ONLINE_DYN, return the
2536 * dynamically allocated state in case of success.
2538 if (!ret
&& dynstate
)
2542 EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked
);
2544 int __cpuhp_setup_state(enum cpuhp_state state
,
2545 const char *name
, bool invoke
,
2546 int (*startup
)(unsigned int cpu
),
2547 int (*teardown
)(unsigned int cpu
),
2548 bool multi_instance
)
2553 ret
= __cpuhp_setup_state_cpuslocked(state
, name
, invoke
, startup
,
2554 teardown
, multi_instance
);
2558 EXPORT_SYMBOL(__cpuhp_setup_state
);
2560 int __cpuhp_state_remove_instance(enum cpuhp_state state
,
2561 struct hlist_node
*node
, bool invoke
)
2563 struct cpuhp_step
*sp
= cpuhp_get_step(state
);
2566 BUG_ON(cpuhp_cb_check(state
));
2568 if (!sp
->multi_instance
)
2572 mutex_lock(&cpuhp_state_mutex
);
2574 if (!invoke
|| !cpuhp_get_teardown_cb(state
))
2577 * Call the teardown callback for each present cpu depending
2578 * on the hotplug state of the cpu. This function is not
2579 * allowed to fail currently!
2581 for_each_present_cpu(cpu
) {
2582 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
2583 int cpustate
= st
->state
;
2585 if (cpustate
>= state
)
2586 cpuhp_issue_call(cpu
, state
, false, node
);
2591 mutex_unlock(&cpuhp_state_mutex
);
2596 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance
);
2599 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2600 * @state: The state to remove
2601 * @invoke: If true, the teardown function is invoked for cpus where
2602 * cpu state >= @state
2604 * The caller needs to hold cpus read locked while calling this function.
2605 * The teardown callback is currently not allowed to fail. Think
2606 * about module removal!
2608 void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state
, bool invoke
)
2610 struct cpuhp_step
*sp
= cpuhp_get_step(state
);
2613 BUG_ON(cpuhp_cb_check(state
));
2615 lockdep_assert_cpus_held();
2617 mutex_lock(&cpuhp_state_mutex
);
2618 if (sp
->multi_instance
) {
2619 WARN(!hlist_empty(&sp
->list
),
2620 "Error: Removing state %d which has instances left.\n",
2625 if (!invoke
|| !cpuhp_get_teardown_cb(state
))
2629 * Call the teardown callback for each present cpu depending
2630 * on the hotplug state of the cpu. This function is not
2631 * allowed to fail currently!
2633 for_each_present_cpu(cpu
) {
2634 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
2635 int cpustate
= st
->state
;
2637 if (cpustate
>= state
)
2638 cpuhp_issue_call(cpu
, state
, false, NULL
);
2641 cpuhp_store_callbacks(state
, NULL
, NULL
, NULL
, false);
2642 mutex_unlock(&cpuhp_state_mutex
);
2644 EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked
);
2646 void __cpuhp_remove_state(enum cpuhp_state state
, bool invoke
)
2649 __cpuhp_remove_state_cpuslocked(state
, invoke
);
2652 EXPORT_SYMBOL(__cpuhp_remove_state
);
2654 #ifdef CONFIG_HOTPLUG_SMT
2655 static void cpuhp_offline_cpu_device(unsigned int cpu
)
2657 struct device
*dev
= get_cpu_device(cpu
);
2659 dev
->offline
= true;
2660 /* Tell user space about the state change */
2661 kobject_uevent(&dev
->kobj
, KOBJ_OFFLINE
);
2664 static void cpuhp_online_cpu_device(unsigned int cpu
)
2666 struct device
*dev
= get_cpu_device(cpu
);
2668 dev
->offline
= false;
2669 /* Tell user space about the state change */
2670 kobject_uevent(&dev
->kobj
, KOBJ_ONLINE
);
2673 int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval
)
2677 cpu_maps_update_begin();
2678 for_each_online_cpu(cpu
) {
2679 if (topology_is_primary_thread(cpu
))
2682 * Disable can be called with CPU_SMT_ENABLED when changing
2683 * from a higher to lower number of SMT threads per core.
2685 if (ctrlval
== CPU_SMT_ENABLED
&& cpu_smt_thread_allowed(cpu
))
2687 ret
= cpu_down_maps_locked(cpu
, CPUHP_OFFLINE
);
2691 * As this needs to hold the cpu maps lock it's impossible
2692 * to call device_offline() because that ends up calling
2693 * cpu_down() which takes cpu maps lock. cpu maps lock
2694 * needs to be held as this might race against in kernel
2695 * abusers of the hotplug machinery (thermal management).
2697 * So nothing would update device:offline state. That would
2698 * leave the sysfs entry stale and prevent onlining after
2699 * smt control has been changed to 'off' again. This is
2700 * called under the sysfs hotplug lock, so it is properly
2701 * serialized against the regular offline usage.
2703 cpuhp_offline_cpu_device(cpu
);
2706 cpu_smt_control
= ctrlval
;
2707 cpu_maps_update_done();
2711 int cpuhp_smt_enable(void)
2715 cpu_maps_update_begin();
2716 cpu_smt_control
= CPU_SMT_ENABLED
;
2717 for_each_present_cpu(cpu
) {
2718 /* Skip online CPUs and CPUs on offline nodes */
2719 if (cpu_online(cpu
) || !node_online(cpu_to_node(cpu
)))
2721 if (!cpu_smt_thread_allowed(cpu
))
2723 ret
= _cpu_up(cpu
, 0, CPUHP_ONLINE
);
2726 /* See comment in cpuhp_smt_disable() */
2727 cpuhp_online_cpu_device(cpu
);
2729 cpu_maps_update_done();
2734 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
2735 static ssize_t
state_show(struct device
*dev
,
2736 struct device_attribute
*attr
, char *buf
)
2738 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
2740 return sprintf(buf
, "%d\n", st
->state
);
2742 static DEVICE_ATTR_RO(state
);
2744 static ssize_t
target_store(struct device
*dev
, struct device_attribute
*attr
,
2745 const char *buf
, size_t count
)
2747 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
2748 struct cpuhp_step
*sp
;
2751 ret
= kstrtoint(buf
, 10, &target
);
2755 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
2756 if (target
< CPUHP_OFFLINE
|| target
> CPUHP_ONLINE
)
2759 if (target
!= CPUHP_OFFLINE
&& target
!= CPUHP_ONLINE
)
2763 ret
= lock_device_hotplug_sysfs();
2767 mutex_lock(&cpuhp_state_mutex
);
2768 sp
= cpuhp_get_step(target
);
2769 ret
= !sp
->name
|| sp
->cant_stop
? -EINVAL
: 0;
2770 mutex_unlock(&cpuhp_state_mutex
);
2774 if (st
->state
< target
)
2775 ret
= cpu_up(dev
->id
, target
);
2776 else if (st
->state
> target
)
2777 ret
= cpu_down(dev
->id
, target
);
2778 else if (WARN_ON(st
->target
!= target
))
2779 st
->target
= target
;
2781 unlock_device_hotplug();
2782 return ret
? ret
: count
;
2785 static ssize_t
target_show(struct device
*dev
,
2786 struct device_attribute
*attr
, char *buf
)
2788 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
2790 return sprintf(buf
, "%d\n", st
->target
);
2792 static DEVICE_ATTR_RW(target
);
2794 static ssize_t
fail_store(struct device
*dev
, struct device_attribute
*attr
,
2795 const char *buf
, size_t count
)
2797 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
2798 struct cpuhp_step
*sp
;
2801 ret
= kstrtoint(buf
, 10, &fail
);
2805 if (fail
== CPUHP_INVALID
) {
2810 if (fail
< CPUHP_OFFLINE
|| fail
> CPUHP_ONLINE
)
2814 * Cannot fail STARTING/DYING callbacks.
2816 if (cpuhp_is_atomic_state(fail
))
2820 * DEAD callbacks cannot fail...
2821 * ... neither can CPUHP_BRINGUP_CPU during hotunplug. The latter
2822 * triggering STARTING callbacks, a failure in this state would
2825 if (fail
<= CPUHP_BRINGUP_CPU
&& st
->state
> CPUHP_BRINGUP_CPU
)
2829 * Cannot fail anything that doesn't have callbacks.
2831 mutex_lock(&cpuhp_state_mutex
);
2832 sp
= cpuhp_get_step(fail
);
2833 if (!sp
->startup
.single
&& !sp
->teardown
.single
)
2835 mutex_unlock(&cpuhp_state_mutex
);
2844 static ssize_t
fail_show(struct device
*dev
,
2845 struct device_attribute
*attr
, char *buf
)
2847 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
2849 return sprintf(buf
, "%d\n", st
->fail
);
2852 static DEVICE_ATTR_RW(fail
);
2854 static struct attribute
*cpuhp_cpu_attrs
[] = {
2855 &dev_attr_state
.attr
,
2856 &dev_attr_target
.attr
,
2857 &dev_attr_fail
.attr
,
2861 static const struct attribute_group cpuhp_cpu_attr_group
= {
2862 .attrs
= cpuhp_cpu_attrs
,
2867 static ssize_t
states_show(struct device
*dev
,
2868 struct device_attribute
*attr
, char *buf
)
2870 ssize_t cur
, res
= 0;
2873 mutex_lock(&cpuhp_state_mutex
);
2874 for (i
= CPUHP_OFFLINE
; i
<= CPUHP_ONLINE
; i
++) {
2875 struct cpuhp_step
*sp
= cpuhp_get_step(i
);
2878 cur
= sprintf(buf
, "%3d: %s\n", i
, sp
->name
);
2883 mutex_unlock(&cpuhp_state_mutex
);
2886 static DEVICE_ATTR_RO(states
);
2888 static struct attribute
*cpuhp_cpu_root_attrs
[] = {
2889 &dev_attr_states
.attr
,
2893 static const struct attribute_group cpuhp_cpu_root_attr_group
= {
2894 .attrs
= cpuhp_cpu_root_attrs
,
2899 #ifdef CONFIG_HOTPLUG_SMT
2901 static bool cpu_smt_num_threads_valid(unsigned int threads
)
2903 if (IS_ENABLED(CONFIG_SMT_NUM_THREADS_DYNAMIC
))
2904 return threads
>= 1 && threads
<= cpu_smt_max_threads
;
2905 return threads
== 1 || threads
== cpu_smt_max_threads
;
2909 __store_smt_control(struct device
*dev
, struct device_attribute
*attr
,
2910 const char *buf
, size_t count
)
2912 int ctrlval
, ret
, num_threads
, orig_threads
;
2915 if (cpu_smt_control
== CPU_SMT_FORCE_DISABLED
)
2918 if (cpu_smt_control
== CPU_SMT_NOT_SUPPORTED
)
2921 if (sysfs_streq(buf
, "on")) {
2922 ctrlval
= CPU_SMT_ENABLED
;
2923 num_threads
= cpu_smt_max_threads
;
2924 } else if (sysfs_streq(buf
, "off")) {
2925 ctrlval
= CPU_SMT_DISABLED
;
2927 } else if (sysfs_streq(buf
, "forceoff")) {
2928 ctrlval
= CPU_SMT_FORCE_DISABLED
;
2930 } else if (kstrtoint(buf
, 10, &num_threads
) == 0) {
2931 if (num_threads
== 1)
2932 ctrlval
= CPU_SMT_DISABLED
;
2933 else if (cpu_smt_num_threads_valid(num_threads
))
2934 ctrlval
= CPU_SMT_ENABLED
;
2941 ret
= lock_device_hotplug_sysfs();
2945 orig_threads
= cpu_smt_num_threads
;
2946 cpu_smt_num_threads
= num_threads
;
2948 force_off
= ctrlval
!= cpu_smt_control
&& ctrlval
== CPU_SMT_FORCE_DISABLED
;
2950 if (num_threads
> orig_threads
)
2951 ret
= cpuhp_smt_enable();
2952 else if (num_threads
< orig_threads
|| force_off
)
2953 ret
= cpuhp_smt_disable(ctrlval
);
2955 unlock_device_hotplug();
2956 return ret
? ret
: count
;
2959 #else /* !CONFIG_HOTPLUG_SMT */
2961 __store_smt_control(struct device
*dev
, struct device_attribute
*attr
,
2962 const char *buf
, size_t count
)
2966 #endif /* CONFIG_HOTPLUG_SMT */
2968 static const char *smt_states
[] = {
2969 [CPU_SMT_ENABLED
] = "on",
2970 [CPU_SMT_DISABLED
] = "off",
2971 [CPU_SMT_FORCE_DISABLED
] = "forceoff",
2972 [CPU_SMT_NOT_SUPPORTED
] = "notsupported",
2973 [CPU_SMT_NOT_IMPLEMENTED
] = "notimplemented",
2976 static ssize_t
control_show(struct device
*dev
,
2977 struct device_attribute
*attr
, char *buf
)
2979 const char *state
= smt_states
[cpu_smt_control
];
2981 #ifdef CONFIG_HOTPLUG_SMT
2983 * If SMT is enabled but not all threads are enabled then show the
2984 * number of threads. If all threads are enabled show "on". Otherwise
2985 * show the state name.
2987 if (cpu_smt_control
== CPU_SMT_ENABLED
&&
2988 cpu_smt_num_threads
!= cpu_smt_max_threads
)
2989 return sysfs_emit(buf
, "%d\n", cpu_smt_num_threads
);
2992 return snprintf(buf
, PAGE_SIZE
- 2, "%s\n", state
);
2995 static ssize_t
control_store(struct device
*dev
, struct device_attribute
*attr
,
2996 const char *buf
, size_t count
)
2998 return __store_smt_control(dev
, attr
, buf
, count
);
3000 static DEVICE_ATTR_RW(control
);
3002 static ssize_t
active_show(struct device
*dev
,
3003 struct device_attribute
*attr
, char *buf
)
3005 return snprintf(buf
, PAGE_SIZE
- 2, "%d\n", sched_smt_active());
3007 static DEVICE_ATTR_RO(active
);
3009 static struct attribute
*cpuhp_smt_attrs
[] = {
3010 &dev_attr_control
.attr
,
3011 &dev_attr_active
.attr
,
3015 static const struct attribute_group cpuhp_smt_attr_group
= {
3016 .attrs
= cpuhp_smt_attrs
,
3021 static int __init
cpu_smt_sysfs_init(void)
3023 struct device
*dev_root
;
3026 dev_root
= bus_get_dev_root(&cpu_subsys
);
3028 ret
= sysfs_create_group(&dev_root
->kobj
, &cpuhp_smt_attr_group
);
3029 put_device(dev_root
);
3034 static int __init
cpuhp_sysfs_init(void)
3036 struct device
*dev_root
;
3039 ret
= cpu_smt_sysfs_init();
3043 dev_root
= bus_get_dev_root(&cpu_subsys
);
3045 ret
= sysfs_create_group(&dev_root
->kobj
, &cpuhp_cpu_root_attr_group
);
3046 put_device(dev_root
);
3051 for_each_possible_cpu(cpu
) {
3052 struct device
*dev
= get_cpu_device(cpu
);
3056 ret
= sysfs_create_group(&dev
->kobj
, &cpuhp_cpu_attr_group
);
3062 device_initcall(cpuhp_sysfs_init
);
3063 #endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */
3066 * cpu_bit_bitmap[] is a special, "compressed" data structure that
3067 * represents all NR_CPUS bits binary values of 1<<nr.
3069 * It is used by cpumask_of() to get a constant address to a CPU
3070 * mask value that has a single bit set only.
3073 /* cpu_bit_bitmap[0] is empty - so we can back into it */
3074 #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
3075 #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
3076 #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
3077 #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
3079 const unsigned long cpu_bit_bitmap
[BITS_PER_LONG
+1][BITS_TO_LONGS(NR_CPUS
)] = {
3081 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
3082 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
3083 #if BITS_PER_LONG > 32
3084 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
3085 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
3088 EXPORT_SYMBOL_GPL(cpu_bit_bitmap
);
3090 const DECLARE_BITMAP(cpu_all_bits
, NR_CPUS
) = CPU_BITS_ALL
;
3091 EXPORT_SYMBOL(cpu_all_bits
);
3093 #ifdef CONFIG_INIT_ALL_POSSIBLE
3094 struct cpumask __cpu_possible_mask __read_mostly
3097 struct cpumask __cpu_possible_mask __read_mostly
;
3099 EXPORT_SYMBOL(__cpu_possible_mask
);
3101 struct cpumask __cpu_online_mask __read_mostly
;
3102 EXPORT_SYMBOL(__cpu_online_mask
);
3104 struct cpumask __cpu_present_mask __read_mostly
;
3105 EXPORT_SYMBOL(__cpu_present_mask
);
3107 struct cpumask __cpu_active_mask __read_mostly
;
3108 EXPORT_SYMBOL(__cpu_active_mask
);
3110 struct cpumask __cpu_dying_mask __read_mostly
;
3111 EXPORT_SYMBOL(__cpu_dying_mask
);
3113 atomic_t __num_online_cpus __read_mostly
;
3114 EXPORT_SYMBOL(__num_online_cpus
);
3116 void init_cpu_present(const struct cpumask
*src
)
3118 cpumask_copy(&__cpu_present_mask
, src
);
3121 void init_cpu_possible(const struct cpumask
*src
)
3123 cpumask_copy(&__cpu_possible_mask
, src
);
3126 void init_cpu_online(const struct cpumask
*src
)
3128 cpumask_copy(&__cpu_online_mask
, src
);
3131 void set_cpu_online(unsigned int cpu
, bool online
)
3134 * atomic_inc/dec() is required to handle the horrid abuse of this
3135 * function by the reboot and kexec code which invoke it from
3136 * IPI/NMI broadcasts when shutting down CPUs. Invocation from
3137 * regular CPU hotplug is properly serialized.
3139 * Note, that the fact that __num_online_cpus is of type atomic_t
3140 * does not protect readers which are not serialized against
3141 * concurrent hotplug operations.
3144 if (!cpumask_test_and_set_cpu(cpu
, &__cpu_online_mask
))
3145 atomic_inc(&__num_online_cpus
);
3147 if (cpumask_test_and_clear_cpu(cpu
, &__cpu_online_mask
))
3148 atomic_dec(&__num_online_cpus
);
3153 * Activate the first processor.
3155 void __init
boot_cpu_init(void)
3157 int cpu
= smp_processor_id();
3159 /* Mark the boot cpu "present", "online" etc for SMP and UP case */
3160 set_cpu_online(cpu
, true);
3161 set_cpu_active(cpu
, true);
3162 set_cpu_present(cpu
, true);
3163 set_cpu_possible(cpu
, true);
3166 __boot_cpu_id
= cpu
;
3171 * Must be called _AFTER_ setting up the per_cpu areas
3173 void __init
boot_cpu_hotplug_init(void)
3176 cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask
);
3177 atomic_set(this_cpu_ptr(&cpuhp_state
.ap_sync_state
), SYNC_STATE_ONLINE
);
3179 this_cpu_write(cpuhp_state
.state
, CPUHP_ONLINE
);
3180 this_cpu_write(cpuhp_state
.target
, CPUHP_ONLINE
);
3184 * These are used for a global "mitigations=" cmdline option for toggling
3185 * optional CPU mitigations.
3187 enum cpu_mitigations
{
3188 CPU_MITIGATIONS_OFF
,
3189 CPU_MITIGATIONS_AUTO
,
3190 CPU_MITIGATIONS_AUTO_NOSMT
,
3193 static enum cpu_mitigations cpu_mitigations __ro_after_init
=
3194 CPU_MITIGATIONS_AUTO
;
3196 static int __init
mitigations_parse_cmdline(char *arg
)
3198 if (!strcmp(arg
, "off"))
3199 cpu_mitigations
= CPU_MITIGATIONS_OFF
;
3200 else if (!strcmp(arg
, "auto"))
3201 cpu_mitigations
= CPU_MITIGATIONS_AUTO
;
3202 else if (!strcmp(arg
, "auto,nosmt"))
3203 cpu_mitigations
= CPU_MITIGATIONS_AUTO_NOSMT
;
3205 pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
3210 early_param("mitigations", mitigations_parse_cmdline
);
3212 /* mitigations=off */
3213 bool cpu_mitigations_off(void)
3215 return cpu_mitigations
== CPU_MITIGATIONS_OFF
;
3217 EXPORT_SYMBOL_GPL(cpu_mitigations_off
);
3219 /* mitigations=auto,nosmt */
3220 bool cpu_mitigations_auto_nosmt(void)
3222 return cpu_mitigations
== CPU_MITIGATIONS_AUTO_NOSMT
;
3224 EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt
);