1 // SPDX-License-Identifier: GPL-2.0
3 * SMP related functions
5 * Copyright IBM Corp. 1999, 2012
6 * Author(s): Denis Joseph Barrow,
7 * Martin Schwidefsky <schwidefsky@de.ibm.com>,
8 * Heiko Carstens <heiko.carstens@de.ibm.com>,
10 * based on other smp stuff by
11 * (c) 1995 Alan Cox, CymruNET Ltd <alan@cymru.net>
12 * (c) 1998 Ingo Molnar
14 * The code outside of smp.c uses logical cpu numbers, only smp.c does
15 * the translation of logical to physical cpu ids. All new code that
16 * operates on physical cpu numbers needs to go into smp.c.
19 #define KMSG_COMPONENT "cpu"
20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
22 #include <linux/workqueue.h>
23 #include <linux/memblock.h>
24 #include <linux/export.h>
25 #include <linux/init.h>
27 #include <linux/err.h>
28 #include <linux/spinlock.h>
29 #include <linux/kernel_stat.h>
30 #include <linux/delay.h>
31 #include <linux/interrupt.h>
32 #include <linux/irqflags.h>
33 #include <linux/cpu.h>
34 #include <linux/slab.h>
35 #include <linux/sched/hotplug.h>
36 #include <linux/sched/task_stack.h>
37 #include <linux/crash_dump.h>
38 #include <linux/kprobes.h>
39 #include <asm/asm-offsets.h>
41 #include <asm/switch_to.h>
42 #include <asm/facility.h>
44 #include <asm/setup.h>
46 #include <asm/tlbflush.h>
47 #include <asm/vtimer.h>
48 #include <asm/lowcore.h>
51 #include <asm/debug.h>
52 #include <asm/os_info.h>
56 #include <asm/stacktrace.h>
57 #include <asm/topology.h>
62 ec_call_function_single
,
71 static DEFINE_PER_CPU(struct cpu
*, cpu_device
);
74 struct lowcore
*lowcore
; /* lowcore page(s) for the cpu */
75 unsigned long ec_mask
; /* bit mask for ec_xxx functions */
76 unsigned long ec_clk
; /* sigp timestamp for ec_xxx */
77 signed char state
; /* physical cpu state */
78 signed char polarization
; /* physical polarization */
79 u16 address
; /* physical cpu address */
82 static u8 boot_core_type
;
83 static struct pcpu pcpu_devices
[NR_CPUS
];
85 unsigned int smp_cpu_mt_shift
;
86 EXPORT_SYMBOL(smp_cpu_mt_shift
);
88 unsigned int smp_cpu_mtid
;
89 EXPORT_SYMBOL(smp_cpu_mtid
);
91 #ifdef CONFIG_CRASH_DUMP
92 __vector128 __initdata boot_cpu_vector_save_area
[__NUM_VXRS
];
95 static unsigned int smp_max_threads __initdata
= -1U;
97 static int __init
early_nosmt(char *s
)
102 early_param("nosmt", early_nosmt
);
104 static int __init
early_smt(char *s
)
106 get_option(&s
, &smp_max_threads
);
109 early_param("smt", early_smt
);
112 * The smp_cpu_state_mutex must be held when changing the state or polarization
113 * member of a pcpu data structure within the pcpu_devices arreay.
115 DEFINE_MUTEX(smp_cpu_state_mutex
);
118 * Signal processor helper functions.
120 static inline int __pcpu_sigp_relax(u16 addr
, u8 order
, unsigned long parm
)
125 cc
= __pcpu_sigp(addr
, order
, parm
, NULL
);
126 if (cc
!= SIGP_CC_BUSY
)
132 static int pcpu_sigp_retry(struct pcpu
*pcpu
, u8 order
, u32 parm
)
136 for (retry
= 0; ; retry
++) {
137 cc
= __pcpu_sigp(pcpu
->address
, order
, parm
, NULL
);
138 if (cc
!= SIGP_CC_BUSY
)
146 static inline int pcpu_stopped(struct pcpu
*pcpu
)
148 u32
uninitialized_var(status
);
150 if (__pcpu_sigp(pcpu
->address
, SIGP_SENSE
,
151 0, &status
) != SIGP_CC_STATUS_STORED
)
153 return !!(status
& (SIGP_STATUS_CHECK_STOP
|SIGP_STATUS_STOPPED
));
156 static inline int pcpu_running(struct pcpu
*pcpu
)
158 if (__pcpu_sigp(pcpu
->address
, SIGP_SENSE_RUNNING
,
159 0, NULL
) != SIGP_CC_STATUS_STORED
)
161 /* Status stored condition code is equivalent to cpu not running. */
166 * Find struct pcpu by cpu address.
168 static struct pcpu
*pcpu_find_address(const struct cpumask
*mask
, u16 address
)
172 for_each_cpu(cpu
, mask
)
173 if (pcpu_devices
[cpu
].address
== address
)
174 return pcpu_devices
+ cpu
;
178 static void pcpu_ec_call(struct pcpu
*pcpu
, int ec_bit
)
182 if (test_and_set_bit(ec_bit
, &pcpu
->ec_mask
))
184 order
= pcpu_running(pcpu
) ? SIGP_EXTERNAL_CALL
: SIGP_EMERGENCY_SIGNAL
;
185 pcpu
->ec_clk
= get_tod_clock_fast();
186 pcpu_sigp_retry(pcpu
, order
, 0);
189 static int pcpu_alloc_lowcore(struct pcpu
*pcpu
, int cpu
)
191 unsigned long async_stack
, nodat_stack
;
194 if (pcpu
!= &pcpu_devices
[0]) {
195 pcpu
->lowcore
= (struct lowcore
*)
196 __get_free_pages(GFP_KERNEL
| GFP_DMA
, LC_ORDER
);
197 nodat_stack
= __get_free_pages(GFP_KERNEL
, THREAD_SIZE_ORDER
);
198 if (!pcpu
->lowcore
|| !nodat_stack
)
201 nodat_stack
= pcpu
->lowcore
->nodat_stack
- STACK_INIT_OFFSET
;
203 async_stack
= stack_alloc();
207 memcpy(lc
, &S390_lowcore
, 512);
208 memset((char *) lc
+ 512, 0, sizeof(*lc
) - 512);
209 lc
->async_stack
= async_stack
+ STACK_INIT_OFFSET
;
210 lc
->nodat_stack
= nodat_stack
+ STACK_INIT_OFFSET
;
212 lc
->spinlock_lockval
= arch_spin_lockval(cpu
);
213 lc
->spinlock_index
= 0;
214 lc
->br_r1_trampoline
= 0x07f1; /* br %r1 */
215 lc
->return_lpswe
= gen_lpswe(__LC_RETURN_PSW
);
216 lc
->return_mcck_lpswe
= gen_lpswe(__LC_RETURN_MCCK_PSW
);
217 if (nmi_alloc_per_cpu(lc
))
219 if (vdso_alloc_per_cpu(lc
))
221 lowcore_ptr
[cpu
] = lc
;
222 pcpu_sigp_retry(pcpu
, SIGP_SET_PREFIX
, (u32
)(unsigned long) lc
);
226 nmi_free_per_cpu(lc
);
228 stack_free(async_stack
);
230 if (pcpu
!= &pcpu_devices
[0]) {
231 free_pages(nodat_stack
, THREAD_SIZE_ORDER
);
232 free_pages((unsigned long) pcpu
->lowcore
, LC_ORDER
);
237 static void pcpu_free_lowcore(struct pcpu
*pcpu
)
239 unsigned long async_stack
, nodat_stack
, lowcore
;
241 nodat_stack
= pcpu
->lowcore
->nodat_stack
- STACK_INIT_OFFSET
;
242 async_stack
= pcpu
->lowcore
->async_stack
- STACK_INIT_OFFSET
;
243 lowcore
= (unsigned long) pcpu
->lowcore
;
245 pcpu_sigp_retry(pcpu
, SIGP_SET_PREFIX
, 0);
246 lowcore_ptr
[pcpu
- pcpu_devices
] = NULL
;
247 vdso_free_per_cpu(pcpu
->lowcore
);
248 nmi_free_per_cpu(pcpu
->lowcore
);
249 stack_free(async_stack
);
250 if (pcpu
== &pcpu_devices
[0])
252 free_pages(nodat_stack
, THREAD_SIZE_ORDER
);
253 free_pages(lowcore
, LC_ORDER
);
256 static void pcpu_prepare_secondary(struct pcpu
*pcpu
, int cpu
)
258 struct lowcore
*lc
= pcpu
->lowcore
;
260 cpumask_set_cpu(cpu
, &init_mm
.context
.cpu_attach_mask
);
261 cpumask_set_cpu(cpu
, mm_cpumask(&init_mm
));
263 lc
->spinlock_lockval
= arch_spin_lockval(cpu
);
264 lc
->spinlock_index
= 0;
265 lc
->percpu_offset
= __per_cpu_offset
[cpu
];
266 lc
->kernel_asce
= S390_lowcore
.kernel_asce
;
267 lc
->user_asce
= S390_lowcore
.kernel_asce
;
268 lc
->machine_flags
= S390_lowcore
.machine_flags
;
269 lc
->user_timer
= lc
->system_timer
=
270 lc
->steal_timer
= lc
->avg_steal_timer
= 0;
271 __ctl_store(lc
->cregs_save_area
, 0, 15);
272 lc
->cregs_save_area
[1] = lc
->kernel_asce
;
273 lc
->cregs_save_area
[7] = lc
->vdso_asce
;
274 save_access_regs((unsigned int *) lc
->access_regs_save_area
);
275 memcpy(lc
->stfle_fac_list
, S390_lowcore
.stfle_fac_list
,
276 sizeof(lc
->stfle_fac_list
));
277 memcpy(lc
->alt_stfle_fac_list
, S390_lowcore
.alt_stfle_fac_list
,
278 sizeof(lc
->alt_stfle_fac_list
));
279 arch_spin_lock_setup(cpu
);
282 static void pcpu_attach_task(struct pcpu
*pcpu
, struct task_struct
*tsk
)
284 struct lowcore
*lc
= pcpu
->lowcore
;
286 lc
->kernel_stack
= (unsigned long) task_stack_page(tsk
)
287 + THREAD_SIZE
- STACK_FRAME_OVERHEAD
- sizeof(struct pt_regs
);
288 lc
->current_task
= (unsigned long) tsk
;
290 lc
->current_pid
= tsk
->pid
;
291 lc
->user_timer
= tsk
->thread
.user_timer
;
292 lc
->guest_timer
= tsk
->thread
.guest_timer
;
293 lc
->system_timer
= tsk
->thread
.system_timer
;
294 lc
->hardirq_timer
= tsk
->thread
.hardirq_timer
;
295 lc
->softirq_timer
= tsk
->thread
.softirq_timer
;
299 static void pcpu_start_fn(struct pcpu
*pcpu
, void (*func
)(void *), void *data
)
301 struct lowcore
*lc
= pcpu
->lowcore
;
303 lc
->restart_stack
= lc
->nodat_stack
;
304 lc
->restart_fn
= (unsigned long) func
;
305 lc
->restart_data
= (unsigned long) data
;
306 lc
->restart_source
= -1UL;
307 pcpu_sigp_retry(pcpu
, SIGP_RESTART
, 0);
311 * Call function via PSW restart on pcpu and stop the current cpu.
313 static void __pcpu_delegate(void (*func
)(void*), void *data
)
315 func(data
); /* should not return */
318 static void __no_sanitize_address
pcpu_delegate(struct pcpu
*pcpu
,
319 void (*func
)(void *),
320 void *data
, unsigned long stack
)
322 struct lowcore
*lc
= lowcore_ptr
[pcpu
- pcpu_devices
];
323 unsigned long source_cpu
= stap();
325 __load_psw_mask(PSW_KERNEL_BITS
| PSW_MASK_DAT
);
326 if (pcpu
->address
== source_cpu
)
327 CALL_ON_STACK(__pcpu_delegate
, stack
, 2, func
, data
);
328 /* Stop target cpu (if func returns this stops the current cpu). */
329 pcpu_sigp_retry(pcpu
, SIGP_STOP
, 0);
330 /* Restart func on the target cpu and stop the current cpu. */
331 mem_assign_absolute(lc
->restart_stack
, stack
);
332 mem_assign_absolute(lc
->restart_fn
, (unsigned long) func
);
333 mem_assign_absolute(lc
->restart_data
, (unsigned long) data
);
334 mem_assign_absolute(lc
->restart_source
, source_cpu
);
337 "0: sigp 0,%0,%2 # sigp restart to target cpu\n"
338 " brc 2,0b # busy, try again\n"
339 "1: sigp 0,%1,%3 # sigp stop to current cpu\n"
340 " brc 2,1b # busy, try again\n"
341 : : "d" (pcpu
->address
), "d" (source_cpu
),
342 "K" (SIGP_RESTART
), "K" (SIGP_STOP
)
348 * Enable additional logical cpus for multi-threading.
350 static int pcpu_set_smt(unsigned int mtid
)
354 if (smp_cpu_mtid
== mtid
)
356 cc
= __pcpu_sigp(0, SIGP_SET_MULTI_THREADING
, mtid
, NULL
);
359 smp_cpu_mt_shift
= 0;
360 while (smp_cpu_mtid
>= (1U << smp_cpu_mt_shift
))
362 pcpu_devices
[0].address
= stap();
368 * Call function on an online CPU.
370 void smp_call_online_cpu(void (*func
)(void *), void *data
)
374 /* Use the current cpu if it is online. */
375 pcpu
= pcpu_find_address(cpu_online_mask
, stap());
377 /* Use the first online cpu. */
378 pcpu
= pcpu_devices
+ cpumask_first(cpu_online_mask
);
379 pcpu_delegate(pcpu
, func
, data
, (unsigned long) restart_stack
);
383 * Call function on the ipl CPU.
385 void smp_call_ipl_cpu(void (*func
)(void *), void *data
)
387 struct lowcore
*lc
= pcpu_devices
->lowcore
;
389 if (pcpu_devices
[0].address
== stap())
392 pcpu_delegate(&pcpu_devices
[0], func
, data
,
396 int smp_find_processor_id(u16 address
)
400 for_each_present_cpu(cpu
)
401 if (pcpu_devices
[cpu
].address
== address
)
406 bool arch_vcpu_is_preempted(int cpu
)
408 if (test_cpu_flag_of(CIF_ENABLED_WAIT
, cpu
))
410 if (pcpu_running(pcpu_devices
+ cpu
))
414 EXPORT_SYMBOL(arch_vcpu_is_preempted
);
416 void smp_yield_cpu(int cpu
)
418 if (!MACHINE_HAS_DIAG9C
)
420 diag_stat_inc_norecursion(DIAG_STAT_X09C
);
421 asm volatile("diag %0,0,0x9c"
422 : : "d" (pcpu_devices
[cpu
].address
));
426 * Send cpus emergency shutdown signal. This gives the cpus the
427 * opportunity to complete outstanding interrupts.
429 void notrace
smp_emergency_stop(void)
435 cpumask_copy(&cpumask
, cpu_online_mask
);
436 cpumask_clear_cpu(smp_processor_id(), &cpumask
);
438 end
= get_tod_clock() + (1000000UL << 12);
439 for_each_cpu(cpu
, &cpumask
) {
440 struct pcpu
*pcpu
= pcpu_devices
+ cpu
;
441 set_bit(ec_stop_cpu
, &pcpu
->ec_mask
);
442 while (__pcpu_sigp(pcpu
->address
, SIGP_EMERGENCY_SIGNAL
,
443 0, NULL
) == SIGP_CC_BUSY
&&
444 get_tod_clock() < end
)
447 while (get_tod_clock() < end
) {
448 for_each_cpu(cpu
, &cpumask
)
449 if (pcpu_stopped(pcpu_devices
+ cpu
))
450 cpumask_clear_cpu(cpu
, &cpumask
);
451 if (cpumask_empty(&cpumask
))
456 NOKPROBE_SYMBOL(smp_emergency_stop
);
459 * Stop all cpus but the current one.
461 void smp_send_stop(void)
465 /* Disable all interrupts/machine checks */
466 __load_psw_mask(PSW_KERNEL_BITS
| PSW_MASK_DAT
);
467 trace_hardirqs_off();
469 debug_set_critical();
471 if (oops_in_progress
)
472 smp_emergency_stop();
474 /* stop all processors */
475 for_each_online_cpu(cpu
) {
476 if (cpu
== smp_processor_id())
478 pcpu_sigp_retry(pcpu_devices
+ cpu
, SIGP_STOP
, 0);
479 while (!pcpu_stopped(pcpu_devices
+ cpu
))
485 * This is the main routine where commands issued by other
488 static void smp_handle_ext_call(void)
492 /* handle bit signal external calls */
493 bits
= xchg(&pcpu_devices
[smp_processor_id()].ec_mask
, 0);
494 if (test_bit(ec_stop_cpu
, &bits
))
496 if (test_bit(ec_schedule
, &bits
))
498 if (test_bit(ec_call_function_single
, &bits
))
499 generic_smp_call_function_single_interrupt();
502 static void do_ext_call_interrupt(struct ext_code ext_code
,
503 unsigned int param32
, unsigned long param64
)
505 inc_irq_stat(ext_code
.code
== 0x1202 ? IRQEXT_EXC
: IRQEXT_EMS
);
506 smp_handle_ext_call();
509 void arch_send_call_function_ipi_mask(const struct cpumask
*mask
)
513 for_each_cpu(cpu
, mask
)
514 pcpu_ec_call(pcpu_devices
+ cpu
, ec_call_function_single
);
517 void arch_send_call_function_single_ipi(int cpu
)
519 pcpu_ec_call(pcpu_devices
+ cpu
, ec_call_function_single
);
523 * this function sends a 'reschedule' IPI to another CPU.
524 * it goes straight through and wastes no time serializing
525 * anything. Worst case is that we lose a reschedule ...
527 void smp_send_reschedule(int cpu
)
529 pcpu_ec_call(pcpu_devices
+ cpu
, ec_schedule
);
533 * parameter area for the set/clear control bit callbacks
535 struct ec_creg_mask_parms
{
537 unsigned long andval
;
542 * callback for setting/clearing control bits
544 static void smp_ctl_bit_callback(void *info
)
546 struct ec_creg_mask_parms
*pp
= info
;
547 unsigned long cregs
[16];
549 __ctl_store(cregs
, 0, 15);
550 cregs
[pp
->cr
] = (cregs
[pp
->cr
] & pp
->andval
) | pp
->orval
;
551 __ctl_load(cregs
, 0, 15);
555 * Set a bit in a control register of all cpus
557 void smp_ctl_set_bit(int cr
, int bit
)
559 struct ec_creg_mask_parms parms
= { 1UL << bit
, -1UL, cr
};
561 on_each_cpu(smp_ctl_bit_callback
, &parms
, 1);
563 EXPORT_SYMBOL(smp_ctl_set_bit
);
566 * Clear a bit in a control register of all cpus
568 void smp_ctl_clear_bit(int cr
, int bit
)
570 struct ec_creg_mask_parms parms
= { 0, ~(1UL << bit
), cr
};
572 on_each_cpu(smp_ctl_bit_callback
, &parms
, 1);
574 EXPORT_SYMBOL(smp_ctl_clear_bit
);
576 #ifdef CONFIG_CRASH_DUMP
578 int smp_store_status(int cpu
)
580 struct pcpu
*pcpu
= pcpu_devices
+ cpu
;
583 pa
= __pa(&pcpu
->lowcore
->floating_pt_save_area
);
584 if (__pcpu_sigp_relax(pcpu
->address
, SIGP_STORE_STATUS_AT_ADDRESS
,
585 pa
) != SIGP_CC_ORDER_CODE_ACCEPTED
)
587 if (!MACHINE_HAS_VX
&& !MACHINE_HAS_GS
)
589 pa
= __pa(pcpu
->lowcore
->mcesad
& MCESA_ORIGIN_MASK
);
591 pa
|= pcpu
->lowcore
->mcesad
& MCESA_LC_MASK
;
592 if (__pcpu_sigp_relax(pcpu
->address
, SIGP_STORE_ADDITIONAL_STATUS
,
593 pa
) != SIGP_CC_ORDER_CODE_ACCEPTED
)
599 * Collect CPU state of the previous, crashed system.
600 * There are four cases:
601 * 1) standard zfcp dump
602 * condition: OLDMEM_BASE == NULL && ipl_info.type == IPL_TYPE_FCP_DUMP
603 * The state for all CPUs except the boot CPU needs to be collected
604 * with sigp stop-and-store-status. The boot CPU state is located in
605 * the absolute lowcore of the memory stored in the HSA. The zcore code
606 * will copy the boot CPU state from the HSA.
607 * 2) stand-alone kdump for SCSI (zfcp dump with swapped memory)
608 * condition: OLDMEM_BASE != NULL && ipl_info.type == IPL_TYPE_FCP_DUMP
609 * The state for all CPUs except the boot CPU needs to be collected
610 * with sigp stop-and-store-status. The firmware or the boot-loader
611 * stored the registers of the boot CPU in the absolute lowcore in the
612 * memory of the old system.
613 * 3) kdump and the old kernel did not store the CPU state,
614 * or stand-alone kdump for DASD
615 * condition: OLDMEM_BASE != NULL && !is_kdump_kernel()
616 * The state for all CPUs except the boot CPU needs to be collected
617 * with sigp stop-and-store-status. The kexec code or the boot-loader
618 * stored the registers of the boot CPU in the memory of the old system.
619 * 4) kdump and the old kernel stored the CPU state
620 * condition: OLDMEM_BASE != NULL && is_kdump_kernel()
621 * This case does not exist for s390 anymore, setup_arch explicitly
622 * deactivates the elfcorehdr= kernel parameter
624 static __init
void smp_save_cpu_vxrs(struct save_area
*sa
, u16 addr
,
625 bool is_boot_cpu
, unsigned long page
)
627 __vector128
*vxrs
= (__vector128
*) page
;
630 vxrs
= boot_cpu_vector_save_area
;
632 __pcpu_sigp_relax(addr
, SIGP_STORE_ADDITIONAL_STATUS
, page
);
633 save_area_add_vxrs(sa
, vxrs
);
636 static __init
void smp_save_cpu_regs(struct save_area
*sa
, u16 addr
,
637 bool is_boot_cpu
, unsigned long page
)
639 void *regs
= (void *) page
;
642 copy_oldmem_kernel(regs
, (void *) __LC_FPREGS_SAVE_AREA
, 512);
644 __pcpu_sigp_relax(addr
, SIGP_STORE_STATUS_AT_ADDRESS
, page
);
645 save_area_add_regs(sa
, regs
);
648 void __init
smp_save_dump_cpus(void)
650 int addr
, boot_cpu_addr
, max_cpu_addr
;
651 struct save_area
*sa
;
655 if (!(OLDMEM_BASE
|| ipl_info
.type
== IPL_TYPE_FCP_DUMP
))
656 /* No previous system present, normal boot. */
658 /* Allocate a page as dumping area for the store status sigps */
659 page
= memblock_phys_alloc_range(PAGE_SIZE
, PAGE_SIZE
, 0, 1UL << 31);
661 panic("ERROR: Failed to allocate %lx bytes below %lx\n",
662 PAGE_SIZE
, 1UL << 31);
664 /* Set multi-threading state to the previous system. */
665 pcpu_set_smt(sclp
.mtid_prev
);
666 boot_cpu_addr
= stap();
667 max_cpu_addr
= SCLP_MAX_CORES
<< sclp
.mtid_prev
;
668 for (addr
= 0; addr
<= max_cpu_addr
; addr
++) {
669 if (__pcpu_sigp_relax(addr
, SIGP_SENSE
, 0) ==
670 SIGP_CC_NOT_OPERATIONAL
)
672 is_boot_cpu
= (addr
== boot_cpu_addr
);
673 /* Allocate save area */
674 sa
= save_area_alloc(is_boot_cpu
);
676 panic("could not allocate memory for save area\n");
678 /* Get the vector registers */
679 smp_save_cpu_vxrs(sa
, addr
, is_boot_cpu
, page
);
681 * For a zfcp dump OLDMEM_BASE == NULL and the registers
682 * of the boot CPU are stored in the HSA. To retrieve
683 * these registers an SCLP request is required which is
684 * done by drivers/s390/char/zcore.c:init_cpu_info()
686 if (!is_boot_cpu
|| OLDMEM_BASE
)
687 /* Get the CPU registers */
688 smp_save_cpu_regs(sa
, addr
, is_boot_cpu
, page
);
690 memblock_free(page
, PAGE_SIZE
);
691 diag_dma_ops
.diag308_reset();
694 #endif /* CONFIG_CRASH_DUMP */
696 void smp_cpu_set_polarization(int cpu
, int val
)
698 pcpu_devices
[cpu
].polarization
= val
;
701 int smp_cpu_get_polarization(int cpu
)
703 return pcpu_devices
[cpu
].polarization
;
706 int smp_cpu_get_cpu_address(int cpu
)
708 return pcpu_devices
[cpu
].address
;
711 static void __ref
smp_get_core_info(struct sclp_core_info
*info
, int early
)
713 static int use_sigp_detection
;
716 if (use_sigp_detection
|| sclp_get_core_info(info
, early
)) {
717 use_sigp_detection
= 1;
719 address
< (SCLP_MAX_CORES
<< smp_cpu_mt_shift
);
720 address
+= (1U << smp_cpu_mt_shift
)) {
721 if (__pcpu_sigp_relax(address
, SIGP_SENSE
, 0) ==
722 SIGP_CC_NOT_OPERATIONAL
)
724 info
->core
[info
->configured
].core_id
=
725 address
>> smp_cpu_mt_shift
;
728 info
->combined
= info
->configured
;
732 static int smp_add_present_cpu(int cpu
);
734 static int smp_add_core(struct sclp_core_entry
*core
, cpumask_t
*avail
,
735 bool configured
, bool early
)
742 if (sclp
.has_core_type
&& core
->type
!= boot_core_type
)
744 cpu
= cpumask_first(avail
);
745 address
= core
->core_id
<< smp_cpu_mt_shift
;
746 for (i
= 0; (i
<= smp_cpu_mtid
) && (cpu
< nr_cpu_ids
); i
++) {
747 if (pcpu_find_address(cpu_present_mask
, address
+ i
))
749 pcpu
= pcpu_devices
+ cpu
;
750 pcpu
->address
= address
+ i
;
752 pcpu
->state
= CPU_STATE_CONFIGURED
;
754 pcpu
->state
= CPU_STATE_STANDBY
;
755 smp_cpu_set_polarization(cpu
, POLARIZATION_UNKNOWN
);
756 set_cpu_present(cpu
, true);
757 if (!early
&& smp_add_present_cpu(cpu
) != 0)
758 set_cpu_present(cpu
, false);
761 cpumask_clear_cpu(cpu
, avail
);
762 cpu
= cpumask_next(cpu
, avail
);
767 static int __smp_rescan_cpus(struct sclp_core_info
*info
, bool early
)
769 struct sclp_core_entry
*core
;
776 cpumask_xor(&avail
, cpu_possible_mask
, cpu_present_mask
);
778 * Add IPL core first (which got logical CPU number 0) to make sure
779 * that all SMT threads get subsequent logical CPU numbers.
782 core_id
= pcpu_devices
[0].address
>> smp_cpu_mt_shift
;
783 for (i
= 0; i
< info
->configured
; i
++) {
784 core
= &info
->core
[i
];
785 if (core
->core_id
== core_id
) {
786 nr
+= smp_add_core(core
, &avail
, true, early
);
791 for (i
= 0; i
< info
->combined
; i
++) {
792 configured
= i
< info
->configured
;
793 nr
+= smp_add_core(&info
->core
[i
], &avail
, configured
, early
);
798 void __init
smp_detect_cpus(void)
800 unsigned int cpu
, mtid
, c_cpus
, s_cpus
;
801 struct sclp_core_info
*info
;
804 /* Get CPU information */
805 info
= memblock_alloc(sizeof(*info
), 8);
807 panic("%s: Failed to allocate %zu bytes align=0x%x\n",
808 __func__
, sizeof(*info
), 8);
809 smp_get_core_info(info
, 1);
810 /* Find boot CPU type */
811 if (sclp
.has_core_type
) {
813 for (cpu
= 0; cpu
< info
->combined
; cpu
++)
814 if (info
->core
[cpu
].core_id
== address
) {
815 /* The boot cpu dictates the cpu type. */
816 boot_core_type
= info
->core
[cpu
].type
;
819 if (cpu
>= info
->combined
)
820 panic("Could not find boot CPU type");
823 /* Set multi-threading state for the current system */
824 mtid
= boot_core_type
? sclp
.mtid
: sclp
.mtid_cp
;
825 mtid
= (mtid
< smp_max_threads
) ? mtid
: smp_max_threads
- 1;
828 /* Print number of CPUs */
830 for (cpu
= 0; cpu
< info
->combined
; cpu
++) {
831 if (sclp
.has_core_type
&&
832 info
->core
[cpu
].type
!= boot_core_type
)
834 if (cpu
< info
->configured
)
835 c_cpus
+= smp_cpu_mtid
+ 1;
837 s_cpus
+= smp_cpu_mtid
+ 1;
839 pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus
, s_cpus
);
841 /* Add CPUs present at boot */
843 __smp_rescan_cpus(info
, true);
845 memblock_free_early((unsigned long)info
, sizeof(*info
));
848 static void smp_init_secondary(void)
850 int cpu
= smp_processor_id();
852 S390_lowcore
.last_update_clock
= get_tod_clock();
853 restore_access_regs(S390_lowcore
.access_regs_save_area
);
854 set_cpu_flag(CIF_ASCE_PRIMARY
);
855 set_cpu_flag(CIF_ASCE_SECONDARY
);
861 notify_cpu_starting(cpu
);
862 if (topology_cpu_dedicated(cpu
))
863 set_cpu_flag(CIF_DEDICATED_CPU
);
865 clear_cpu_flag(CIF_DEDICATED_CPU
);
866 set_cpu_online(cpu
, true);
868 inc_irq_stat(CPU_RST
);
870 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE
);
874 * Activate a secondary processor.
876 static void __no_sanitize_address
smp_start_secondary(void *cpuvoid
)
878 S390_lowcore
.restart_stack
= (unsigned long) restart_stack
;
879 S390_lowcore
.restart_fn
= (unsigned long) do_restart
;
880 S390_lowcore
.restart_data
= 0;
881 S390_lowcore
.restart_source
= -1UL;
882 __ctl_load(S390_lowcore
.cregs_save_area
, 0, 15);
883 __load_psw_mask(PSW_KERNEL_BITS
| PSW_MASK_DAT
);
884 CALL_ON_STACK_NORETURN(smp_init_secondary
, S390_lowcore
.kernel_stack
);
887 /* Upping and downing of CPUs */
888 int __cpu_up(unsigned int cpu
, struct task_struct
*tidle
)
893 pcpu
= pcpu_devices
+ cpu
;
894 if (pcpu
->state
!= CPU_STATE_CONFIGURED
)
896 base
= smp_get_base_cpu(cpu
);
897 for (i
= 0; i
<= smp_cpu_mtid
; i
++) {
898 if (base
+ i
< nr_cpu_ids
)
899 if (cpu_online(base
+ i
))
903 * If this is the first CPU of the core to get online
904 * do an initial CPU reset.
906 if (i
> smp_cpu_mtid
&&
907 pcpu_sigp_retry(pcpu_devices
+ base
, SIGP_INITIAL_CPU_RESET
, 0) !=
908 SIGP_CC_ORDER_CODE_ACCEPTED
)
911 rc
= pcpu_alloc_lowcore(pcpu
, cpu
);
914 pcpu_prepare_secondary(pcpu
, cpu
);
915 pcpu_attach_task(pcpu
, tidle
);
916 pcpu_start_fn(pcpu
, smp_start_secondary
, NULL
);
917 /* Wait until cpu puts itself in the online & active maps */
918 while (!cpu_online(cpu
))
923 static unsigned int setup_possible_cpus __initdata
;
925 static int __init
_setup_possible_cpus(char *s
)
927 get_option(&s
, &setup_possible_cpus
);
930 early_param("possible_cpus", _setup_possible_cpus
);
932 int __cpu_disable(void)
934 unsigned long cregs
[16];
936 /* Handle possible pending IPIs */
937 smp_handle_ext_call();
938 set_cpu_online(smp_processor_id(), false);
940 /* Disable pseudo page faults on this cpu. */
942 /* Disable interrupt sources via control register. */
943 __ctl_store(cregs
, 0, 15);
944 cregs
[0] &= ~0x0000ee70UL
; /* disable all external interrupts */
945 cregs
[6] &= ~0xff000000UL
; /* disable all I/O interrupts */
946 cregs
[14] &= ~0x1f000000UL
; /* disable most machine checks */
947 __ctl_load(cregs
, 0, 15);
948 clear_cpu_flag(CIF_NOHZ_DELAY
);
952 void __cpu_die(unsigned int cpu
)
956 /* Wait until target cpu is down */
957 pcpu
= pcpu_devices
+ cpu
;
958 while (!pcpu_stopped(pcpu
))
960 pcpu_free_lowcore(pcpu
);
961 cpumask_clear_cpu(cpu
, mm_cpumask(&init_mm
));
962 cpumask_clear_cpu(cpu
, &init_mm
.context
.cpu_attach_mask
);
965 void __noreturn
cpu_die(void)
969 pcpu_sigp_retry(pcpu_devices
+ smp_processor_id(), SIGP_STOP
, 0);
973 void __init
smp_fill_possible_mask(void)
975 unsigned int possible
, sclp_max
, cpu
;
977 sclp_max
= max(sclp
.mtid
, sclp
.mtid_cp
) + 1;
978 sclp_max
= min(smp_max_threads
, sclp_max
);
979 sclp_max
= (sclp
.max_cores
* sclp_max
) ?: nr_cpu_ids
;
980 possible
= setup_possible_cpus
?: nr_cpu_ids
;
981 possible
= min(possible
, sclp_max
);
982 for (cpu
= 0; cpu
< possible
&& cpu
< nr_cpu_ids
; cpu
++)
983 set_cpu_possible(cpu
, true);
986 void __init
smp_prepare_cpus(unsigned int max_cpus
)
988 /* request the 0x1201 emergency signal external interrupt */
989 if (register_external_irq(EXT_IRQ_EMERGENCY_SIG
, do_ext_call_interrupt
))
990 panic("Couldn't request external interrupt 0x1201");
991 /* request the 0x1202 external call external interrupt */
992 if (register_external_irq(EXT_IRQ_EXTERNAL_CALL
, do_ext_call_interrupt
))
993 panic("Couldn't request external interrupt 0x1202");
996 void __init
smp_prepare_boot_cpu(void)
998 struct pcpu
*pcpu
= pcpu_devices
;
1000 WARN_ON(!cpu_present(0) || !cpu_online(0));
1001 pcpu
->state
= CPU_STATE_CONFIGURED
;
1002 pcpu
->lowcore
= (struct lowcore
*)(unsigned long) store_prefix();
1003 S390_lowcore
.percpu_offset
= __per_cpu_offset
[0];
1004 smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN
);
1007 void __init
smp_cpus_done(unsigned int max_cpus
)
1011 void __init
smp_setup_processor_id(void)
1013 pcpu_devices
[0].address
= stap();
1014 S390_lowcore
.cpu_nr
= 0;
1015 S390_lowcore
.spinlock_lockval
= arch_spin_lockval(0);
1016 S390_lowcore
.spinlock_index
= 0;
1020 * the frequency of the profiling timer can be changed
1021 * by writing a multiplier value into /proc/profile.
1023 * usually you want to run this on all CPUs ;)
1025 int setup_profiling_timer(unsigned int multiplier
)
1030 static ssize_t
cpu_configure_show(struct device
*dev
,
1031 struct device_attribute
*attr
, char *buf
)
1035 mutex_lock(&smp_cpu_state_mutex
);
1036 count
= sprintf(buf
, "%d\n", pcpu_devices
[dev
->id
].state
);
1037 mutex_unlock(&smp_cpu_state_mutex
);
1041 static ssize_t
cpu_configure_store(struct device
*dev
,
1042 struct device_attribute
*attr
,
1043 const char *buf
, size_t count
)
1046 int cpu
, val
, rc
, i
;
1049 if (sscanf(buf
, "%d %c", &val
, &delim
) != 1)
1051 if (val
!= 0 && val
!= 1)
1054 mutex_lock(&smp_cpu_state_mutex
);
1056 /* disallow configuration changes of online cpus and cpu 0 */
1058 cpu
= smp_get_base_cpu(cpu
);
1061 for (i
= 0; i
<= smp_cpu_mtid
; i
++)
1062 if (cpu_online(cpu
+ i
))
1064 pcpu
= pcpu_devices
+ cpu
;
1068 if (pcpu
->state
!= CPU_STATE_CONFIGURED
)
1070 rc
= sclp_core_deconfigure(pcpu
->address
>> smp_cpu_mt_shift
);
1073 for (i
= 0; i
<= smp_cpu_mtid
; i
++) {
1074 if (cpu
+ i
>= nr_cpu_ids
|| !cpu_present(cpu
+ i
))
1076 pcpu
[i
].state
= CPU_STATE_STANDBY
;
1077 smp_cpu_set_polarization(cpu
+ i
,
1078 POLARIZATION_UNKNOWN
);
1080 topology_expect_change();
1083 if (pcpu
->state
!= CPU_STATE_STANDBY
)
1085 rc
= sclp_core_configure(pcpu
->address
>> smp_cpu_mt_shift
);
1088 for (i
= 0; i
<= smp_cpu_mtid
; i
++) {
1089 if (cpu
+ i
>= nr_cpu_ids
|| !cpu_present(cpu
+ i
))
1091 pcpu
[i
].state
= CPU_STATE_CONFIGURED
;
1092 smp_cpu_set_polarization(cpu
+ i
,
1093 POLARIZATION_UNKNOWN
);
1095 topology_expect_change();
1101 mutex_unlock(&smp_cpu_state_mutex
);
1103 return rc
? rc
: count
;
1105 static DEVICE_ATTR(configure
, 0644, cpu_configure_show
, cpu_configure_store
);
1107 static ssize_t
show_cpu_address(struct device
*dev
,
1108 struct device_attribute
*attr
, char *buf
)
1110 return sprintf(buf
, "%d\n", pcpu_devices
[dev
->id
].address
);
1112 static DEVICE_ATTR(address
, 0444, show_cpu_address
, NULL
);
1114 static struct attribute
*cpu_common_attrs
[] = {
1115 &dev_attr_configure
.attr
,
1116 &dev_attr_address
.attr
,
1120 static struct attribute_group cpu_common_attr_group
= {
1121 .attrs
= cpu_common_attrs
,
1124 static struct attribute
*cpu_online_attrs
[] = {
1125 &dev_attr_idle_count
.attr
,
1126 &dev_attr_idle_time_us
.attr
,
1130 static struct attribute_group cpu_online_attr_group
= {
1131 .attrs
= cpu_online_attrs
,
1134 static int smp_cpu_online(unsigned int cpu
)
1136 struct device
*s
= &per_cpu(cpu_device
, cpu
)->dev
;
1138 return sysfs_create_group(&s
->kobj
, &cpu_online_attr_group
);
1140 static int smp_cpu_pre_down(unsigned int cpu
)
1142 struct device
*s
= &per_cpu(cpu_device
, cpu
)->dev
;
1144 sysfs_remove_group(&s
->kobj
, &cpu_online_attr_group
);
1148 static int smp_add_present_cpu(int cpu
)
1154 c
= kzalloc(sizeof(*c
), GFP_KERNEL
);
1157 per_cpu(cpu_device
, cpu
) = c
;
1159 c
->hotpluggable
= 1;
1160 rc
= register_cpu(c
, cpu
);
1163 rc
= sysfs_create_group(&s
->kobj
, &cpu_common_attr_group
);
1166 rc
= topology_cpu_init(c
);
1172 sysfs_remove_group(&s
->kobj
, &cpu_common_attr_group
);
1179 int __ref
smp_rescan_cpus(void)
1181 struct sclp_core_info
*info
;
1184 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
1187 smp_get_core_info(info
, 0);
1189 mutex_lock(&smp_cpu_state_mutex
);
1190 nr
= __smp_rescan_cpus(info
, false);
1191 mutex_unlock(&smp_cpu_state_mutex
);
1195 topology_schedule_update();
1199 static ssize_t __ref
rescan_store(struct device
*dev
,
1200 struct device_attribute
*attr
,
1206 rc
= lock_device_hotplug_sysfs();
1209 rc
= smp_rescan_cpus();
1210 unlock_device_hotplug();
1211 return rc
? rc
: count
;
1213 static DEVICE_ATTR_WO(rescan
);
1215 static int __init
s390_smp_init(void)
1219 rc
= device_create_file(cpu_subsys
.dev_root
, &dev_attr_rescan
);
1222 for_each_present_cpu(cpu
) {
1223 rc
= smp_add_present_cpu(cpu
);
1228 rc
= cpuhp_setup_state(CPUHP_AP_ONLINE_DYN
, "s390/smp:online",
1229 smp_cpu_online
, smp_cpu_pre_down
);
1230 rc
= rc
<= 0 ? rc
: 0;
1234 subsys_initcall(s390_smp_init
);