]>
Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
76d2a049 PD |
2 | /* |
3 | * SMP initialisation and IPI support | |
4 | * Based on arch/arm64/kernel/smp.c | |
5 | * | |
6 | * Copyright (C) 2012 ARM Ltd. | |
7 | * Copyright (C) 2015 Regents of the University of California | |
8 | * Copyright (C) 2017 SiFive | |
76d2a049 PD |
9 | */ |
10 | ||
5ed881bc | 11 | #include <linux/cpu.h> |
76d2a049 | 12 | #include <linux/interrupt.h> |
7391efa4 | 13 | #include <linux/module.h> |
5ed881bc | 14 | #include <linux/profile.h> |
76d2a049 PD |
15 | #include <linux/smp.h> |
16 | #include <linux/sched.h> | |
8b20d2db | 17 | #include <linux/seq_file.h> |
37a107ff | 18 | #include <linux/delay.h> |
76d2a049 | 19 | |
fcdc6537 | 20 | #include <asm/clint.h> |
76d2a049 PD |
21 | #include <asm/sbi.h> |
22 | #include <asm/tlbflush.h> | |
23 | #include <asm/cacheflush.h> | |
24 | ||
76d2a049 PD |
25 | enum ipi_message_type { |
26 | IPI_RESCHEDULE, | |
27 | IPI_CALL_FUNC, | |
37a107ff | 28 | IPI_CPU_STOP, |
76d2a049 PD |
29 | IPI_MAX |
30 | }; | |
31 | ||
78d1daa3 AP |
32 | unsigned long __cpuid_to_hartid_map[NR_CPUS] = { |
33 | [0 ... NR_CPUS-1] = INVALID_HARTID | |
34 | }; | |
35 | ||
36 | void __init smp_setup_processor_id(void) | |
37 | { | |
f1f47c6c | 38 | cpuid_to_hartid_map(0) = boot_cpu_hartid; |
78d1daa3 AP |
39 | } |
40 | ||
8b20d2db AP |
41 | /* A collection of single bit ipi messages. */ |
42 | static struct { | |
43 | unsigned long stats[IPI_MAX] ____cacheline_aligned; | |
44 | unsigned long bits ____cacheline_aligned; | |
45 | } ipi_data[NR_CPUS] __cacheline_aligned; | |
46 | ||
6825c7a8 AP |
47 | int riscv_hartid_to_cpuid(int hartid) |
48 | { | |
f1f47c6c | 49 | int i; |
6825c7a8 AP |
50 | |
51 | for (i = 0; i < NR_CPUS; i++) | |
52 | if (cpuid_to_hartid_map(i) == hartid) | |
53 | return i; | |
54 | ||
55 | pr_err("Couldn't find cpu id for hartid [%d]\n", hartid); | |
6825c7a8 AP |
56 | return i; |
57 | } | |
4bde6328 | 58 | |
6825c7a8 AP |
59 | void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out) |
60 | { | |
61 | int cpu; | |
62 | ||
f5bf645d | 63 | cpumask_clear(out); |
6825c7a8 AP |
64 | for_each_cpu(cpu, in) |
65 | cpumask_set_cpu(cpuid_to_hartid_map(cpu), out); | |
66 | } | |
7391efa4 | 67 | EXPORT_SYMBOL_GPL(riscv_cpuid_to_hartid_mask); |
70114560 AP |
68 | |
69 | bool arch_match_cpu_phys_id(int cpu, u64 phys_id) | |
70 | { | |
71 | return phys_id == cpuid_to_hartid_map(cpu); | |
72 | } | |
73 | ||
4bde6328 OJ |
74 | /* Unsupported */ |
75 | int setup_profiling_timer(unsigned int multiplier) | |
76 | { | |
77 | return -EINVAL; | |
78 | } | |
79 | ||
37a107ff AS |
80 | static void ipi_stop(void) |
81 | { | |
82 | set_cpu_online(smp_processor_id(), false); | |
83 | while (1) | |
84 | wait_for_interrupt(); | |
85 | } | |
86 | ||
7e0e5089 CH |
87 | static void send_ipi_mask(const struct cpumask *mask, enum ipi_message_type op) |
88 | { | |
7e0e5089 | 89 | struct cpumask hartid_mask; |
1db7a7ca CH |
90 | int cpu; |
91 | ||
92 | smp_mb__before_atomic(); | |
93 | for_each_cpu(cpu, mask) | |
94 | set_bit(op, &ipi_data[cpu].bits); | |
95 | smp_mb__after_atomic(); | |
7e0e5089 | 96 | |
1db7a7ca | 97 | riscv_cpuid_to_hartid_mask(mask, &hartid_mask); |
fcdc6537 CH |
98 | if (IS_ENABLED(CONFIG_RISCV_SBI)) |
99 | sbi_send_ipi(cpumask_bits(&hartid_mask)); | |
100 | else | |
3384b043 | 101 | clint_send_ipi_mask(mask); |
7e0e5089 CH |
102 | } |
103 | ||
104 | static void send_ipi_single(int cpu, enum ipi_message_type op) | |
105 | { | |
e11ea2a0 CH |
106 | int hartid = cpuid_to_hartid_map(cpu); |
107 | ||
108 | smp_mb__before_atomic(); | |
109 | set_bit(op, &ipi_data[cpu].bits); | |
110 | smp_mb__after_atomic(); | |
111 | ||
fcdc6537 CH |
112 | if (IS_ENABLED(CONFIG_RISCV_SBI)) |
113 | sbi_send_ipi(cpumask_bits(cpumask_of(hartid))); | |
114 | else | |
115 | clint_send_ipi_single(hartid); | |
7e0e5089 CH |
116 | } |
117 | ||
118 | static inline void clear_ipi(void) | |
119 | { | |
fcdc6537 CH |
120 | if (IS_ENABLED(CONFIG_RISCV_SBI)) |
121 | csr_clear(CSR_IP, IE_SIE); | |
122 | else | |
123 | clint_clear_ipi(cpuid_to_hartid_map(smp_processor_id())); | |
7e0e5089 CH |
124 | } |
125 | ||
b9d55357 | 126 | void riscv_software_interrupt(void) |
76d2a049 PD |
127 | { |
128 | unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits; | |
8b20d2db | 129 | unsigned long *stats = ipi_data[smp_processor_id()].stats; |
76d2a049 | 130 | |
7e0e5089 | 131 | clear_ipi(); |
76d2a049 PD |
132 | |
133 | while (true) { | |
134 | unsigned long ops; | |
135 | ||
136 | /* Order bit clearing and data access. */ | |
137 | mb(); | |
138 | ||
139 | ops = xchg(pending_ipis, 0); | |
140 | if (ops == 0) | |
b9d55357 | 141 | return; |
76d2a049 | 142 | |
8b20d2db AP |
143 | if (ops & (1 << IPI_RESCHEDULE)) { |
144 | stats[IPI_RESCHEDULE]++; | |
76d2a049 | 145 | scheduler_ipi(); |
8b20d2db | 146 | } |
76d2a049 | 147 | |
8b20d2db AP |
148 | if (ops & (1 << IPI_CALL_FUNC)) { |
149 | stats[IPI_CALL_FUNC]++; | |
76d2a049 | 150 | generic_smp_call_function_interrupt(); |
8b20d2db | 151 | } |
76d2a049 | 152 | |
37a107ff AS |
153 | if (ops & (1 << IPI_CPU_STOP)) { |
154 | stats[IPI_CPU_STOP]++; | |
155 | ipi_stop(); | |
156 | } | |
157 | ||
76d2a049 PD |
158 | BUG_ON((ops >> IPI_MAX) != 0); |
159 | ||
160 | /* Order data access and bit testing. */ | |
161 | mb(); | |
162 | } | |
76d2a049 PD |
163 | } |
164 | ||
8b20d2db AP |
165 | static const char * const ipi_names[] = { |
166 | [IPI_RESCHEDULE] = "Rescheduling interrupts", | |
167 | [IPI_CALL_FUNC] = "Function call interrupts", | |
37a107ff | 168 | [IPI_CPU_STOP] = "CPU stop interrupts", |
8b20d2db AP |
169 | }; |
170 | ||
171 | void show_ipi_stats(struct seq_file *p, int prec) | |
172 | { | |
173 | unsigned int cpu, i; | |
174 | ||
175 | for (i = 0; i < IPI_MAX; i++) { | |
176 | seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, | |
177 | prec >= 4 ? " " : ""); | |
178 | for_each_online_cpu(cpu) | |
179 | seq_printf(p, "%10lu ", ipi_data[cpu].stats[i]); | |
180 | seq_printf(p, " %s\n", ipi_names[i]); | |
181 | } | |
182 | } | |
183 | ||
76d2a049 PD |
184 | void arch_send_call_function_ipi_mask(struct cpumask *mask) |
185 | { | |
7e0e5089 | 186 | send_ipi_mask(mask, IPI_CALL_FUNC); |
76d2a049 PD |
187 | } |
188 | ||
189 | void arch_send_call_function_single_ipi(int cpu) | |
190 | { | |
7e0e5089 | 191 | send_ipi_single(cpu, IPI_CALL_FUNC); |
76d2a049 PD |
192 | } |
193 | ||
76d2a049 PD |
194 | void smp_send_stop(void) |
195 | { | |
37a107ff AS |
196 | unsigned long timeout; |
197 | ||
198 | if (num_online_cpus() > 1) { | |
199 | cpumask_t mask; | |
200 | ||
201 | cpumask_copy(&mask, cpu_online_mask); | |
202 | cpumask_clear_cpu(smp_processor_id(), &mask); | |
203 | ||
204 | if (system_state <= SYSTEM_RUNNING) | |
205 | pr_crit("SMP: stopping secondary CPUs\n"); | |
7e0e5089 | 206 | send_ipi_mask(&mask, IPI_CPU_STOP); |
37a107ff AS |
207 | } |
208 | ||
209 | /* Wait up to one second for other CPUs to stop */ | |
210 | timeout = USEC_PER_SEC; | |
211 | while (num_online_cpus() > 1 && timeout--) | |
212 | udelay(1); | |
213 | ||
214 | if (num_online_cpus() > 1) | |
215 | pr_warn("SMP: failed to stop secondary CPUs %*pbl\n", | |
216 | cpumask_pr_args(cpu_online_mask)); | |
76d2a049 PD |
217 | } |
218 | ||
219 | void smp_send_reschedule(int cpu) | |
220 | { | |
7e0e5089 | 221 | send_ipi_single(cpu, IPI_RESCHEDULE); |
76d2a049 | 222 | } |
d3d7a0ce | 223 | EXPORT_SYMBOL_GPL(smp_send_reschedule); |