1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/threads.h>
3 #include <linux/cpumask.h>
4 #include <linux/string.h>
5 #include <linux/kernel.h>
6 #include <linux/ctype.h>
7 #include <linux/dmar.h>
12 #include <asm/x2apic.h>
14 static DEFINE_PER_CPU(u32
, x86_cpu_to_logical_apicid
);
15 static DEFINE_PER_CPU(cpumask_var_t
, cpus_in_cluster
);
16 static DEFINE_PER_CPU(cpumask_var_t
, ipi_mask
);
18 static int x2apic_acpi_madt_oem_check(char *oem_id
, char *oem_table_id
)
20 return x2apic_enabled();
23 static inline u32
x2apic_cluster(int cpu
)
25 return per_cpu(x86_cpu_to_logical_apicid
, cpu
) >> 16;
28 static void x2apic_send_IPI(int cpu
, int vector
)
30 u32 dest
= per_cpu(x86_cpu_to_logical_apicid
, cpu
);
33 __x2apic_send_IPI_dest(dest
, vector
, APIC_DEST_LOGICAL
);
37 __x2apic_send_IPI_mask(const struct cpumask
*mask
, int vector
, int apic_dest
)
39 struct cpumask
*cpus_in_cluster_ptr
;
40 struct cpumask
*ipi_mask_ptr
;
41 unsigned int cpu
, this_cpu
;
47 local_irq_save(flags
);
49 this_cpu
= smp_processor_id();
52 * We are to modify mask, so we need an own copy
53 * and be sure it's manipulated with irq off.
55 ipi_mask_ptr
= this_cpu_cpumask_var_ptr(ipi_mask
);
56 cpumask_copy(ipi_mask_ptr
, mask
);
59 * The idea is to send one IPI per cluster.
61 for_each_cpu(cpu
, ipi_mask_ptr
) {
64 cpus_in_cluster_ptr
= per_cpu(cpus_in_cluster
, cpu
);
67 /* Collect cpus in cluster. */
68 for_each_cpu_and(i
, ipi_mask_ptr
, cpus_in_cluster_ptr
) {
69 if (apic_dest
== APIC_DEST_ALLINC
|| i
!= this_cpu
)
70 dest
|= per_cpu(x86_cpu_to_logical_apicid
, i
);
76 __x2apic_send_IPI_dest(dest
, vector
, apic
->dest_logical
);
78 * Cluster sibling cpus should be discared now so
79 * we would not send IPI them second time.
81 cpumask_andnot(ipi_mask_ptr
, ipi_mask_ptr
, cpus_in_cluster_ptr
);
84 local_irq_restore(flags
);
87 static void x2apic_send_IPI_mask(const struct cpumask
*mask
, int vector
)
89 __x2apic_send_IPI_mask(mask
, vector
, APIC_DEST_ALLINC
);
93 x2apic_send_IPI_mask_allbutself(const struct cpumask
*mask
, int vector
)
95 __x2apic_send_IPI_mask(mask
, vector
, APIC_DEST_ALLBUT
);
98 static void x2apic_send_IPI_allbutself(int vector
)
100 __x2apic_send_IPI_mask(cpu_online_mask
, vector
, APIC_DEST_ALLBUT
);
103 static void x2apic_send_IPI_all(int vector
)
105 __x2apic_send_IPI_mask(cpu_online_mask
, vector
, APIC_DEST_ALLINC
);
109 x2apic_cpu_mask_to_apicid(const struct cpumask
*mask
, struct irq_data
*irqdata
,
110 unsigned int *apicid
)
112 struct cpumask
*effmsk
= irq_data_get_effective_affinity_mask(irqdata
);
117 cpu
= cpumask_first(mask
);
118 if (cpu
>= nr_cpu_ids
)
121 dest
= per_cpu(x86_cpu_to_logical_apicid
, cpu
);
122 cluster
= x2apic_cluster(cpu
);
124 cpumask_clear(effmsk
);
125 for_each_cpu(cpu
, mask
) {
126 if (cluster
!= x2apic_cluster(cpu
))
128 dest
|= per_cpu(x86_cpu_to_logical_apicid
, cpu
);
129 cpumask_set_cpu(cpu
, effmsk
);
136 static void init_x2apic_ldr(void)
138 unsigned int this_cpu
= smp_processor_id();
141 per_cpu(x86_cpu_to_logical_apicid
, this_cpu
) = apic_read(APIC_LDR
);
143 cpumask_set_cpu(this_cpu
, per_cpu(cpus_in_cluster
, this_cpu
));
144 for_each_online_cpu(cpu
) {
145 if (x2apic_cluster(this_cpu
) != x2apic_cluster(cpu
))
147 cpumask_set_cpu(this_cpu
, per_cpu(cpus_in_cluster
, cpu
));
148 cpumask_set_cpu(cpu
, per_cpu(cpus_in_cluster
, this_cpu
));
153 * At CPU state changes, update the x2apic cluster sibling info.
155 static int x2apic_prepare_cpu(unsigned int cpu
)
157 if (!zalloc_cpumask_var(&per_cpu(cpus_in_cluster
, cpu
), GFP_KERNEL
))
160 if (!zalloc_cpumask_var(&per_cpu(ipi_mask
, cpu
), GFP_KERNEL
)) {
161 free_cpumask_var(per_cpu(cpus_in_cluster
, cpu
));
168 static int x2apic_dead_cpu(unsigned int this_cpu
)
172 for_each_online_cpu(cpu
) {
173 if (x2apic_cluster(this_cpu
) != x2apic_cluster(cpu
))
175 cpumask_clear_cpu(this_cpu
, per_cpu(cpus_in_cluster
, cpu
));
176 cpumask_clear_cpu(cpu
, per_cpu(cpus_in_cluster
, this_cpu
));
178 free_cpumask_var(per_cpu(cpus_in_cluster
, this_cpu
));
179 free_cpumask_var(per_cpu(ipi_mask
, this_cpu
));
183 static int x2apic_cluster_probe(void)
185 int cpu
= smp_processor_id();
191 ret
= cpuhp_setup_state(CPUHP_X2APIC_PREPARE
, "x86/x2apic:prepare",
192 x2apic_prepare_cpu
, x2apic_dead_cpu
);
194 pr_err("Failed to register X2APIC_PREPARE\n");
197 cpumask_set_cpu(cpu
, per_cpu(cpus_in_cluster
, cpu
));
201 static const struct cpumask
*x2apic_cluster_target_cpus(void)
207 * Each x2apic cluster is an allocation domain.
209 static void cluster_vector_allocation_domain(int cpu
, struct cpumask
*retmask
,
210 const struct cpumask
*mask
)
213 * To minimize vector pressure, default case of boot, device bringup
214 * etc will use a single cpu for the interrupt destination.
216 * On explicit migration requests coming from irqbalance etc,
217 * interrupts will be routed to the x2apic cluster (cluster-id
218 * derived from the first cpu in the mask) members specified
221 if (mask
== x2apic_cluster_target_cpus())
222 cpumask_copy(retmask
, cpumask_of(cpu
));
224 cpumask_and(retmask
, mask
, per_cpu(cpus_in_cluster
, cpu
));
227 static struct apic apic_x2apic_cluster __ro_after_init
= {
229 .name
= "cluster x2apic",
230 .probe
= x2apic_cluster_probe
,
231 .acpi_madt_oem_check
= x2apic_acpi_madt_oem_check
,
232 .apic_id_valid
= x2apic_apic_id_valid
,
233 .apic_id_registered
= x2apic_apic_id_registered
,
235 .irq_delivery_mode
= dest_LowestPrio
,
236 .irq_dest_mode
= 1, /* logical */
238 .target_cpus
= x2apic_cluster_target_cpus
,
240 .dest_logical
= APIC_DEST_LOGICAL
,
241 .check_apicid_used
= NULL
,
243 .vector_allocation_domain
= cluster_vector_allocation_domain
,
244 .init_apic_ldr
= init_x2apic_ldr
,
246 .ioapic_phys_id_map
= NULL
,
247 .setup_apic_routing
= NULL
,
248 .cpu_present_to_apicid
= default_cpu_present_to_apicid
,
249 .apicid_to_cpu_present
= NULL
,
250 .check_phys_apicid_present
= default_check_phys_apicid_present
,
251 .phys_pkg_id
= x2apic_phys_pkg_id
,
253 .get_apic_id
= x2apic_get_apic_id
,
254 .set_apic_id
= x2apic_set_apic_id
,
256 .cpu_mask_to_apicid
= x2apic_cpu_mask_to_apicid
,
258 .send_IPI
= x2apic_send_IPI
,
259 .send_IPI_mask
= x2apic_send_IPI_mask
,
260 .send_IPI_mask_allbutself
= x2apic_send_IPI_mask_allbutself
,
261 .send_IPI_allbutself
= x2apic_send_IPI_allbutself
,
262 .send_IPI_all
= x2apic_send_IPI_all
,
263 .send_IPI_self
= x2apic_send_IPI_self
,
265 .inquire_remote_apic
= NULL
,
267 .read
= native_apic_msr_read
,
268 .write
= native_apic_msr_write
,
269 .eoi_write
= native_apic_msr_eoi_write
,
270 .icr_read
= native_x2apic_icr_read
,
271 .icr_write
= native_x2apic_icr_write
,
272 .wait_icr_idle
= native_x2apic_wait_icr_idle
,
273 .safe_wait_icr_idle
= native_safe_x2apic_wait_icr_idle
,
276 apic_driver(apic_x2apic_cluster
);