1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
7 #define pr_fmt(fmt) "GICv3: " fmt
9 #include <linux/acpi.h>
10 #include <linux/cpu.h>
11 #include <linux/cpu_pm.h>
12 #include <linux/delay.h>
13 #include <linux/interrupt.h>
14 #include <linux/irqdomain.h>
16 #include <linux/of_address.h>
17 #include <linux/of_irq.h>
18 #include <linux/percpu.h>
19 #include <linux/refcount.h>
20 #include <linux/slab.h>
22 #include <linux/irqchip.h>
23 #include <linux/irqchip/arm-gic-common.h>
24 #include <linux/irqchip/arm-gic-v3.h>
25 #include <linux/irqchip/irq-partition-percpu.h>
27 #include <asm/cputype.h>
28 #include <asm/exception.h>
29 #include <asm/smp_plat.h>
32 #include "irq-gic-common.h"
34 #define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80)
36 #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0)
38 struct redist_region
{
39 void __iomem
*redist_base
;
40 phys_addr_t phys_base
;
44 struct gic_chip_data
{
45 struct fwnode_handle
*fwnode
;
46 void __iomem
*dist_base
;
47 struct redist_region
*redist_regions
;
49 struct irq_domain
*domain
;
51 u32 nr_redist_regions
;
55 struct partition_desc
**ppi_descs
;
58 static struct gic_chip_data gic_data __read_mostly
;
59 static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key
);
61 #define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer))
62 #define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U)
63 #define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer)
66 * The behaviours of RPR and PMR registers differ depending on the value of
67 * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the
68 * distributor and redistributors depends on whether security is enabled in the
71 * When security is enabled, non-secure priority values from the (re)distributor
72 * are presented to the GIC CPUIF as follow:
73 * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80;
75 * If SCR_EL3.FIQ == 1, the values writen to/read from PMR and RPR at non-secure
76 * EL1 are subject to a similar operation thus matching the priorities presented
77 * from the (re)distributor when security is enabled.
79 * see GICv3/GICv4 Architecture Specification (IHI0069D):
80 * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt
82 * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1
85 * For now, we only support pseudo-NMIs if we have non-secure view of
88 static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis
);
90 /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */
91 static refcount_t
*ppi_nmi_refs
;
93 static struct gic_kvm_info gic_v3_kvm_info
;
94 static DEFINE_PER_CPU(bool, has_rss
);
96 #define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4)
97 #define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist))
98 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
99 #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K)
101 /* Our default, arbitrary priority value. Linux only uses one anyway. */
102 #define DEFAULT_PMR_VALUE 0xf0
104 enum gic_intid_range
{
113 static enum gic_intid_range
__get_intid_range(irq_hw_number_t hwirq
)
120 case EPPI_BASE_INTID
... (EPPI_BASE_INTID
+ 63):
122 case ESPI_BASE_INTID
... (ESPI_BASE_INTID
+ 1023):
124 case 8192 ... GENMASK(23, 0):
127 return __INVALID_RANGE__
;
131 static enum gic_intid_range
get_intid_range(struct irq_data
*d
)
133 return __get_intid_range(d
->hwirq
);
136 static inline unsigned int gic_irq(struct irq_data
*d
)
141 static inline int gic_irq_in_rdist(struct irq_data
*d
)
143 enum gic_intid_range range
= get_intid_range(d
);
144 return range
== PPI_RANGE
|| range
== EPPI_RANGE
;
147 static inline void __iomem
*gic_dist_base(struct irq_data
*d
)
149 switch (get_intid_range(d
)) {
152 /* SGI+PPI -> SGI_base for this CPU */
153 return gic_data_rdist_sgi_base();
157 /* SPI -> dist_base */
158 return gic_data
.dist_base
;
165 static void gic_do_wait_for_rwp(void __iomem
*base
)
167 u32 count
= 1000000; /* 1s! */
169 while (readl_relaxed(base
+ GICD_CTLR
) & GICD_CTLR_RWP
) {
172 pr_err_ratelimited("RWP timeout, gone fishing\n");
180 /* Wait for completion of a distributor change */
181 static void gic_dist_wait_for_rwp(void)
183 gic_do_wait_for_rwp(gic_data
.dist_base
);
186 /* Wait for completion of a redistributor change */
187 static void gic_redist_wait_for_rwp(void)
189 gic_do_wait_for_rwp(gic_data_rdist_rd_base());
194 static u64 __maybe_unused
gic_read_iar(void)
196 if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154
))
197 return gic_read_iar_cavium_thunderx();
199 return gic_read_iar_common();
203 static void gic_enable_redist(bool enable
)
206 u32 count
= 1000000; /* 1s! */
209 if (gic_data
.flags
& FLAGS_WORKAROUND_GICR_WAKER_MSM8996
)
212 rbase
= gic_data_rdist_rd_base();
214 val
= readl_relaxed(rbase
+ GICR_WAKER
);
216 /* Wake up this CPU redistributor */
217 val
&= ~GICR_WAKER_ProcessorSleep
;
219 val
|= GICR_WAKER_ProcessorSleep
;
220 writel_relaxed(val
, rbase
+ GICR_WAKER
);
222 if (!enable
) { /* Check that GICR_WAKER is writeable */
223 val
= readl_relaxed(rbase
+ GICR_WAKER
);
224 if (!(val
& GICR_WAKER_ProcessorSleep
))
225 return; /* No PM support in this redistributor */
229 val
= readl_relaxed(rbase
+ GICR_WAKER
);
230 if (enable
^ (bool)(val
& GICR_WAKER_ChildrenAsleep
))
236 pr_err_ratelimited("redistributor failed to %s...\n",
237 enable
? "wakeup" : "sleep");
241 * Routines to disable, enable, EOI and route interrupts
243 static u32
convert_offset_index(struct irq_data
*d
, u32 offset
, u32
*index
)
245 switch (get_intid_range(d
)) {
252 * Contrary to the ESPI range, the EPPI range is contiguous
253 * to the PPI range in the registers, so let's adjust the
254 * displacement accordingly. Consistency is overrated.
256 *index
= d
->hwirq
- EPPI_BASE_INTID
+ 32;
259 *index
= d
->hwirq
- ESPI_BASE_INTID
;
262 return GICD_ISENABLERnE
;
264 return GICD_ICENABLERnE
;
266 return GICD_ISPENDRnE
;
268 return GICD_ICPENDRnE
;
270 return GICD_ISACTIVERnE
;
272 return GICD_ICACTIVERnE
;
273 case GICD_IPRIORITYR
:
274 return GICD_IPRIORITYRnE
;
278 return GICD_IROUTERnE
;
292 static int gic_peek_irq(struct irq_data
*d
, u32 offset
)
297 offset
= convert_offset_index(d
, offset
, &index
);
298 mask
= 1 << (index
% 32);
300 if (gic_irq_in_rdist(d
))
301 base
= gic_data_rdist_sgi_base();
303 base
= gic_data
.dist_base
;
305 return !!(readl_relaxed(base
+ offset
+ (index
/ 32) * 4) & mask
);
308 static void gic_poke_irq(struct irq_data
*d
, u32 offset
)
310 void (*rwp_wait
)(void);
314 offset
= convert_offset_index(d
, offset
, &index
);
315 mask
= 1 << (index
% 32);
317 if (gic_irq_in_rdist(d
)) {
318 base
= gic_data_rdist_sgi_base();
319 rwp_wait
= gic_redist_wait_for_rwp
;
321 base
= gic_data
.dist_base
;
322 rwp_wait
= gic_dist_wait_for_rwp
;
325 writel_relaxed(mask
, base
+ offset
+ (index
/ 32) * 4);
329 static void gic_mask_irq(struct irq_data
*d
)
331 gic_poke_irq(d
, GICD_ICENABLER
);
334 static void gic_eoimode1_mask_irq(struct irq_data
*d
)
338 * When masking a forwarded interrupt, make sure it is
339 * deactivated as well.
341 * This ensures that an interrupt that is getting
342 * disabled/masked will not get "stuck", because there is
343 * noone to deactivate it (guest is being terminated).
345 if (irqd_is_forwarded_to_vcpu(d
))
346 gic_poke_irq(d
, GICD_ICACTIVER
);
349 static void gic_unmask_irq(struct irq_data
*d
)
351 gic_poke_irq(d
, GICD_ISENABLER
);
354 static inline bool gic_supports_nmi(void)
356 return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI
) &&
357 static_branch_likely(&supports_pseudo_nmis
);
360 static int gic_irq_set_irqchip_state(struct irq_data
*d
,
361 enum irqchip_irq_state which
, bool val
)
365 if (d
->hwirq
>= 8192) /* PPI/SPI only */
369 case IRQCHIP_STATE_PENDING
:
370 reg
= val
? GICD_ISPENDR
: GICD_ICPENDR
;
373 case IRQCHIP_STATE_ACTIVE
:
374 reg
= val
? GICD_ISACTIVER
: GICD_ICACTIVER
;
377 case IRQCHIP_STATE_MASKED
:
378 reg
= val
? GICD_ICENABLER
: GICD_ISENABLER
;
385 gic_poke_irq(d
, reg
);
389 static int gic_irq_get_irqchip_state(struct irq_data
*d
,
390 enum irqchip_irq_state which
, bool *val
)
392 if (d
->hwirq
>= 8192) /* PPI/SPI only */
396 case IRQCHIP_STATE_PENDING
:
397 *val
= gic_peek_irq(d
, GICD_ISPENDR
);
400 case IRQCHIP_STATE_ACTIVE
:
401 *val
= gic_peek_irq(d
, GICD_ISACTIVER
);
404 case IRQCHIP_STATE_MASKED
:
405 *val
= !gic_peek_irq(d
, GICD_ISENABLER
);
415 static void gic_irq_set_prio(struct irq_data
*d
, u8 prio
)
417 void __iomem
*base
= gic_dist_base(d
);
420 offset
= convert_offset_index(d
, GICD_IPRIORITYR
, &index
);
422 writeb_relaxed(prio
, base
+ offset
+ index
);
425 static u32
gic_get_ppi_index(struct irq_data
*d
)
427 switch (get_intid_range(d
)) {
429 return d
->hwirq
- 16;
431 return d
->hwirq
- EPPI_BASE_INTID
+ 16;
437 static int gic_irq_nmi_setup(struct irq_data
*d
)
439 struct irq_desc
*desc
= irq_to_desc(d
->irq
);
441 if (!gic_supports_nmi())
444 if (gic_peek_irq(d
, GICD_ISENABLER
)) {
445 pr_err("Cannot set NMI property of enabled IRQ %u\n", d
->irq
);
450 * A secondary irq_chip should be in charge of LPI request,
451 * it should not be possible to get there
453 if (WARN_ON(gic_irq(d
) >= 8192))
456 /* desc lock should already be held */
457 if (gic_irq_in_rdist(d
)) {
458 u32 idx
= gic_get_ppi_index(d
);
460 /* Setting up PPI as NMI, only switch handler for first NMI */
461 if (!refcount_inc_not_zero(&ppi_nmi_refs
[idx
])) {
462 refcount_set(&ppi_nmi_refs
[idx
], 1);
463 desc
->handle_irq
= handle_percpu_devid_fasteoi_nmi
;
466 desc
->handle_irq
= handle_fasteoi_nmi
;
469 gic_irq_set_prio(d
, GICD_INT_NMI_PRI
);
474 static void gic_irq_nmi_teardown(struct irq_data
*d
)
476 struct irq_desc
*desc
= irq_to_desc(d
->irq
);
478 if (WARN_ON(!gic_supports_nmi()))
481 if (gic_peek_irq(d
, GICD_ISENABLER
)) {
482 pr_err("Cannot set NMI property of enabled IRQ %u\n", d
->irq
);
487 * A secondary irq_chip should be in charge of LPI request,
488 * it should not be possible to get there
490 if (WARN_ON(gic_irq(d
) >= 8192))
493 /* desc lock should already be held */
494 if (gic_irq_in_rdist(d
)) {
495 u32 idx
= gic_get_ppi_index(d
);
497 /* Tearing down NMI, only switch handler for last NMI */
498 if (refcount_dec_and_test(&ppi_nmi_refs
[idx
]))
499 desc
->handle_irq
= handle_percpu_devid_irq
;
501 desc
->handle_irq
= handle_fasteoi_irq
;
504 gic_irq_set_prio(d
, GICD_INT_DEF_PRI
);
507 static void gic_eoi_irq(struct irq_data
*d
)
509 gic_write_eoir(gic_irq(d
));
512 static void gic_eoimode1_eoi_irq(struct irq_data
*d
)
515 * No need to deactivate an LPI, or an interrupt that
516 * is is getting forwarded to a vcpu.
518 if (gic_irq(d
) >= 8192 || irqd_is_forwarded_to_vcpu(d
))
520 gic_write_dir(gic_irq(d
));
523 static int gic_set_type(struct irq_data
*d
, unsigned int type
)
525 enum gic_intid_range range
;
526 unsigned int irq
= gic_irq(d
);
527 void (*rwp_wait
)(void);
532 /* Interrupt configuration for SGIs can't be changed */
536 range
= get_intid_range(d
);
538 /* SPIs have restrictions on the supported types */
539 if ((range
== SPI_RANGE
|| range
== ESPI_RANGE
) &&
540 type
!= IRQ_TYPE_LEVEL_HIGH
&& type
!= IRQ_TYPE_EDGE_RISING
)
543 if (gic_irq_in_rdist(d
)) {
544 base
= gic_data_rdist_sgi_base();
545 rwp_wait
= gic_redist_wait_for_rwp
;
547 base
= gic_data
.dist_base
;
548 rwp_wait
= gic_dist_wait_for_rwp
;
551 offset
= convert_offset_index(d
, GICD_ICFGR
, &index
);
553 ret
= gic_configure_irq(index
, type
, base
+ offset
, rwp_wait
);
554 if (ret
&& (range
== PPI_RANGE
|| range
== EPPI_RANGE
)) {
555 /* Misconfigured PPIs are usually not fatal */
556 pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq
);
563 static int gic_irq_set_vcpu_affinity(struct irq_data
*d
, void *vcpu
)
566 irqd_set_forwarded_to_vcpu(d
);
568 irqd_clr_forwarded_to_vcpu(d
);
572 static u64
gic_mpidr_to_affinity(unsigned long mpidr
)
576 aff
= ((u64
)MPIDR_AFFINITY_LEVEL(mpidr
, 3) << 32 |
577 MPIDR_AFFINITY_LEVEL(mpidr
, 2) << 16 |
578 MPIDR_AFFINITY_LEVEL(mpidr
, 1) << 8 |
579 MPIDR_AFFINITY_LEVEL(mpidr
, 0));
584 static void gic_deactivate_unhandled(u32 irqnr
)
586 if (static_branch_likely(&supports_deactivate_key
)) {
588 gic_write_dir(irqnr
);
590 gic_write_eoir(irqnr
);
594 static inline void gic_handle_nmi(u32 irqnr
, struct pt_regs
*regs
)
596 bool irqs_enabled
= interrupts_enabled(regs
);
602 if (static_branch_likely(&supports_deactivate_key
))
603 gic_write_eoir(irqnr
);
605 * Leave the PSR.I bit set to prevent other NMIs to be
606 * received while handling this one.
607 * PSR.I will be restored when we ERET to the
608 * interrupted context.
610 err
= handle_domain_nmi(gic_data
.domain
, irqnr
, regs
);
612 gic_deactivate_unhandled(irqnr
);
618 static asmlinkage
void __exception_irq_entry
gic_handle_irq(struct pt_regs
*regs
)
622 irqnr
= gic_read_iar();
624 if (gic_supports_nmi() &&
625 unlikely(gic_read_rpr() == GICD_INT_NMI_PRI
)) {
626 gic_handle_nmi(irqnr
, regs
);
630 if (gic_prio_masking_enabled()) {
632 gic_arch_enable_irqs();
635 /* Check for special IDs first */
636 if ((irqnr
>= 1020 && irqnr
<= 1023))
639 /* Treat anything but SGIs in a uniform way */
640 if (likely(irqnr
> 15)) {
643 if (static_branch_likely(&supports_deactivate_key
))
644 gic_write_eoir(irqnr
);
648 err
= handle_domain_irq(gic_data
.domain
, irqnr
, regs
);
650 WARN_ONCE(true, "Unexpected interrupt received!\n");
651 gic_deactivate_unhandled(irqnr
);
656 gic_write_eoir(irqnr
);
657 if (static_branch_likely(&supports_deactivate_key
))
658 gic_write_dir(irqnr
);
661 * Unlike GICv2, we don't need an smp_rmb() here.
662 * The control dependency from gic_read_iar to
663 * the ISB in gic_write_eoir is enough to ensure
664 * that any shared data read by handle_IPI will
665 * be read after the ACK.
667 handle_IPI(irqnr
, regs
);
669 WARN_ONCE(true, "Unexpected SGI received!\n");
674 static u32
gic_get_pribits(void)
678 pribits
= gic_read_ctlr();
679 pribits
&= ICC_CTLR_EL1_PRI_BITS_MASK
;
680 pribits
>>= ICC_CTLR_EL1_PRI_BITS_SHIFT
;
686 static bool gic_has_group0(void)
691 old_pmr
= gic_read_pmr();
694 * Let's find out if Group0 is under control of EL3 or not by
695 * setting the highest possible, non-zero priority in PMR.
697 * If SCR_EL3.FIQ is set, the priority gets shifted down in
698 * order for the CPU interface to set bit 7, and keep the
699 * actual priority in the non-secure range. In the process, it
700 * looses the least significant bit and the actual priority
701 * becomes 0x80. Reading it back returns 0, indicating that
702 * we're don't have access to Group0.
704 gic_write_pmr(BIT(8 - gic_get_pribits()));
705 val
= gic_read_pmr();
707 gic_write_pmr(old_pmr
);
712 static void __init
gic_dist_init(void)
716 void __iomem
*base
= gic_data
.dist_base
;
718 /* Disable the distributor */
719 writel_relaxed(0, base
+ GICD_CTLR
);
720 gic_dist_wait_for_rwp();
723 * Configure SPIs as non-secure Group-1. This will only matter
724 * if the GIC only has a single security state. This will not
725 * do the right thing if the kernel is running in secure mode,
726 * but that's not the intended use case anyway.
728 for (i
= 32; i
< GIC_LINE_NR
; i
+= 32)
729 writel_relaxed(~0, base
+ GICD_IGROUPR
+ i
/ 8);
731 /* Extended SPI range, not handled by the GICv2/GICv3 common code */
732 for (i
= 0; i
< GIC_ESPI_NR
; i
+= 32) {
733 writel_relaxed(~0U, base
+ GICD_ICENABLERnE
+ i
/ 8);
734 writel_relaxed(~0U, base
+ GICD_ICACTIVERnE
+ i
/ 8);
737 for (i
= 0; i
< GIC_ESPI_NR
; i
+= 32)
738 writel_relaxed(~0U, base
+ GICD_IGROUPRnE
+ i
/ 8);
740 for (i
= 0; i
< GIC_ESPI_NR
; i
+= 16)
741 writel_relaxed(0, base
+ GICD_ICFGRnE
+ i
/ 4);
743 for (i
= 0; i
< GIC_ESPI_NR
; i
+= 4)
744 writel_relaxed(GICD_INT_DEF_PRI_X4
, base
+ GICD_IPRIORITYRnE
+ i
);
746 /* Now do the common stuff, and wait for the distributor to drain */
747 gic_dist_config(base
, GIC_LINE_NR
, gic_dist_wait_for_rwp
);
749 /* Enable distributor with ARE, Group1 */
750 writel_relaxed(GICD_CTLR_ARE_NS
| GICD_CTLR_ENABLE_G1A
| GICD_CTLR_ENABLE_G1
,
754 * Set all global interrupts to the boot CPU only. ARE must be
757 affinity
= gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
758 for (i
= 32; i
< GIC_LINE_NR
; i
++)
759 gic_write_irouter(affinity
, base
+ GICD_IROUTER
+ i
* 8);
761 for (i
= 0; i
< GIC_ESPI_NR
; i
++)
762 gic_write_irouter(affinity
, base
+ GICD_IROUTERnE
+ i
* 8);
765 static int gic_iterate_rdists(int (*fn
)(struct redist_region
*, void __iomem
*))
770 for (i
= 0; i
< gic_data
.nr_redist_regions
; i
++) {
771 void __iomem
*ptr
= gic_data
.redist_regions
[i
].redist_base
;
775 reg
= readl_relaxed(ptr
+ GICR_PIDR2
) & GIC_PIDR2_ARCH_MASK
;
776 if (reg
!= GIC_PIDR2_ARCH_GICv3
&&
777 reg
!= GIC_PIDR2_ARCH_GICv4
) { /* We're in trouble... */
778 pr_warn("No redistributor present @%p\n", ptr
);
783 typer
= gic_read_typer(ptr
+ GICR_TYPER
);
784 ret
= fn(gic_data
.redist_regions
+ i
, ptr
);
788 if (gic_data
.redist_regions
[i
].single_redist
)
791 if (gic_data
.redist_stride
) {
792 ptr
+= gic_data
.redist_stride
;
794 ptr
+= SZ_64K
* 2; /* Skip RD_base + SGI_base */
795 if (typer
& GICR_TYPER_VLPIS
)
796 ptr
+= SZ_64K
* 2; /* Skip VLPI_base + reserved page */
798 } while (!(typer
& GICR_TYPER_LAST
));
801 return ret
? -ENODEV
: 0;
804 static int __gic_populate_rdist(struct redist_region
*region
, void __iomem
*ptr
)
806 unsigned long mpidr
= cpu_logical_map(smp_processor_id());
811 * Convert affinity to a 32bit value that can be matched to
812 * GICR_TYPER bits [63:32].
814 aff
= (MPIDR_AFFINITY_LEVEL(mpidr
, 3) << 24 |
815 MPIDR_AFFINITY_LEVEL(mpidr
, 2) << 16 |
816 MPIDR_AFFINITY_LEVEL(mpidr
, 1) << 8 |
817 MPIDR_AFFINITY_LEVEL(mpidr
, 0));
819 typer
= gic_read_typer(ptr
+ GICR_TYPER
);
820 if ((typer
>> 32) == aff
) {
821 u64 offset
= ptr
- region
->redist_base
;
822 gic_data_rdist_rd_base() = ptr
;
823 gic_data_rdist()->phys_base
= region
->phys_base
+ offset
;
825 pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
826 smp_processor_id(), mpidr
,
827 (int)(region
- gic_data
.redist_regions
),
828 &gic_data_rdist()->phys_base
);
836 static int gic_populate_rdist(void)
838 if (gic_iterate_rdists(__gic_populate_rdist
) == 0)
841 /* We couldn't even deal with ourselves... */
842 WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
844 (unsigned long)cpu_logical_map(smp_processor_id()));
848 static int __gic_update_rdist_properties(struct redist_region
*region
,
851 u64 typer
= gic_read_typer(ptr
+ GICR_TYPER
);
852 gic_data
.rdists
.has_vlpis
&= !!(typer
& GICR_TYPER_VLPIS
);
853 gic_data
.rdists
.has_direct_lpi
&= !!(typer
& GICR_TYPER_DirectLPIS
);
854 gic_data
.ppi_nr
= min(GICR_TYPER_NR_PPIS(typer
), gic_data
.ppi_nr
);
859 static void gic_update_rdist_properties(void)
861 gic_data
.ppi_nr
= UINT_MAX
;
862 gic_iterate_rdists(__gic_update_rdist_properties
);
863 if (WARN_ON(gic_data
.ppi_nr
== UINT_MAX
))
865 pr_info("%d PPIs implemented\n", gic_data
.ppi_nr
);
866 pr_info("%sVLPI support, %sdirect LPI support\n",
867 !gic_data
.rdists
.has_vlpis
? "no " : "",
868 !gic_data
.rdists
.has_direct_lpi
? "no " : "");
871 /* Check whether it's single security state view */
872 static inline bool gic_dist_security_disabled(void)
874 return readl_relaxed(gic_data
.dist_base
+ GICD_CTLR
) & GICD_CTLR_DS
;
877 static void gic_cpu_sys_reg_init(void)
879 int i
, cpu
= smp_processor_id();
880 u64 mpidr
= cpu_logical_map(cpu
);
881 u64 need_rss
= MPIDR_RS(mpidr
);
886 * Need to check that the SRE bit has actually been set. If
887 * not, it means that SRE is disabled at EL2. We're going to
888 * die painfully, and there is nothing we can do about it.
890 * Kindly inform the luser.
892 if (!gic_enable_sre())
893 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
895 pribits
= gic_get_pribits();
897 group0
= gic_has_group0();
899 /* Set priority mask register */
900 if (!gic_prio_masking_enabled()) {
901 write_gicreg(DEFAULT_PMR_VALUE
, ICC_PMR_EL1
);
904 * Mismatch configuration with boot CPU, the system is likely
905 * to die as interrupt masking will not work properly on all
908 WARN_ON(gic_supports_nmi() && group0
&&
909 !gic_dist_security_disabled());
913 * Some firmwares hand over to the kernel with the BPR changed from
914 * its reset value (and with a value large enough to prevent
915 * any pre-emptive interrupts from working at all). Writing a zero
916 * to BPR restores is reset value.
920 if (static_branch_likely(&supports_deactivate_key
)) {
921 /* EOI drops priority only (mode 1) */
922 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop
);
924 /* EOI deactivates interrupt too (mode 0) */
925 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir
);
928 /* Always whack Group0 before Group1 */
933 write_gicreg(0, ICC_AP0R3_EL1
);
934 write_gicreg(0, ICC_AP0R2_EL1
);
937 write_gicreg(0, ICC_AP0R1_EL1
);
941 write_gicreg(0, ICC_AP0R0_EL1
);
950 write_gicreg(0, ICC_AP1R3_EL1
);
951 write_gicreg(0, ICC_AP1R2_EL1
);
954 write_gicreg(0, ICC_AP1R1_EL1
);
958 write_gicreg(0, ICC_AP1R0_EL1
);
963 /* ... and let's hit the road... */
966 /* Keep the RSS capability status in per_cpu variable */
967 per_cpu(has_rss
, cpu
) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS
);
969 /* Check all the CPUs have capable of sending SGIs to other CPUs */
970 for_each_online_cpu(i
) {
971 bool have_rss
= per_cpu(has_rss
, i
) && per_cpu(has_rss
, cpu
);
973 need_rss
|= MPIDR_RS(cpu_logical_map(i
));
974 if (need_rss
&& (!have_rss
))
975 pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n",
976 cpu
, (unsigned long)mpidr
,
977 i
, (unsigned long)cpu_logical_map(i
));
981 * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0,
982 * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED
983 * UNPREDICTABLE choice of :
984 * - The write is ignored.
985 * - The RS field is treated as 0.
987 if (need_rss
&& (!gic_data
.has_rss
))
988 pr_crit_once("RSS is required but GICD doesn't support it\n");
991 static bool gicv3_nolpi
;
993 static int __init
gicv3_nolpi_cfg(char *buf
)
995 return strtobool(buf
, &gicv3_nolpi
);
997 early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg
);
999 static int gic_dist_supports_lpis(void)
1001 return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS
) &&
1002 !!(readl_relaxed(gic_data
.dist_base
+ GICD_TYPER
) & GICD_TYPER_LPIS
) &&
1006 static void gic_cpu_init(void)
1008 void __iomem
*rbase
;
1011 /* Register ourselves with the rest of the world */
1012 if (gic_populate_rdist())
1015 gic_enable_redist(true);
1017 WARN((gic_data
.ppi_nr
> 16 || GIC_ESPI_NR
!= 0) &&
1018 !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange
),
1019 "Distributor has extended ranges, but CPU%d doesn't\n",
1020 smp_processor_id());
1022 rbase
= gic_data_rdist_sgi_base();
1024 /* Configure SGIs/PPIs as non-secure Group-1 */
1025 for (i
= 0; i
< gic_data
.ppi_nr
+ 16; i
+= 32)
1026 writel_relaxed(~0, rbase
+ GICR_IGROUPR0
+ i
/ 8);
1028 gic_cpu_config(rbase
, gic_data
.ppi_nr
+ 16, gic_redist_wait_for_rwp
);
1030 /* initialise system registers */
1031 gic_cpu_sys_reg_init();
1036 #define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT)
1037 #define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL)
1039 static int gic_starting_cpu(unsigned int cpu
)
1043 if (gic_dist_supports_lpis())
1049 static u16
gic_compute_target_list(int *base_cpu
, const struct cpumask
*mask
,
1050 unsigned long cluster_id
)
1052 int next_cpu
, cpu
= *base_cpu
;
1053 unsigned long mpidr
= cpu_logical_map(cpu
);
1056 while (cpu
< nr_cpu_ids
) {
1057 tlist
|= 1 << (mpidr
& 0xf);
1059 next_cpu
= cpumask_next(cpu
, mask
);
1060 if (next_cpu
>= nr_cpu_ids
)
1064 mpidr
= cpu_logical_map(cpu
);
1066 if (cluster_id
!= MPIDR_TO_SGI_CLUSTER_ID(mpidr
)) {
1076 #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
1077 (MPIDR_AFFINITY_LEVEL(cluster_id, level) \
1078 << ICC_SGI1R_AFFINITY_## level ##_SHIFT)
1080 static void gic_send_sgi(u64 cluster_id
, u16 tlist
, unsigned int irq
)
1084 val
= (MPIDR_TO_SGI_AFFINITY(cluster_id
, 3) |
1085 MPIDR_TO_SGI_AFFINITY(cluster_id
, 2) |
1086 irq
<< ICC_SGI1R_SGI_ID_SHIFT
|
1087 MPIDR_TO_SGI_AFFINITY(cluster_id
, 1) |
1088 MPIDR_TO_SGI_RS(cluster_id
) |
1089 tlist
<< ICC_SGI1R_TARGET_LIST_SHIFT
);
1091 pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val
);
1092 gic_write_sgi1r(val
);
1095 static void gic_raise_softirq(const struct cpumask
*mask
, unsigned int irq
)
1099 if (WARN_ON(irq
>= 16))
1103 * Ensure that stores to Normal memory are visible to the
1104 * other CPUs before issuing the IPI.
1108 for_each_cpu(cpu
, mask
) {
1109 u64 cluster_id
= MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu
));
1112 tlist
= gic_compute_target_list(&cpu
, mask
, cluster_id
);
1113 gic_send_sgi(cluster_id
, tlist
, irq
);
1116 /* Force the above writes to ICC_SGI1R_EL1 to be executed */
1120 static void gic_smp_init(void)
1122 set_smp_cross_call(gic_raise_softirq
);
1123 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING
,
1124 "irqchip/arm/gicv3:starting",
1125 gic_starting_cpu
, NULL
);
1128 static int gic_set_affinity(struct irq_data
*d
, const struct cpumask
*mask_val
,
1138 cpu
= cpumask_first(mask_val
);
1140 cpu
= cpumask_any_and(mask_val
, cpu_online_mask
);
1142 if (cpu
>= nr_cpu_ids
)
1145 if (gic_irq_in_rdist(d
))
1148 /* If interrupt was enabled, disable it first */
1149 enabled
= gic_peek_irq(d
, GICD_ISENABLER
);
1153 offset
= convert_offset_index(d
, GICD_IROUTER
, &index
);
1154 reg
= gic_dist_base(d
) + offset
+ (index
* 8);
1155 val
= gic_mpidr_to_affinity(cpu_logical_map(cpu
));
1157 gic_write_irouter(val
, reg
);
1160 * If the interrupt was enabled, enabled it again. Otherwise,
1161 * just wait for the distributor to have digested our changes.
1166 gic_dist_wait_for_rwp();
1168 irq_data_update_effective_affinity(d
, cpumask_of(cpu
));
1170 return IRQ_SET_MASK_OK_DONE
;
1173 #define gic_set_affinity NULL
1174 #define gic_smp_init() do { } while(0)
1177 #ifdef CONFIG_CPU_PM
1178 static int gic_cpu_pm_notifier(struct notifier_block
*self
,
1179 unsigned long cmd
, void *v
)
1181 if (cmd
== CPU_PM_EXIT
) {
1182 if (gic_dist_security_disabled())
1183 gic_enable_redist(true);
1184 gic_cpu_sys_reg_init();
1185 } else if (cmd
== CPU_PM_ENTER
&& gic_dist_security_disabled()) {
1186 gic_write_grpen1(0);
1187 gic_enable_redist(false);
1192 static struct notifier_block gic_cpu_pm_notifier_block
= {
1193 .notifier_call
= gic_cpu_pm_notifier
,
1196 static void gic_cpu_pm_init(void)
1198 cpu_pm_register_notifier(&gic_cpu_pm_notifier_block
);
1202 static inline void gic_cpu_pm_init(void) { }
1203 #endif /* CONFIG_CPU_PM */
1205 static struct irq_chip gic_chip
= {
1207 .irq_mask
= gic_mask_irq
,
1208 .irq_unmask
= gic_unmask_irq
,
1209 .irq_eoi
= gic_eoi_irq
,
1210 .irq_set_type
= gic_set_type
,
1211 .irq_set_affinity
= gic_set_affinity
,
1212 .irq_get_irqchip_state
= gic_irq_get_irqchip_state
,
1213 .irq_set_irqchip_state
= gic_irq_set_irqchip_state
,
1214 .irq_nmi_setup
= gic_irq_nmi_setup
,
1215 .irq_nmi_teardown
= gic_irq_nmi_teardown
,
1216 .flags
= IRQCHIP_SET_TYPE_MASKED
|
1217 IRQCHIP_SKIP_SET_WAKE
|
1218 IRQCHIP_MASK_ON_SUSPEND
,
1221 static struct irq_chip gic_eoimode1_chip
= {
1223 .irq_mask
= gic_eoimode1_mask_irq
,
1224 .irq_unmask
= gic_unmask_irq
,
1225 .irq_eoi
= gic_eoimode1_eoi_irq
,
1226 .irq_set_type
= gic_set_type
,
1227 .irq_set_affinity
= gic_set_affinity
,
1228 .irq_get_irqchip_state
= gic_irq_get_irqchip_state
,
1229 .irq_set_irqchip_state
= gic_irq_set_irqchip_state
,
1230 .irq_set_vcpu_affinity
= gic_irq_set_vcpu_affinity
,
1231 .irq_nmi_setup
= gic_irq_nmi_setup
,
1232 .irq_nmi_teardown
= gic_irq_nmi_teardown
,
1233 .flags
= IRQCHIP_SET_TYPE_MASKED
|
1234 IRQCHIP_SKIP_SET_WAKE
|
1235 IRQCHIP_MASK_ON_SUSPEND
,
1238 static int gic_irq_domain_map(struct irq_domain
*d
, unsigned int irq
,
1241 struct irq_chip
*chip
= &gic_chip
;
1243 if (static_branch_likely(&supports_deactivate_key
))
1244 chip
= &gic_eoimode1_chip
;
1246 switch (__get_intid_range(hw
)) {
1249 irq_set_percpu_devid(irq
);
1250 irq_domain_set_info(d
, irq
, hw
, chip
, d
->host_data
,
1251 handle_percpu_devid_irq
, NULL
, NULL
);
1252 irq_set_status_flags(irq
, IRQ_NOAUTOEN
);
1257 irq_domain_set_info(d
, irq
, hw
, chip
, d
->host_data
,
1258 handle_fasteoi_irq
, NULL
, NULL
);
1260 irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq
)));
1264 if (!gic_dist_supports_lpis())
1266 irq_domain_set_info(d
, irq
, hw
, chip
, d
->host_data
,
1267 handle_fasteoi_irq
, NULL
, NULL
);
1277 #define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1)
1279 static int gic_irq_domain_translate(struct irq_domain
*d
,
1280 struct irq_fwspec
*fwspec
,
1281 unsigned long *hwirq
,
1284 if (is_of_node(fwspec
->fwnode
)) {
1285 if (fwspec
->param_count
< 3)
1288 switch (fwspec
->param
[0]) {
1290 *hwirq
= fwspec
->param
[1] + 32;
1293 *hwirq
= fwspec
->param
[1] + 16;
1296 *hwirq
= fwspec
->param
[1] + ESPI_BASE_INTID
;
1299 *hwirq
= fwspec
->param
[1] + EPPI_BASE_INTID
;
1301 case GIC_IRQ_TYPE_LPI
: /* LPI */
1302 *hwirq
= fwspec
->param
[1];
1304 case GIC_IRQ_TYPE_PARTITION
:
1305 *hwirq
= fwspec
->param
[1];
1306 if (fwspec
->param
[1] >= 16)
1307 *hwirq
+= EPPI_BASE_INTID
- 16;
1315 *type
= fwspec
->param
[2] & IRQ_TYPE_SENSE_MASK
;
1318 * Make it clear that broken DTs are... broken.
1319 * Partitionned PPIs are an unfortunate exception.
1321 WARN_ON(*type
== IRQ_TYPE_NONE
&&
1322 fwspec
->param
[0] != GIC_IRQ_TYPE_PARTITION
);
1326 if (is_fwnode_irqchip(fwspec
->fwnode
)) {
1327 if(fwspec
->param_count
!= 2)
1330 *hwirq
= fwspec
->param
[0];
1331 *type
= fwspec
->param
[1];
1333 WARN_ON(*type
== IRQ_TYPE_NONE
);
1340 static int gic_irq_domain_alloc(struct irq_domain
*domain
, unsigned int virq
,
1341 unsigned int nr_irqs
, void *arg
)
1344 irq_hw_number_t hwirq
;
1345 unsigned int type
= IRQ_TYPE_NONE
;
1346 struct irq_fwspec
*fwspec
= arg
;
1348 ret
= gic_irq_domain_translate(domain
, fwspec
, &hwirq
, &type
);
1352 for (i
= 0; i
< nr_irqs
; i
++) {
1353 ret
= gic_irq_domain_map(domain
, virq
+ i
, hwirq
+ i
);
1361 static void gic_irq_domain_free(struct irq_domain
*domain
, unsigned int virq
,
1362 unsigned int nr_irqs
)
1366 for (i
= 0; i
< nr_irqs
; i
++) {
1367 struct irq_data
*d
= irq_domain_get_irq_data(domain
, virq
+ i
);
1368 irq_set_handler(virq
+ i
, NULL
);
1369 irq_domain_reset_irq_data(d
);
1373 static int gic_irq_domain_select(struct irq_domain
*d
,
1374 struct irq_fwspec
*fwspec
,
1375 enum irq_domain_bus_token bus_token
)
1378 if (fwspec
->fwnode
!= d
->fwnode
)
1381 /* If this is not DT, then we have a single domain */
1382 if (!is_of_node(fwspec
->fwnode
))
1386 * If this is a PPI and we have a 4th (non-null) parameter,
1387 * then we need to match the partition domain.
1389 if (fwspec
->param_count
>= 4 &&
1390 fwspec
->param
[0] == 1 && fwspec
->param
[3] != 0 &&
1392 return d
== partition_get_domain(gic_data
.ppi_descs
[fwspec
->param
[1]]);
1394 return d
== gic_data
.domain
;
1397 static const struct irq_domain_ops gic_irq_domain_ops
= {
1398 .translate
= gic_irq_domain_translate
,
1399 .alloc
= gic_irq_domain_alloc
,
1400 .free
= gic_irq_domain_free
,
1401 .select
= gic_irq_domain_select
,
1404 static int partition_domain_translate(struct irq_domain
*d
,
1405 struct irq_fwspec
*fwspec
,
1406 unsigned long *hwirq
,
1409 struct device_node
*np
;
1412 if (!gic_data
.ppi_descs
)
1415 np
= of_find_node_by_phandle(fwspec
->param
[3]);
1419 ret
= partition_translate_id(gic_data
.ppi_descs
[fwspec
->param
[1]],
1420 of_node_to_fwnode(np
));
1425 *type
= fwspec
->param
[2] & IRQ_TYPE_SENSE_MASK
;
1430 static const struct irq_domain_ops partition_domain_ops
= {
1431 .translate
= partition_domain_translate
,
1432 .select
= gic_irq_domain_select
,
1435 static bool gic_enable_quirk_msm8996(void *data
)
1437 struct gic_chip_data
*d
= data
;
1439 d
->flags
|= FLAGS_WORKAROUND_GICR_WAKER_MSM8996
;
1444 static bool gic_enable_quirk_hip06_07(void *data
)
1446 struct gic_chip_data
*d
= data
;
1449 * HIP06 GICD_IIDR clashes with GIC-600 product number (despite
1450 * not being an actual ARM implementation). The saving grace is
1451 * that GIC-600 doesn't have ESPI, so nothing to do in that case.
1452 * HIP07 doesn't even have a proper IIDR, and still pretends to
1453 * have ESPI. In both cases, put them right.
1455 if (d
->rdists
.gicd_typer
& GICD_TYPER_ESPI
) {
1456 /* Zero both ESPI and the RES0 field next to it... */
1457 d
->rdists
.gicd_typer
&= ~GENMASK(9, 8);
1464 static const struct gic_quirk gic_quirks
[] = {
1466 .desc
= "GICv3: Qualcomm MSM8996 broken firmware",
1467 .compatible
= "qcom,msm8996-gic-v3",
1468 .init
= gic_enable_quirk_msm8996
,
1471 .desc
= "GICv3: HIP06 erratum 161010803",
1474 .init
= gic_enable_quirk_hip06_07
,
1477 .desc
= "GICv3: HIP07 erratum 161010803",
1480 .init
= gic_enable_quirk_hip06_07
,
1486 static void gic_enable_nmi_support(void)
1490 if (!gic_prio_masking_enabled())
1493 if (gic_has_group0() && !gic_dist_security_disabled()) {
1494 pr_warn("SCR_EL3.FIQ is cleared, cannot enable use of pseudo-NMIs\n");
1498 ppi_nmi_refs
= kcalloc(gic_data
.ppi_nr
, sizeof(*ppi_nmi_refs
), GFP_KERNEL
);
1502 for (i
= 0; i
< gic_data
.ppi_nr
; i
++)
1503 refcount_set(&ppi_nmi_refs
[i
], 0);
1505 static_branch_enable(&supports_pseudo_nmis
);
1507 if (static_branch_likely(&supports_deactivate_key
))
1508 gic_eoimode1_chip
.flags
|= IRQCHIP_SUPPORTS_NMI
;
1510 gic_chip
.flags
|= IRQCHIP_SUPPORTS_NMI
;
1513 static int __init
gic_init_bases(void __iomem
*dist_base
,
1514 struct redist_region
*rdist_regs
,
1515 u32 nr_redist_regions
,
1517 struct fwnode_handle
*handle
)
1522 if (!is_hyp_mode_available())
1523 static_branch_disable(&supports_deactivate_key
);
1525 if (static_branch_likely(&supports_deactivate_key
))
1526 pr_info("GIC: Using split EOI/Deactivate mode\n");
1528 gic_data
.fwnode
= handle
;
1529 gic_data
.dist_base
= dist_base
;
1530 gic_data
.redist_regions
= rdist_regs
;
1531 gic_data
.nr_redist_regions
= nr_redist_regions
;
1532 gic_data
.redist_stride
= redist_stride
;
1535 * Find out how many interrupts are supported.
1537 typer
= readl_relaxed(gic_data
.dist_base
+ GICD_TYPER
);
1538 gic_data
.rdists
.gicd_typer
= typer
;
1540 gic_enable_quirks(readl_relaxed(gic_data
.dist_base
+ GICD_IIDR
),
1541 gic_quirks
, &gic_data
);
1543 pr_info("%d SPIs implemented\n", GIC_LINE_NR
- 32);
1544 pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR
);
1545 gic_data
.domain
= irq_domain_create_tree(handle
, &gic_irq_domain_ops
,
1547 irq_domain_update_bus_token(gic_data
.domain
, DOMAIN_BUS_WIRED
);
1548 gic_data
.rdists
.rdist
= alloc_percpu(typeof(*gic_data
.rdists
.rdist
));
1549 gic_data
.rdists
.has_vlpis
= true;
1550 gic_data
.rdists
.has_direct_lpi
= true;
1552 if (WARN_ON(!gic_data
.domain
) || WARN_ON(!gic_data
.rdists
.rdist
)) {
1557 gic_data
.has_rss
= !!(typer
& GICD_TYPER_RSS
);
1558 pr_info("Distributor has %sRange Selector support\n",
1559 gic_data
.has_rss
? "" : "no ");
1561 if (typer
& GICD_TYPER_MBIS
) {
1562 err
= mbi_init(handle
, gic_data
.domain
);
1564 pr_err("Failed to initialize MBIs\n");
1567 set_handle_irq(gic_handle_irq
);
1569 gic_update_rdist_properties();
1576 if (gic_dist_supports_lpis()) {
1577 its_init(handle
, &gic_data
.rdists
, gic_data
.domain
);
1580 if (IS_ENABLED(CONFIG_ARM_GIC_V2M
))
1581 gicv2m_init(handle
, gic_data
.domain
);
1584 gic_enable_nmi_support();
1589 if (gic_data
.domain
)
1590 irq_domain_remove(gic_data
.domain
);
1591 free_percpu(gic_data
.rdists
.rdist
);
1595 static int __init
gic_validate_dist_version(void __iomem
*dist_base
)
1597 u32 reg
= readl_relaxed(dist_base
+ GICD_PIDR2
) & GIC_PIDR2_ARCH_MASK
;
1599 if (reg
!= GIC_PIDR2_ARCH_GICv3
&& reg
!= GIC_PIDR2_ARCH_GICv4
)
1605 /* Create all possible partitions at boot time */
1606 static void __init
gic_populate_ppi_partitions(struct device_node
*gic_node
)
1608 struct device_node
*parts_node
, *child_part
;
1609 int part_idx
= 0, i
;
1611 struct partition_affinity
*parts
;
1613 parts_node
= of_get_child_by_name(gic_node
, "ppi-partitions");
1617 gic_data
.ppi_descs
= kcalloc(gic_data
.ppi_nr
, sizeof(*gic_data
.ppi_descs
), GFP_KERNEL
);
1618 if (!gic_data
.ppi_descs
)
1621 nr_parts
= of_get_child_count(parts_node
);
1626 parts
= kcalloc(nr_parts
, sizeof(*parts
), GFP_KERNEL
);
1627 if (WARN_ON(!parts
))
1630 for_each_child_of_node(parts_node
, child_part
) {
1631 struct partition_affinity
*part
;
1634 part
= &parts
[part_idx
];
1636 part
->partition_id
= of_node_to_fwnode(child_part
);
1638 pr_info("GIC: PPI partition %pOFn[%d] { ",
1639 child_part
, part_idx
);
1641 n
= of_property_count_elems_of_size(child_part
, "affinity",
1645 for (i
= 0; i
< n
; i
++) {
1648 struct device_node
*cpu_node
;
1650 err
= of_property_read_u32_index(child_part
, "affinity",
1655 cpu_node
= of_find_node_by_phandle(cpu_phandle
);
1656 if (WARN_ON(!cpu_node
))
1659 cpu
= of_cpu_node_to_id(cpu_node
);
1660 if (WARN_ON(cpu
< 0))
1663 pr_cont("%pOF[%d] ", cpu_node
, cpu
);
1665 cpumask_set_cpu(cpu
, &part
->mask
);
1672 for (i
= 0; i
< gic_data
.ppi_nr
; i
++) {
1674 struct partition_desc
*desc
;
1675 struct irq_fwspec ppi_fwspec
= {
1676 .fwnode
= gic_data
.fwnode
,
1679 [0] = GIC_IRQ_TYPE_PARTITION
,
1681 [2] = IRQ_TYPE_NONE
,
1685 irq
= irq_create_fwspec_mapping(&ppi_fwspec
);
1688 desc
= partition_create_desc(gic_data
.fwnode
, parts
, nr_parts
,
1689 irq
, &partition_domain_ops
);
1693 gic_data
.ppi_descs
[i
] = desc
;
1697 of_node_put(parts_node
);
1700 static void __init
gic_of_setup_kvm_info(struct device_node
*node
)
1706 gic_v3_kvm_info
.type
= GIC_V3
;
1708 gic_v3_kvm_info
.maint_irq
= irq_of_parse_and_map(node
, 0);
1709 if (!gic_v3_kvm_info
.maint_irq
)
1712 if (of_property_read_u32(node
, "#redistributor-regions",
1716 gicv_idx
+= 3; /* Also skip GICD, GICC, GICH */
1717 ret
= of_address_to_resource(node
, gicv_idx
, &r
);
1719 gic_v3_kvm_info
.vcpu
= r
;
1721 gic_v3_kvm_info
.has_v4
= gic_data
.rdists
.has_vlpis
;
1722 gic_set_kvm_info(&gic_v3_kvm_info
);
1725 static int __init
gic_of_init(struct device_node
*node
, struct device_node
*parent
)
1727 void __iomem
*dist_base
;
1728 struct redist_region
*rdist_regs
;
1730 u32 nr_redist_regions
;
1733 dist_base
= of_iomap(node
, 0);
1735 pr_err("%pOF: unable to map gic dist registers\n", node
);
1739 err
= gic_validate_dist_version(dist_base
);
1741 pr_err("%pOF: no distributor detected, giving up\n", node
);
1742 goto out_unmap_dist
;
1745 if (of_property_read_u32(node
, "#redistributor-regions", &nr_redist_regions
))
1746 nr_redist_regions
= 1;
1748 rdist_regs
= kcalloc(nr_redist_regions
, sizeof(*rdist_regs
),
1752 goto out_unmap_dist
;
1755 for (i
= 0; i
< nr_redist_regions
; i
++) {
1756 struct resource res
;
1759 ret
= of_address_to_resource(node
, 1 + i
, &res
);
1760 rdist_regs
[i
].redist_base
= of_iomap(node
, 1 + i
);
1761 if (ret
|| !rdist_regs
[i
].redist_base
) {
1762 pr_err("%pOF: couldn't map region %d\n", node
, i
);
1764 goto out_unmap_rdist
;
1766 rdist_regs
[i
].phys_base
= res
.start
;
1769 if (of_property_read_u64(node
, "redistributor-stride", &redist_stride
))
1772 gic_enable_of_quirks(node
, gic_quirks
, &gic_data
);
1774 err
= gic_init_bases(dist_base
, rdist_regs
, nr_redist_regions
,
1775 redist_stride
, &node
->fwnode
);
1777 goto out_unmap_rdist
;
1779 gic_populate_ppi_partitions(node
);
1781 if (static_branch_likely(&supports_deactivate_key
))
1782 gic_of_setup_kvm_info(node
);
1786 for (i
= 0; i
< nr_redist_regions
; i
++)
1787 if (rdist_regs
[i
].redist_base
)
1788 iounmap(rdist_regs
[i
].redist_base
);
1795 IRQCHIP_DECLARE(gic_v3
, "arm,gic-v3", gic_of_init
);
1800 void __iomem
*dist_base
;
1801 struct redist_region
*redist_regs
;
1802 u32 nr_redist_regions
;
1806 phys_addr_t vcpu_base
;
1807 } acpi_data __initdata
;
1810 gic_acpi_register_redist(phys_addr_t phys_base
, void __iomem
*redist_base
)
1812 static int count
= 0;
1814 acpi_data
.redist_regs
[count
].phys_base
= phys_base
;
1815 acpi_data
.redist_regs
[count
].redist_base
= redist_base
;
1816 acpi_data
.redist_regs
[count
].single_redist
= acpi_data
.single_redist
;
1821 gic_acpi_parse_madt_redist(union acpi_subtable_headers
*header
,
1822 const unsigned long end
)
1824 struct acpi_madt_generic_redistributor
*redist
=
1825 (struct acpi_madt_generic_redistributor
*)header
;
1826 void __iomem
*redist_base
;
1828 redist_base
= ioremap(redist
->base_address
, redist
->length
);
1830 pr_err("Couldn't map GICR region @%llx\n", redist
->base_address
);
1834 gic_acpi_register_redist(redist
->base_address
, redist_base
);
1839 gic_acpi_parse_madt_gicc(union acpi_subtable_headers
*header
,
1840 const unsigned long end
)
1842 struct acpi_madt_generic_interrupt
*gicc
=
1843 (struct acpi_madt_generic_interrupt
*)header
;
1844 u32 reg
= readl_relaxed(acpi_data
.dist_base
+ GICD_PIDR2
) & GIC_PIDR2_ARCH_MASK
;
1845 u32 size
= reg
== GIC_PIDR2_ARCH_GICv4
? SZ_64K
* 4 : SZ_64K
* 2;
1846 void __iomem
*redist_base
;
1848 /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */
1849 if (!(gicc
->flags
& ACPI_MADT_ENABLED
))
1852 redist_base
= ioremap(gicc
->gicr_base_address
, size
);
1856 gic_acpi_register_redist(gicc
->gicr_base_address
, redist_base
);
1860 static int __init
gic_acpi_collect_gicr_base(void)
1862 acpi_tbl_entry_handler redist_parser
;
1863 enum acpi_madt_type type
;
1865 if (acpi_data
.single_redist
) {
1866 type
= ACPI_MADT_TYPE_GENERIC_INTERRUPT
;
1867 redist_parser
= gic_acpi_parse_madt_gicc
;
1869 type
= ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR
;
1870 redist_parser
= gic_acpi_parse_madt_redist
;
1873 /* Collect redistributor base addresses in GICR entries */
1874 if (acpi_table_parse_madt(type
, redist_parser
, 0) > 0)
1877 pr_info("No valid GICR entries exist\n");
1881 static int __init
gic_acpi_match_gicr(union acpi_subtable_headers
*header
,
1882 const unsigned long end
)
1884 /* Subtable presence means that redist exists, that's it */
1888 static int __init
gic_acpi_match_gicc(union acpi_subtable_headers
*header
,
1889 const unsigned long end
)
1891 struct acpi_madt_generic_interrupt
*gicc
=
1892 (struct acpi_madt_generic_interrupt
*)header
;
1895 * If GICC is enabled and has valid gicr base address, then it means
1896 * GICR base is presented via GICC
1898 if ((gicc
->flags
& ACPI_MADT_ENABLED
) && gicc
->gicr_base_address
)
1902 * It's perfectly valid firmware can pass disabled GICC entry, driver
1903 * should not treat as errors, skip the entry instead of probe fail.
1905 if (!(gicc
->flags
& ACPI_MADT_ENABLED
))
1911 static int __init
gic_acpi_count_gicr_regions(void)
1916 * Count how many redistributor regions we have. It is not allowed
1917 * to mix redistributor description, GICR and GICC subtables have to be
1918 * mutually exclusive.
1920 count
= acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR
,
1921 gic_acpi_match_gicr
, 0);
1923 acpi_data
.single_redist
= false;
1927 count
= acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT
,
1928 gic_acpi_match_gicc
, 0);
1930 acpi_data
.single_redist
= true;
1935 static bool __init
acpi_validate_gic_table(struct acpi_subtable_header
*header
,
1936 struct acpi_probe_entry
*ape
)
1938 struct acpi_madt_generic_distributor
*dist
;
1941 dist
= (struct acpi_madt_generic_distributor
*)header
;
1942 if (dist
->version
!= ape
->driver_data
)
1945 /* We need to do that exercise anyway, the sooner the better */
1946 count
= gic_acpi_count_gicr_regions();
1950 acpi_data
.nr_redist_regions
= count
;
1954 static int __init
gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers
*header
,
1955 const unsigned long end
)
1957 struct acpi_madt_generic_interrupt
*gicc
=
1958 (struct acpi_madt_generic_interrupt
*)header
;
1960 static int first_madt
= true;
1962 /* Skip unusable CPUs */
1963 if (!(gicc
->flags
& ACPI_MADT_ENABLED
))
1966 maint_irq_mode
= (gicc
->flags
& ACPI_MADT_VGIC_IRQ_MODE
) ?
1967 ACPI_EDGE_SENSITIVE
: ACPI_LEVEL_SENSITIVE
;
1972 acpi_data
.maint_irq
= gicc
->vgic_interrupt
;
1973 acpi_data
.maint_irq_mode
= maint_irq_mode
;
1974 acpi_data
.vcpu_base
= gicc
->gicv_base_address
;
1980 * The maintenance interrupt and GICV should be the same for every CPU
1982 if ((acpi_data
.maint_irq
!= gicc
->vgic_interrupt
) ||
1983 (acpi_data
.maint_irq_mode
!= maint_irq_mode
) ||
1984 (acpi_data
.vcpu_base
!= gicc
->gicv_base_address
))
1990 static bool __init
gic_acpi_collect_virt_info(void)
1994 count
= acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT
,
1995 gic_acpi_parse_virt_madt_gicc
, 0);
2000 #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K)
2001 #define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K)
2002 #define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K)
2004 static void __init
gic_acpi_setup_kvm_info(void)
2008 if (!gic_acpi_collect_virt_info()) {
2009 pr_warn("Unable to get hardware information used for virtualization\n");
2013 gic_v3_kvm_info
.type
= GIC_V3
;
2015 irq
= acpi_register_gsi(NULL
, acpi_data
.maint_irq
,
2016 acpi_data
.maint_irq_mode
,
2021 gic_v3_kvm_info
.maint_irq
= irq
;
2023 if (acpi_data
.vcpu_base
) {
2024 struct resource
*vcpu
= &gic_v3_kvm_info
.vcpu
;
2026 vcpu
->flags
= IORESOURCE_MEM
;
2027 vcpu
->start
= acpi_data
.vcpu_base
;
2028 vcpu
->end
= vcpu
->start
+ ACPI_GICV2_VCPU_MEM_SIZE
- 1;
2031 gic_v3_kvm_info
.has_v4
= gic_data
.rdists
.has_vlpis
;
2032 gic_set_kvm_info(&gic_v3_kvm_info
);
2036 gic_acpi_init(struct acpi_subtable_header
*header
, const unsigned long end
)
2038 struct acpi_madt_generic_distributor
*dist
;
2039 struct fwnode_handle
*domain_handle
;
2043 /* Get distributor base address */
2044 dist
= (struct acpi_madt_generic_distributor
*)header
;
2045 acpi_data
.dist_base
= ioremap(dist
->base_address
,
2046 ACPI_GICV3_DIST_MEM_SIZE
);
2047 if (!acpi_data
.dist_base
) {
2048 pr_err("Unable to map GICD registers\n");
2052 err
= gic_validate_dist_version(acpi_data
.dist_base
);
2054 pr_err("No distributor detected at @%p, giving up\n",
2055 acpi_data
.dist_base
);
2056 goto out_dist_unmap
;
2059 size
= sizeof(*acpi_data
.redist_regs
) * acpi_data
.nr_redist_regions
;
2060 acpi_data
.redist_regs
= kzalloc(size
, GFP_KERNEL
);
2061 if (!acpi_data
.redist_regs
) {
2063 goto out_dist_unmap
;
2066 err
= gic_acpi_collect_gicr_base();
2068 goto out_redist_unmap
;
2070 domain_handle
= irq_domain_alloc_fwnode(&dist
->base_address
);
2071 if (!domain_handle
) {
2073 goto out_redist_unmap
;
2076 err
= gic_init_bases(acpi_data
.dist_base
, acpi_data
.redist_regs
,
2077 acpi_data
.nr_redist_regions
, 0, domain_handle
);
2079 goto out_fwhandle_free
;
2081 acpi_set_irq_model(ACPI_IRQ_MODEL_GIC
, domain_handle
);
2083 if (static_branch_likely(&supports_deactivate_key
))
2084 gic_acpi_setup_kvm_info();
2089 irq_domain_free_fwnode(domain_handle
);
2091 for (i
= 0; i
< acpi_data
.nr_redist_regions
; i
++)
2092 if (acpi_data
.redist_regs
[i
].redist_base
)
2093 iounmap(acpi_data
.redist_regs
[i
].redist_base
);
2094 kfree(acpi_data
.redist_regs
);
2096 iounmap(acpi_data
.dist_base
);
2099 IRQCHIP_ACPI_DECLARE(gic_v3
, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR
,
2100 acpi_validate_gic_table
, ACPI_MADT_GIC_VERSION_V3
,
2102 IRQCHIP_ACPI_DECLARE(gic_v4
, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR
,
2103 acpi_validate_gic_table
, ACPI_MADT_GIC_VERSION_V4
,
2105 IRQCHIP_ACPI_DECLARE(gic_v3_or_v4
, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR
,
2106 acpi_validate_gic_table
, ACPI_MADT_GIC_VERSION_NONE
,