1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
7 #include <linux/acpi.h>
8 #include <linux/acpi_iort.h>
9 #include <linux/bitfield.h>
10 #include <linux/bitmap.h>
11 #include <linux/cpu.h>
12 #include <linux/crash_dump.h>
13 #include <linux/delay.h>
14 #include <linux/efi.h>
15 #include <linux/interrupt.h>
16 #include <linux/iommu.h>
17 #include <linux/iopoll.h>
18 #include <linux/irqdomain.h>
19 #include <linux/list.h>
20 #include <linux/log2.h>
21 #include <linux/memblock.h>
23 #include <linux/msi.h>
25 #include <linux/of_address.h>
26 #include <linux/of_irq.h>
27 #include <linux/of_pci.h>
28 #include <linux/of_platform.h>
29 #include <linux/percpu.h>
30 #include <linux/slab.h>
31 #include <linux/syscore_ops.h>
33 #include <linux/irqchip.h>
34 #include <linux/irqchip/arm-gic-v3.h>
35 #include <linux/irqchip/arm-gic-v4.h>
37 #include <asm/cputype.h>
38 #include <asm/exception.h>
40 #include "irq-gic-common.h"
42 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0)
43 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1)
44 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2)
45 #define ITS_FLAGS_FORCE_NON_SHAREABLE (1ULL << 3)
47 #define RD_LOCAL_LPI_ENABLED BIT(0)
48 #define RD_LOCAL_PENDTABLE_PREALLOCATED BIT(1)
49 #define RD_LOCAL_MEMRESERVE_DONE BIT(2)
51 static u32 lpi_id_bits
;
54 * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
55 * deal with (one configuration byte per interrupt). PENDBASE has to
56 * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
58 #define LPI_NRBITS lpi_id_bits
59 #define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K)
60 #define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
62 #define LPI_PROP_DEFAULT_PRIO GICD_INT_DEF_PRI
65 * Collection structure - just an ID, and a redistributor address to
66 * ping. We use one per CPU as a bag of interrupts assigned to this
69 struct its_collection
{
75 * The ITS_BASER structure - contains memory information, cached
76 * value of BASER register configuration and ITS page size.
88 * The ITS structure - contains most of the infrastructure, with the
89 * top-level MSI domain, the command queue, the collections, and the
90 * list of devices writing to it.
92 * dev_alloc_lock has to be taken for device allocations, while the
93 * spinlock must be taken to parse data structures such as the device
98 struct mutex dev_alloc_lock
;
99 struct list_head entry
;
101 void __iomem
*sgir_base
;
102 phys_addr_t phys_base
;
103 struct its_cmd_block
*cmd_base
;
104 struct its_cmd_block
*cmd_write
;
105 struct its_baser tables
[GITS_BASER_NR_REGS
];
106 struct its_collection
*collections
;
107 struct fwnode_handle
*fwnode_handle
;
108 u64 (*get_msi_base
)(struct its_device
*its_dev
);
113 struct list_head its_device_list
;
115 unsigned long list_nr
;
117 unsigned int msi_domain_flags
;
118 u32 pre_its_base
; /* for Socionext Synquacer */
119 int vlpi_redist_offset
;
122 #define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS))
123 #define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP))
124 #define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
126 #define ITS_ITT_ALIGN SZ_256
128 /* The maximum number of VPEID bits supported by VLPI commands */
129 #define ITS_MAX_VPEID_BITS \
132 if (gic_rdists->has_rvpeid && \
133 gic_rdists->gicd_typer2 & GICD_TYPER2_VIL) \
134 nvpeid = 1 + (gic_rdists->gicd_typer2 & \
139 #define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS))
141 /* Convert page order to size in bytes */
142 #define PAGE_ORDER_TO_SIZE(o) (PAGE_SIZE << (o))
144 struct event_lpi_map
{
145 unsigned long *lpi_map
;
147 irq_hw_number_t lpi_base
;
149 raw_spinlock_t vlpi_lock
;
151 struct its_vlpi_map
*vlpi_maps
;
156 * The ITS view of a device - belongs to an ITS, owns an interrupt
157 * translation table, and a list of interrupts. If it some of its
158 * LPIs are injected into a guest (GICv4), the event_map.vm field
159 * indicates which one.
162 struct list_head entry
;
163 struct its_node
*its
;
164 struct event_lpi_map event_map
;
173 struct its_device
*dev
;
174 struct its_vpe
**vpes
;
178 struct cpu_lpi_count
{
183 static DEFINE_PER_CPU(struct cpu_lpi_count
, cpu_lpi_count
);
185 static LIST_HEAD(its_nodes
);
186 static DEFINE_RAW_SPINLOCK(its_lock
);
187 static struct rdists
*gic_rdists
;
188 static struct irq_domain
*its_parent
;
190 static unsigned long its_list_map
;
191 static u16 vmovp_seq_num
;
192 static DEFINE_RAW_SPINLOCK(vmovp_lock
);
194 static DEFINE_IDA(its_vpeid_ida
);
196 #define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist))
197 #define gic_data_rdist_cpu(cpu) (per_cpu_ptr(gic_rdists->rdist, cpu))
198 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
199 #define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K)
202 * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
203 * always have vSGIs mapped.
205 static bool require_its_list_vmovp(struct its_vm
*vm
, struct its_node
*its
)
207 return (gic_rdists
->has_rvpeid
|| vm
->vlpi_count
[its
->list_nr
]);
210 static u16
get_its_list(struct its_vm
*vm
)
212 struct its_node
*its
;
213 unsigned long its_list
= 0;
215 list_for_each_entry(its
, &its_nodes
, entry
) {
219 if (require_its_list_vmovp(vm
, its
))
220 __set_bit(its
->list_nr
, &its_list
);
223 return (u16
)its_list
;
226 static inline u32
its_get_event_id(struct irq_data
*d
)
228 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
229 return d
->hwirq
- its_dev
->event_map
.lpi_base
;
232 static struct its_collection
*dev_event_to_col(struct its_device
*its_dev
,
235 struct its_node
*its
= its_dev
->its
;
237 return its
->collections
+ its_dev
->event_map
.col_map
[event
];
240 static struct its_vlpi_map
*dev_event_to_vlpi_map(struct its_device
*its_dev
,
243 if (WARN_ON_ONCE(event
>= its_dev
->event_map
.nr_lpis
))
246 return &its_dev
->event_map
.vlpi_maps
[event
];
249 static struct its_vlpi_map
*get_vlpi_map(struct irq_data
*d
)
251 if (irqd_is_forwarded_to_vcpu(d
)) {
252 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
253 u32 event
= its_get_event_id(d
);
255 return dev_event_to_vlpi_map(its_dev
, event
);
261 static int vpe_to_cpuid_lock(struct its_vpe
*vpe
, unsigned long *flags
)
263 raw_spin_lock_irqsave(&vpe
->vpe_lock
, *flags
);
267 static void vpe_to_cpuid_unlock(struct its_vpe
*vpe
, unsigned long flags
)
269 raw_spin_unlock_irqrestore(&vpe
->vpe_lock
, flags
);
272 static struct irq_chip its_vpe_irq_chip
;
274 static int irq_to_cpuid_lock(struct irq_data
*d
, unsigned long *flags
)
276 struct its_vpe
*vpe
= NULL
;
279 if (d
->chip
== &its_vpe_irq_chip
) {
280 vpe
= irq_data_get_irq_chip_data(d
);
282 struct its_vlpi_map
*map
= get_vlpi_map(d
);
288 cpu
= vpe_to_cpuid_lock(vpe
, flags
);
290 /* Physical LPIs are already locked via the irq_desc lock */
291 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
292 cpu
= its_dev
->event_map
.col_map
[its_get_event_id(d
)];
293 /* Keep GCC quiet... */
300 static void irq_to_cpuid_unlock(struct irq_data
*d
, unsigned long flags
)
302 struct its_vpe
*vpe
= NULL
;
304 if (d
->chip
== &its_vpe_irq_chip
) {
305 vpe
= irq_data_get_irq_chip_data(d
);
307 struct its_vlpi_map
*map
= get_vlpi_map(d
);
313 vpe_to_cpuid_unlock(vpe
, flags
);
316 static struct its_collection
*valid_col(struct its_collection
*col
)
318 if (WARN_ON_ONCE(col
->target_address
& GENMASK_ULL(15, 0)))
324 static struct its_vpe
*valid_vpe(struct its_node
*its
, struct its_vpe
*vpe
)
326 if (valid_col(its
->collections
+ vpe
->col_idx
))
333 * ITS command descriptors - parameters to be encoded in a command
336 struct its_cmd_desc
{
339 struct its_device
*dev
;
344 struct its_device
*dev
;
349 struct its_device
*dev
;
354 struct its_device
*dev
;
359 struct its_collection
*col
;
364 struct its_device
*dev
;
370 struct its_device
*dev
;
371 struct its_collection
*col
;
376 struct its_device
*dev
;
381 struct its_collection
*col
;
390 struct its_collection
*col
;
396 struct its_device
*dev
;
404 struct its_device
*dev
;
411 struct its_collection
*col
;
432 * The ITS command block, which is what the ITS actually parses.
434 struct its_cmd_block
{
437 __le64 raw_cmd_le
[4];
441 #define ITS_CMD_QUEUE_SZ SZ_64K
442 #define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
444 typedef struct its_collection
*(*its_cmd_builder_t
)(struct its_node
*,
445 struct its_cmd_block
*,
446 struct its_cmd_desc
*);
448 typedef struct its_vpe
*(*its_cmd_vbuilder_t
)(struct its_node
*,
449 struct its_cmd_block
*,
450 struct its_cmd_desc
*);
452 static void its_mask_encode(u64
*raw_cmd
, u64 val
, int h
, int l
)
454 u64 mask
= GENMASK_ULL(h
, l
);
456 *raw_cmd
|= (val
<< l
) & mask
;
459 static void its_encode_cmd(struct its_cmd_block
*cmd
, u8 cmd_nr
)
461 its_mask_encode(&cmd
->raw_cmd
[0], cmd_nr
, 7, 0);
464 static void its_encode_devid(struct its_cmd_block
*cmd
, u32 devid
)
466 its_mask_encode(&cmd
->raw_cmd
[0], devid
, 63, 32);
469 static void its_encode_event_id(struct its_cmd_block
*cmd
, u32 id
)
471 its_mask_encode(&cmd
->raw_cmd
[1], id
, 31, 0);
474 static void its_encode_phys_id(struct its_cmd_block
*cmd
, u32 phys_id
)
476 its_mask_encode(&cmd
->raw_cmd
[1], phys_id
, 63, 32);
479 static void its_encode_size(struct its_cmd_block
*cmd
, u8 size
)
481 its_mask_encode(&cmd
->raw_cmd
[1], size
, 4, 0);
484 static void its_encode_itt(struct its_cmd_block
*cmd
, u64 itt_addr
)
486 its_mask_encode(&cmd
->raw_cmd
[2], itt_addr
>> 8, 51, 8);
489 static void its_encode_valid(struct its_cmd_block
*cmd
, int valid
)
491 its_mask_encode(&cmd
->raw_cmd
[2], !!valid
, 63, 63);
494 static void its_encode_target(struct its_cmd_block
*cmd
, u64 target_addr
)
496 its_mask_encode(&cmd
->raw_cmd
[2], target_addr
>> 16, 51, 16);
499 static void its_encode_collection(struct its_cmd_block
*cmd
, u16 col
)
501 its_mask_encode(&cmd
->raw_cmd
[2], col
, 15, 0);
504 static void its_encode_vpeid(struct its_cmd_block
*cmd
, u16 vpeid
)
506 its_mask_encode(&cmd
->raw_cmd
[1], vpeid
, 47, 32);
509 static void its_encode_virt_id(struct its_cmd_block
*cmd
, u32 virt_id
)
511 its_mask_encode(&cmd
->raw_cmd
[2], virt_id
, 31, 0);
514 static void its_encode_db_phys_id(struct its_cmd_block
*cmd
, u32 db_phys_id
)
516 its_mask_encode(&cmd
->raw_cmd
[2], db_phys_id
, 63, 32);
519 static void its_encode_db_valid(struct its_cmd_block
*cmd
, bool db_valid
)
521 its_mask_encode(&cmd
->raw_cmd
[2], db_valid
, 0, 0);
524 static void its_encode_seq_num(struct its_cmd_block
*cmd
, u16 seq_num
)
526 its_mask_encode(&cmd
->raw_cmd
[0], seq_num
, 47, 32);
529 static void its_encode_its_list(struct its_cmd_block
*cmd
, u16 its_list
)
531 its_mask_encode(&cmd
->raw_cmd
[1], its_list
, 15, 0);
534 static void its_encode_vpt_addr(struct its_cmd_block
*cmd
, u64 vpt_pa
)
536 its_mask_encode(&cmd
->raw_cmd
[3], vpt_pa
>> 16, 51, 16);
539 static void its_encode_vpt_size(struct its_cmd_block
*cmd
, u8 vpt_size
)
541 its_mask_encode(&cmd
->raw_cmd
[3], vpt_size
, 4, 0);
544 static void its_encode_vconf_addr(struct its_cmd_block
*cmd
, u64 vconf_pa
)
546 its_mask_encode(&cmd
->raw_cmd
[0], vconf_pa
>> 16, 51, 16);
549 static void its_encode_alloc(struct its_cmd_block
*cmd
, bool alloc
)
551 its_mask_encode(&cmd
->raw_cmd
[0], alloc
, 8, 8);
554 static void its_encode_ptz(struct its_cmd_block
*cmd
, bool ptz
)
556 its_mask_encode(&cmd
->raw_cmd
[0], ptz
, 9, 9);
559 static void its_encode_vmapp_default_db(struct its_cmd_block
*cmd
,
562 its_mask_encode(&cmd
->raw_cmd
[1], vpe_db_lpi
, 31, 0);
565 static void its_encode_vmovp_default_db(struct its_cmd_block
*cmd
,
568 its_mask_encode(&cmd
->raw_cmd
[3], vpe_db_lpi
, 31, 0);
571 static void its_encode_db(struct its_cmd_block
*cmd
, bool db
)
573 its_mask_encode(&cmd
->raw_cmd
[2], db
, 63, 63);
576 static void its_encode_sgi_intid(struct its_cmd_block
*cmd
, u8 sgi
)
578 its_mask_encode(&cmd
->raw_cmd
[0], sgi
, 35, 32);
581 static void its_encode_sgi_priority(struct its_cmd_block
*cmd
, u8 prio
)
583 its_mask_encode(&cmd
->raw_cmd
[0], prio
>> 4, 23, 20);
586 static void its_encode_sgi_group(struct its_cmd_block
*cmd
, bool grp
)
588 its_mask_encode(&cmd
->raw_cmd
[0], grp
, 10, 10);
591 static void its_encode_sgi_clear(struct its_cmd_block
*cmd
, bool clr
)
593 its_mask_encode(&cmd
->raw_cmd
[0], clr
, 9, 9);
596 static void its_encode_sgi_enable(struct its_cmd_block
*cmd
, bool en
)
598 its_mask_encode(&cmd
->raw_cmd
[0], en
, 8, 8);
601 static inline void its_fixup_cmd(struct its_cmd_block
*cmd
)
603 /* Let's fixup BE commands */
604 cmd
->raw_cmd_le
[0] = cpu_to_le64(cmd
->raw_cmd
[0]);
605 cmd
->raw_cmd_le
[1] = cpu_to_le64(cmd
->raw_cmd
[1]);
606 cmd
->raw_cmd_le
[2] = cpu_to_le64(cmd
->raw_cmd
[2]);
607 cmd
->raw_cmd_le
[3] = cpu_to_le64(cmd
->raw_cmd
[3]);
610 static struct its_collection
*its_build_mapd_cmd(struct its_node
*its
,
611 struct its_cmd_block
*cmd
,
612 struct its_cmd_desc
*desc
)
614 unsigned long itt_addr
;
615 u8 size
= ilog2(desc
->its_mapd_cmd
.dev
->nr_ites
);
617 itt_addr
= virt_to_phys(desc
->its_mapd_cmd
.dev
->itt
);
618 itt_addr
= ALIGN(itt_addr
, ITS_ITT_ALIGN
);
620 its_encode_cmd(cmd
, GITS_CMD_MAPD
);
621 its_encode_devid(cmd
, desc
->its_mapd_cmd
.dev
->device_id
);
622 its_encode_size(cmd
, size
- 1);
623 its_encode_itt(cmd
, itt_addr
);
624 its_encode_valid(cmd
, desc
->its_mapd_cmd
.valid
);
631 static struct its_collection
*its_build_mapc_cmd(struct its_node
*its
,
632 struct its_cmd_block
*cmd
,
633 struct its_cmd_desc
*desc
)
635 its_encode_cmd(cmd
, GITS_CMD_MAPC
);
636 its_encode_collection(cmd
, desc
->its_mapc_cmd
.col
->col_id
);
637 its_encode_target(cmd
, desc
->its_mapc_cmd
.col
->target_address
);
638 its_encode_valid(cmd
, desc
->its_mapc_cmd
.valid
);
642 return desc
->its_mapc_cmd
.col
;
645 static struct its_collection
*its_build_mapti_cmd(struct its_node
*its
,
646 struct its_cmd_block
*cmd
,
647 struct its_cmd_desc
*desc
)
649 struct its_collection
*col
;
651 col
= dev_event_to_col(desc
->its_mapti_cmd
.dev
,
652 desc
->its_mapti_cmd
.event_id
);
654 its_encode_cmd(cmd
, GITS_CMD_MAPTI
);
655 its_encode_devid(cmd
, desc
->its_mapti_cmd
.dev
->device_id
);
656 its_encode_event_id(cmd
, desc
->its_mapti_cmd
.event_id
);
657 its_encode_phys_id(cmd
, desc
->its_mapti_cmd
.phys_id
);
658 its_encode_collection(cmd
, col
->col_id
);
662 return valid_col(col
);
665 static struct its_collection
*its_build_movi_cmd(struct its_node
*its
,
666 struct its_cmd_block
*cmd
,
667 struct its_cmd_desc
*desc
)
669 struct its_collection
*col
;
671 col
= dev_event_to_col(desc
->its_movi_cmd
.dev
,
672 desc
->its_movi_cmd
.event_id
);
674 its_encode_cmd(cmd
, GITS_CMD_MOVI
);
675 its_encode_devid(cmd
, desc
->its_movi_cmd
.dev
->device_id
);
676 its_encode_event_id(cmd
, desc
->its_movi_cmd
.event_id
);
677 its_encode_collection(cmd
, desc
->its_movi_cmd
.col
->col_id
);
681 return valid_col(col
);
684 static struct its_collection
*its_build_discard_cmd(struct its_node
*its
,
685 struct its_cmd_block
*cmd
,
686 struct its_cmd_desc
*desc
)
688 struct its_collection
*col
;
690 col
= dev_event_to_col(desc
->its_discard_cmd
.dev
,
691 desc
->its_discard_cmd
.event_id
);
693 its_encode_cmd(cmd
, GITS_CMD_DISCARD
);
694 its_encode_devid(cmd
, desc
->its_discard_cmd
.dev
->device_id
);
695 its_encode_event_id(cmd
, desc
->its_discard_cmd
.event_id
);
699 return valid_col(col
);
702 static struct its_collection
*its_build_inv_cmd(struct its_node
*its
,
703 struct its_cmd_block
*cmd
,
704 struct its_cmd_desc
*desc
)
706 struct its_collection
*col
;
708 col
= dev_event_to_col(desc
->its_inv_cmd
.dev
,
709 desc
->its_inv_cmd
.event_id
);
711 its_encode_cmd(cmd
, GITS_CMD_INV
);
712 its_encode_devid(cmd
, desc
->its_inv_cmd
.dev
->device_id
);
713 its_encode_event_id(cmd
, desc
->its_inv_cmd
.event_id
);
717 return valid_col(col
);
720 static struct its_collection
*its_build_int_cmd(struct its_node
*its
,
721 struct its_cmd_block
*cmd
,
722 struct its_cmd_desc
*desc
)
724 struct its_collection
*col
;
726 col
= dev_event_to_col(desc
->its_int_cmd
.dev
,
727 desc
->its_int_cmd
.event_id
);
729 its_encode_cmd(cmd
, GITS_CMD_INT
);
730 its_encode_devid(cmd
, desc
->its_int_cmd
.dev
->device_id
);
731 its_encode_event_id(cmd
, desc
->its_int_cmd
.event_id
);
735 return valid_col(col
);
738 static struct its_collection
*its_build_clear_cmd(struct its_node
*its
,
739 struct its_cmd_block
*cmd
,
740 struct its_cmd_desc
*desc
)
742 struct its_collection
*col
;
744 col
= dev_event_to_col(desc
->its_clear_cmd
.dev
,
745 desc
->its_clear_cmd
.event_id
);
747 its_encode_cmd(cmd
, GITS_CMD_CLEAR
);
748 its_encode_devid(cmd
, desc
->its_clear_cmd
.dev
->device_id
);
749 its_encode_event_id(cmd
, desc
->its_clear_cmd
.event_id
);
753 return valid_col(col
);
756 static struct its_collection
*its_build_invall_cmd(struct its_node
*its
,
757 struct its_cmd_block
*cmd
,
758 struct its_cmd_desc
*desc
)
760 its_encode_cmd(cmd
, GITS_CMD_INVALL
);
761 its_encode_collection(cmd
, desc
->its_invall_cmd
.col
->col_id
);
765 return desc
->its_invall_cmd
.col
;
768 static struct its_vpe
*its_build_vinvall_cmd(struct its_node
*its
,
769 struct its_cmd_block
*cmd
,
770 struct its_cmd_desc
*desc
)
772 its_encode_cmd(cmd
, GITS_CMD_VINVALL
);
773 its_encode_vpeid(cmd
, desc
->its_vinvall_cmd
.vpe
->vpe_id
);
777 return valid_vpe(its
, desc
->its_vinvall_cmd
.vpe
);
780 static struct its_vpe
*its_build_vmapp_cmd(struct its_node
*its
,
781 struct its_cmd_block
*cmd
,
782 struct its_cmd_desc
*desc
)
784 unsigned long vpt_addr
, vconf_addr
;
788 its_encode_cmd(cmd
, GITS_CMD_VMAPP
);
789 its_encode_vpeid(cmd
, desc
->its_vmapp_cmd
.vpe
->vpe_id
);
790 its_encode_valid(cmd
, desc
->its_vmapp_cmd
.valid
);
792 if (!desc
->its_vmapp_cmd
.valid
) {
794 alloc
= !atomic_dec_return(&desc
->its_vmapp_cmd
.vpe
->vmapp_count
);
795 its_encode_alloc(cmd
, alloc
);
801 vpt_addr
= virt_to_phys(page_address(desc
->its_vmapp_cmd
.vpe
->vpt_page
));
802 target
= desc
->its_vmapp_cmd
.col
->target_address
+ its
->vlpi_redist_offset
;
804 its_encode_target(cmd
, target
);
805 its_encode_vpt_addr(cmd
, vpt_addr
);
806 its_encode_vpt_size(cmd
, LPI_NRBITS
- 1);
811 vconf_addr
= virt_to_phys(page_address(desc
->its_vmapp_cmd
.vpe
->its_vm
->vprop_page
));
813 alloc
= !atomic_fetch_inc(&desc
->its_vmapp_cmd
.vpe
->vmapp_count
);
815 its_encode_alloc(cmd
, alloc
);
818 * GICv4.1 provides a way to get the VLPI state, which needs the vPE
819 * to be unmapped first, and in this case, we may remap the vPE
820 * back while the VPT is not empty. So we can't assume that the
821 * VPT is empty on map. This is why we never advertise PTZ.
823 its_encode_ptz(cmd
, false);
824 its_encode_vconf_addr(cmd
, vconf_addr
);
825 its_encode_vmapp_default_db(cmd
, desc
->its_vmapp_cmd
.vpe
->vpe_db_lpi
);
830 return valid_vpe(its
, desc
->its_vmapp_cmd
.vpe
);
833 static struct its_vpe
*its_build_vmapti_cmd(struct its_node
*its
,
834 struct its_cmd_block
*cmd
,
835 struct its_cmd_desc
*desc
)
839 if (!is_v4_1(its
) && desc
->its_vmapti_cmd
.db_enabled
)
840 db
= desc
->its_vmapti_cmd
.vpe
->vpe_db_lpi
;
844 its_encode_cmd(cmd
, GITS_CMD_VMAPTI
);
845 its_encode_devid(cmd
, desc
->its_vmapti_cmd
.dev
->device_id
);
846 its_encode_vpeid(cmd
, desc
->its_vmapti_cmd
.vpe
->vpe_id
);
847 its_encode_event_id(cmd
, desc
->its_vmapti_cmd
.event_id
);
848 its_encode_db_phys_id(cmd
, db
);
849 its_encode_virt_id(cmd
, desc
->its_vmapti_cmd
.virt_id
);
853 return valid_vpe(its
, desc
->its_vmapti_cmd
.vpe
);
856 static struct its_vpe
*its_build_vmovi_cmd(struct its_node
*its
,
857 struct its_cmd_block
*cmd
,
858 struct its_cmd_desc
*desc
)
862 if (!is_v4_1(its
) && desc
->its_vmovi_cmd
.db_enabled
)
863 db
= desc
->its_vmovi_cmd
.vpe
->vpe_db_lpi
;
867 its_encode_cmd(cmd
, GITS_CMD_VMOVI
);
868 its_encode_devid(cmd
, desc
->its_vmovi_cmd
.dev
->device_id
);
869 its_encode_vpeid(cmd
, desc
->its_vmovi_cmd
.vpe
->vpe_id
);
870 its_encode_event_id(cmd
, desc
->its_vmovi_cmd
.event_id
);
871 its_encode_db_phys_id(cmd
, db
);
872 its_encode_db_valid(cmd
, true);
876 return valid_vpe(its
, desc
->its_vmovi_cmd
.vpe
);
879 static struct its_vpe
*its_build_vmovp_cmd(struct its_node
*its
,
880 struct its_cmd_block
*cmd
,
881 struct its_cmd_desc
*desc
)
885 target
= desc
->its_vmovp_cmd
.col
->target_address
+ its
->vlpi_redist_offset
;
886 its_encode_cmd(cmd
, GITS_CMD_VMOVP
);
887 its_encode_seq_num(cmd
, desc
->its_vmovp_cmd
.seq_num
);
888 its_encode_its_list(cmd
, desc
->its_vmovp_cmd
.its_list
);
889 its_encode_vpeid(cmd
, desc
->its_vmovp_cmd
.vpe
->vpe_id
);
890 its_encode_target(cmd
, target
);
893 its_encode_db(cmd
, true);
894 its_encode_vmovp_default_db(cmd
, desc
->its_vmovp_cmd
.vpe
->vpe_db_lpi
);
899 return valid_vpe(its
, desc
->its_vmovp_cmd
.vpe
);
902 static struct its_vpe
*its_build_vinv_cmd(struct its_node
*its
,
903 struct its_cmd_block
*cmd
,
904 struct its_cmd_desc
*desc
)
906 struct its_vlpi_map
*map
;
908 map
= dev_event_to_vlpi_map(desc
->its_inv_cmd
.dev
,
909 desc
->its_inv_cmd
.event_id
);
911 its_encode_cmd(cmd
, GITS_CMD_INV
);
912 its_encode_devid(cmd
, desc
->its_inv_cmd
.dev
->device_id
);
913 its_encode_event_id(cmd
, desc
->its_inv_cmd
.event_id
);
917 return valid_vpe(its
, map
->vpe
);
920 static struct its_vpe
*its_build_vint_cmd(struct its_node
*its
,
921 struct its_cmd_block
*cmd
,
922 struct its_cmd_desc
*desc
)
924 struct its_vlpi_map
*map
;
926 map
= dev_event_to_vlpi_map(desc
->its_int_cmd
.dev
,
927 desc
->its_int_cmd
.event_id
);
929 its_encode_cmd(cmd
, GITS_CMD_INT
);
930 its_encode_devid(cmd
, desc
->its_int_cmd
.dev
->device_id
);
931 its_encode_event_id(cmd
, desc
->its_int_cmd
.event_id
);
935 return valid_vpe(its
, map
->vpe
);
938 static struct its_vpe
*its_build_vclear_cmd(struct its_node
*its
,
939 struct its_cmd_block
*cmd
,
940 struct its_cmd_desc
*desc
)
942 struct its_vlpi_map
*map
;
944 map
= dev_event_to_vlpi_map(desc
->its_clear_cmd
.dev
,
945 desc
->its_clear_cmd
.event_id
);
947 its_encode_cmd(cmd
, GITS_CMD_CLEAR
);
948 its_encode_devid(cmd
, desc
->its_clear_cmd
.dev
->device_id
);
949 its_encode_event_id(cmd
, desc
->its_clear_cmd
.event_id
);
953 return valid_vpe(its
, map
->vpe
);
956 static struct its_vpe
*its_build_invdb_cmd(struct its_node
*its
,
957 struct its_cmd_block
*cmd
,
958 struct its_cmd_desc
*desc
)
960 if (WARN_ON(!is_v4_1(its
)))
963 its_encode_cmd(cmd
, GITS_CMD_INVDB
);
964 its_encode_vpeid(cmd
, desc
->its_invdb_cmd
.vpe
->vpe_id
);
968 return valid_vpe(its
, desc
->its_invdb_cmd
.vpe
);
971 static struct its_vpe
*its_build_vsgi_cmd(struct its_node
*its
,
972 struct its_cmd_block
*cmd
,
973 struct its_cmd_desc
*desc
)
975 if (WARN_ON(!is_v4_1(its
)))
978 its_encode_cmd(cmd
, GITS_CMD_VSGI
);
979 its_encode_vpeid(cmd
, desc
->its_vsgi_cmd
.vpe
->vpe_id
);
980 its_encode_sgi_intid(cmd
, desc
->its_vsgi_cmd
.sgi
);
981 its_encode_sgi_priority(cmd
, desc
->its_vsgi_cmd
.priority
);
982 its_encode_sgi_group(cmd
, desc
->its_vsgi_cmd
.group
);
983 its_encode_sgi_clear(cmd
, desc
->its_vsgi_cmd
.clear
);
984 its_encode_sgi_enable(cmd
, desc
->its_vsgi_cmd
.enable
);
988 return valid_vpe(its
, desc
->its_vsgi_cmd
.vpe
);
991 static u64
its_cmd_ptr_to_offset(struct its_node
*its
,
992 struct its_cmd_block
*ptr
)
994 return (ptr
- its
->cmd_base
) * sizeof(*ptr
);
997 static int its_queue_full(struct its_node
*its
)
1002 widx
= its
->cmd_write
- its
->cmd_base
;
1003 ridx
= readl_relaxed(its
->base
+ GITS_CREADR
) / sizeof(struct its_cmd_block
);
1005 /* This is incredibly unlikely to happen, unless the ITS locks up. */
1006 if (((widx
+ 1) % ITS_CMD_QUEUE_NR_ENTRIES
) == ridx
)
1012 static struct its_cmd_block
*its_allocate_entry(struct its_node
*its
)
1014 struct its_cmd_block
*cmd
;
1015 u32 count
= 1000000; /* 1s! */
1017 while (its_queue_full(its
)) {
1020 pr_err_ratelimited("ITS queue not draining\n");
1027 cmd
= its
->cmd_write
++;
1029 /* Handle queue wrapping */
1030 if (its
->cmd_write
== (its
->cmd_base
+ ITS_CMD_QUEUE_NR_ENTRIES
))
1031 its
->cmd_write
= its
->cmd_base
;
1034 cmd
->raw_cmd
[0] = 0;
1035 cmd
->raw_cmd
[1] = 0;
1036 cmd
->raw_cmd
[2] = 0;
1037 cmd
->raw_cmd
[3] = 0;
1042 static struct its_cmd_block
*its_post_commands(struct its_node
*its
)
1044 u64 wr
= its_cmd_ptr_to_offset(its
, its
->cmd_write
);
1046 writel_relaxed(wr
, its
->base
+ GITS_CWRITER
);
1048 return its
->cmd_write
;
1051 static void its_flush_cmd(struct its_node
*its
, struct its_cmd_block
*cmd
)
1054 * Make sure the commands written to memory are observable by
1057 if (its
->flags
& ITS_FLAGS_CMDQ_NEEDS_FLUSHING
)
1058 gic_flush_dcache_to_poc(cmd
, sizeof(*cmd
));
1063 static int its_wait_for_range_completion(struct its_node
*its
,
1065 struct its_cmd_block
*to
)
1067 u64 rd_idx
, to_idx
, linear_idx
;
1068 u32 count
= 1000000; /* 1s! */
1070 /* Linearize to_idx if the command set has wrapped around */
1071 to_idx
= its_cmd_ptr_to_offset(its
, to
);
1072 if (to_idx
< prev_idx
)
1073 to_idx
+= ITS_CMD_QUEUE_SZ
;
1075 linear_idx
= prev_idx
;
1080 rd_idx
= readl_relaxed(its
->base
+ GITS_CREADR
);
1083 * Compute the read pointer progress, taking the
1084 * potential wrap-around into account.
1086 delta
= rd_idx
- prev_idx
;
1087 if (rd_idx
< prev_idx
)
1088 delta
+= ITS_CMD_QUEUE_SZ
;
1090 linear_idx
+= delta
;
1091 if (linear_idx
>= to_idx
)
1096 pr_err_ratelimited("ITS queue timeout (%llu %llu)\n",
1097 to_idx
, linear_idx
);
1108 /* Warning, macro hell follows */
1109 #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn) \
1110 void name(struct its_node *its, \
1111 buildtype builder, \
1112 struct its_cmd_desc *desc) \
1114 struct its_cmd_block *cmd, *sync_cmd, *next_cmd; \
1115 synctype *sync_obj; \
1116 unsigned long flags; \
1119 raw_spin_lock_irqsave(&its->lock, flags); \
1121 cmd = its_allocate_entry(its); \
1122 if (!cmd) { /* We're soooooo screewed... */ \
1123 raw_spin_unlock_irqrestore(&its->lock, flags); \
1126 sync_obj = builder(its, cmd, desc); \
1127 its_flush_cmd(its, cmd); \
1130 sync_cmd = its_allocate_entry(its); \
1134 buildfn(its, sync_cmd, sync_obj); \
1135 its_flush_cmd(its, sync_cmd); \
1139 rd_idx = readl_relaxed(its->base + GITS_CREADR); \
1140 next_cmd = its_post_commands(its); \
1141 raw_spin_unlock_irqrestore(&its->lock, flags); \
1143 if (its_wait_for_range_completion(its, rd_idx, next_cmd)) \
1144 pr_err_ratelimited("ITS cmd %ps failed\n", builder); \
1147 static void its_build_sync_cmd(struct its_node
*its
,
1148 struct its_cmd_block
*sync_cmd
,
1149 struct its_collection
*sync_col
)
1151 its_encode_cmd(sync_cmd
, GITS_CMD_SYNC
);
1152 its_encode_target(sync_cmd
, sync_col
->target_address
);
1154 its_fixup_cmd(sync_cmd
);
1157 static BUILD_SINGLE_CMD_FUNC(its_send_single_command
, its_cmd_builder_t
,
1158 struct its_collection
, its_build_sync_cmd
)
1160 static void its_build_vsync_cmd(struct its_node
*its
,
1161 struct its_cmd_block
*sync_cmd
,
1162 struct its_vpe
*sync_vpe
)
1164 its_encode_cmd(sync_cmd
, GITS_CMD_VSYNC
);
1165 its_encode_vpeid(sync_cmd
, sync_vpe
->vpe_id
);
1167 its_fixup_cmd(sync_cmd
);
1170 static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand
, its_cmd_vbuilder_t
,
1171 struct its_vpe
, its_build_vsync_cmd
)
1173 static void its_send_int(struct its_device
*dev
, u32 event_id
)
1175 struct its_cmd_desc desc
;
1177 desc
.its_int_cmd
.dev
= dev
;
1178 desc
.its_int_cmd
.event_id
= event_id
;
1180 its_send_single_command(dev
->its
, its_build_int_cmd
, &desc
);
1183 static void its_send_clear(struct its_device
*dev
, u32 event_id
)
1185 struct its_cmd_desc desc
;
1187 desc
.its_clear_cmd
.dev
= dev
;
1188 desc
.its_clear_cmd
.event_id
= event_id
;
1190 its_send_single_command(dev
->its
, its_build_clear_cmd
, &desc
);
1193 static void its_send_inv(struct its_device
*dev
, u32 event_id
)
1195 struct its_cmd_desc desc
;
1197 desc
.its_inv_cmd
.dev
= dev
;
1198 desc
.its_inv_cmd
.event_id
= event_id
;
1200 its_send_single_command(dev
->its
, its_build_inv_cmd
, &desc
);
1203 static void its_send_mapd(struct its_device
*dev
, int valid
)
1205 struct its_cmd_desc desc
;
1207 desc
.its_mapd_cmd
.dev
= dev
;
1208 desc
.its_mapd_cmd
.valid
= !!valid
;
1210 its_send_single_command(dev
->its
, its_build_mapd_cmd
, &desc
);
1213 static void its_send_mapc(struct its_node
*its
, struct its_collection
*col
,
1216 struct its_cmd_desc desc
;
1218 desc
.its_mapc_cmd
.col
= col
;
1219 desc
.its_mapc_cmd
.valid
= !!valid
;
1221 its_send_single_command(its
, its_build_mapc_cmd
, &desc
);
1224 static void its_send_mapti(struct its_device
*dev
, u32 irq_id
, u32 id
)
1226 struct its_cmd_desc desc
;
1228 desc
.its_mapti_cmd
.dev
= dev
;
1229 desc
.its_mapti_cmd
.phys_id
= irq_id
;
1230 desc
.its_mapti_cmd
.event_id
= id
;
1232 its_send_single_command(dev
->its
, its_build_mapti_cmd
, &desc
);
1235 static void its_send_movi(struct its_device
*dev
,
1236 struct its_collection
*col
, u32 id
)
1238 struct its_cmd_desc desc
;
1240 desc
.its_movi_cmd
.dev
= dev
;
1241 desc
.its_movi_cmd
.col
= col
;
1242 desc
.its_movi_cmd
.event_id
= id
;
1244 its_send_single_command(dev
->its
, its_build_movi_cmd
, &desc
);
1247 static void its_send_discard(struct its_device
*dev
, u32 id
)
1249 struct its_cmd_desc desc
;
1251 desc
.its_discard_cmd
.dev
= dev
;
1252 desc
.its_discard_cmd
.event_id
= id
;
1254 its_send_single_command(dev
->its
, its_build_discard_cmd
, &desc
);
1257 static void its_send_invall(struct its_node
*its
, struct its_collection
*col
)
1259 struct its_cmd_desc desc
;
1261 desc
.its_invall_cmd
.col
= col
;
1263 its_send_single_command(its
, its_build_invall_cmd
, &desc
);
1266 static void its_send_vmapti(struct its_device
*dev
, u32 id
)
1268 struct its_vlpi_map
*map
= dev_event_to_vlpi_map(dev
, id
);
1269 struct its_cmd_desc desc
;
1271 desc
.its_vmapti_cmd
.vpe
= map
->vpe
;
1272 desc
.its_vmapti_cmd
.dev
= dev
;
1273 desc
.its_vmapti_cmd
.virt_id
= map
->vintid
;
1274 desc
.its_vmapti_cmd
.event_id
= id
;
1275 desc
.its_vmapti_cmd
.db_enabled
= map
->db_enabled
;
1277 its_send_single_vcommand(dev
->its
, its_build_vmapti_cmd
, &desc
);
1280 static void its_send_vmovi(struct its_device
*dev
, u32 id
)
1282 struct its_vlpi_map
*map
= dev_event_to_vlpi_map(dev
, id
);
1283 struct its_cmd_desc desc
;
1285 desc
.its_vmovi_cmd
.vpe
= map
->vpe
;
1286 desc
.its_vmovi_cmd
.dev
= dev
;
1287 desc
.its_vmovi_cmd
.event_id
= id
;
1288 desc
.its_vmovi_cmd
.db_enabled
= map
->db_enabled
;
1290 its_send_single_vcommand(dev
->its
, its_build_vmovi_cmd
, &desc
);
1293 static void its_send_vmapp(struct its_node
*its
,
1294 struct its_vpe
*vpe
, bool valid
)
1296 struct its_cmd_desc desc
;
1298 desc
.its_vmapp_cmd
.vpe
= vpe
;
1299 desc
.its_vmapp_cmd
.valid
= valid
;
1300 desc
.its_vmapp_cmd
.col
= &its
->collections
[vpe
->col_idx
];
1302 its_send_single_vcommand(its
, its_build_vmapp_cmd
, &desc
);
1305 static void its_send_vmovp(struct its_vpe
*vpe
)
1307 struct its_cmd_desc desc
= {};
1308 struct its_node
*its
;
1309 unsigned long flags
;
1310 int col_id
= vpe
->col_idx
;
1312 desc
.its_vmovp_cmd
.vpe
= vpe
;
1314 if (!its_list_map
) {
1315 its
= list_first_entry(&its_nodes
, struct its_node
, entry
);
1316 desc
.its_vmovp_cmd
.col
= &its
->collections
[col_id
];
1317 its_send_single_vcommand(its
, its_build_vmovp_cmd
, &desc
);
1322 * Yet another marvel of the architecture. If using the
1323 * its_list "feature", we need to make sure that all ITSs
1324 * receive all VMOVP commands in the same order. The only way
1325 * to guarantee this is to make vmovp a serialization point.
1329 raw_spin_lock_irqsave(&vmovp_lock
, flags
);
1331 desc
.its_vmovp_cmd
.seq_num
= vmovp_seq_num
++;
1332 desc
.its_vmovp_cmd
.its_list
= get_its_list(vpe
->its_vm
);
1335 list_for_each_entry(its
, &its_nodes
, entry
) {
1339 if (!require_its_list_vmovp(vpe
->its_vm
, its
))
1342 desc
.its_vmovp_cmd
.col
= &its
->collections
[col_id
];
1343 its_send_single_vcommand(its
, its_build_vmovp_cmd
, &desc
);
1346 raw_spin_unlock_irqrestore(&vmovp_lock
, flags
);
1349 static void its_send_vinvall(struct its_node
*its
, struct its_vpe
*vpe
)
1351 struct its_cmd_desc desc
;
1353 desc
.its_vinvall_cmd
.vpe
= vpe
;
1354 its_send_single_vcommand(its
, its_build_vinvall_cmd
, &desc
);
1357 static void its_send_vinv(struct its_device
*dev
, u32 event_id
)
1359 struct its_cmd_desc desc
;
1362 * There is no real VINV command. This is just a normal INV,
1363 * with a VSYNC instead of a SYNC.
1365 desc
.its_inv_cmd
.dev
= dev
;
1366 desc
.its_inv_cmd
.event_id
= event_id
;
1368 its_send_single_vcommand(dev
->its
, its_build_vinv_cmd
, &desc
);
1371 static void its_send_vint(struct its_device
*dev
, u32 event_id
)
1373 struct its_cmd_desc desc
;
1376 * There is no real VINT command. This is just a normal INT,
1377 * with a VSYNC instead of a SYNC.
1379 desc
.its_int_cmd
.dev
= dev
;
1380 desc
.its_int_cmd
.event_id
= event_id
;
1382 its_send_single_vcommand(dev
->its
, its_build_vint_cmd
, &desc
);
1385 static void its_send_vclear(struct its_device
*dev
, u32 event_id
)
1387 struct its_cmd_desc desc
;
1390 * There is no real VCLEAR command. This is just a normal CLEAR,
1391 * with a VSYNC instead of a SYNC.
1393 desc
.its_clear_cmd
.dev
= dev
;
1394 desc
.its_clear_cmd
.event_id
= event_id
;
1396 its_send_single_vcommand(dev
->its
, its_build_vclear_cmd
, &desc
);
1399 static void its_send_invdb(struct its_node
*its
, struct its_vpe
*vpe
)
1401 struct its_cmd_desc desc
;
1403 desc
.its_invdb_cmd
.vpe
= vpe
;
1404 its_send_single_vcommand(its
, its_build_invdb_cmd
, &desc
);
1408 * irqchip functions - assumes MSI, mostly.
1410 static void lpi_write_config(struct irq_data
*d
, u8 clr
, u8 set
)
1412 struct its_vlpi_map
*map
= get_vlpi_map(d
);
1413 irq_hw_number_t hwirq
;
1418 va
= page_address(map
->vm
->vprop_page
);
1419 hwirq
= map
->vintid
;
1421 /* Remember the updated property */
1422 map
->properties
&= ~clr
;
1423 map
->properties
|= set
| LPI_PROP_GROUP1
;
1425 va
= gic_rdists
->prop_table_va
;
1429 cfg
= va
+ hwirq
- 8192;
1431 *cfg
|= set
| LPI_PROP_GROUP1
;
1434 * Make the above write visible to the redistributors.
1435 * And yes, we're flushing exactly: One. Single. Byte.
1438 if (gic_rdists
->flags
& RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING
)
1439 gic_flush_dcache_to_poc(cfg
, sizeof(*cfg
));
1444 static void wait_for_syncr(void __iomem
*rdbase
)
1446 while (readl_relaxed(rdbase
+ GICR_SYNCR
) & 1)
1450 static void __direct_lpi_inv(struct irq_data
*d
, u64 val
)
1452 void __iomem
*rdbase
;
1453 unsigned long flags
;
1456 /* Target the redistributor this LPI is currently routed to */
1457 cpu
= irq_to_cpuid_lock(d
, &flags
);
1458 raw_spin_lock(&gic_data_rdist_cpu(cpu
)->rd_lock
);
1460 rdbase
= per_cpu_ptr(gic_rdists
->rdist
, cpu
)->rd_base
;
1461 gic_write_lpir(val
, rdbase
+ GICR_INVLPIR
);
1462 wait_for_syncr(rdbase
);
1464 raw_spin_unlock(&gic_data_rdist_cpu(cpu
)->rd_lock
);
1465 irq_to_cpuid_unlock(d
, flags
);
1468 static void direct_lpi_inv(struct irq_data
*d
)
1470 struct its_vlpi_map
*map
= get_vlpi_map(d
);
1474 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1476 WARN_ON(!is_v4_1(its_dev
->its
));
1478 val
= GICR_INVLPIR_V
;
1479 val
|= FIELD_PREP(GICR_INVLPIR_VPEID
, map
->vpe
->vpe_id
);
1480 val
|= FIELD_PREP(GICR_INVLPIR_INTID
, map
->vintid
);
1485 __direct_lpi_inv(d
, val
);
1488 static void lpi_update_config(struct irq_data
*d
, u8 clr
, u8 set
)
1490 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1492 lpi_write_config(d
, clr
, set
);
1493 if (gic_rdists
->has_direct_lpi
&&
1494 (is_v4_1(its_dev
->its
) || !irqd_is_forwarded_to_vcpu(d
)))
1496 else if (!irqd_is_forwarded_to_vcpu(d
))
1497 its_send_inv(its_dev
, its_get_event_id(d
));
1499 its_send_vinv(its_dev
, its_get_event_id(d
));
1502 static void its_vlpi_set_doorbell(struct irq_data
*d
, bool enable
)
1504 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1505 u32 event
= its_get_event_id(d
);
1506 struct its_vlpi_map
*map
;
1509 * GICv4.1 does away with the per-LPI nonsense, nothing to do
1512 if (is_v4_1(its_dev
->its
))
1515 map
= dev_event_to_vlpi_map(its_dev
, event
);
1517 if (map
->db_enabled
== enable
)
1520 map
->db_enabled
= enable
;
1523 * More fun with the architecture:
1525 * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1526 * value or to 1023, depending on the enable bit. But that
1527 * would be issuing a mapping for an /existing/ DevID+EventID
1528 * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1529 * to the /same/ vPE, using this opportunity to adjust the
1530 * doorbell. Mouahahahaha. We loves it, Precious.
1532 its_send_vmovi(its_dev
, event
);
1535 static void its_mask_irq(struct irq_data
*d
)
1537 if (irqd_is_forwarded_to_vcpu(d
))
1538 its_vlpi_set_doorbell(d
, false);
1540 lpi_update_config(d
, LPI_PROP_ENABLED
, 0);
1543 static void its_unmask_irq(struct irq_data
*d
)
1545 if (irqd_is_forwarded_to_vcpu(d
))
1546 its_vlpi_set_doorbell(d
, true);
1548 lpi_update_config(d
, 0, LPI_PROP_ENABLED
);
1551 static __maybe_unused u32
its_read_lpi_count(struct irq_data
*d
, int cpu
)
1553 if (irqd_affinity_is_managed(d
))
1554 return atomic_read(&per_cpu_ptr(&cpu_lpi_count
, cpu
)->managed
);
1556 return atomic_read(&per_cpu_ptr(&cpu_lpi_count
, cpu
)->unmanaged
);
1559 static void its_inc_lpi_count(struct irq_data
*d
, int cpu
)
1561 if (irqd_affinity_is_managed(d
))
1562 atomic_inc(&per_cpu_ptr(&cpu_lpi_count
, cpu
)->managed
);
1564 atomic_inc(&per_cpu_ptr(&cpu_lpi_count
, cpu
)->unmanaged
);
1567 static void its_dec_lpi_count(struct irq_data
*d
, int cpu
)
1569 if (irqd_affinity_is_managed(d
))
1570 atomic_dec(&per_cpu_ptr(&cpu_lpi_count
, cpu
)->managed
);
1572 atomic_dec(&per_cpu_ptr(&cpu_lpi_count
, cpu
)->unmanaged
);
1575 static unsigned int cpumask_pick_least_loaded(struct irq_data
*d
,
1576 const struct cpumask
*cpu_mask
)
1578 unsigned int cpu
= nr_cpu_ids
, tmp
;
1579 int count
= S32_MAX
;
1581 for_each_cpu(tmp
, cpu_mask
) {
1582 int this_count
= its_read_lpi_count(d
, tmp
);
1583 if (this_count
< count
) {
1593 * As suggested by Thomas Gleixner in:
1594 * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
1596 static int its_select_cpu(struct irq_data
*d
,
1597 const struct cpumask
*aff_mask
)
1599 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1600 static DEFINE_RAW_SPINLOCK(tmpmask_lock
);
1601 static struct cpumask __tmpmask
;
1602 struct cpumask
*tmpmask
;
1603 unsigned long flags
;
1605 node
= its_dev
->its
->numa_node
;
1606 tmpmask
= &__tmpmask
;
1608 raw_spin_lock_irqsave(&tmpmask_lock
, flags
);
1610 if (!irqd_affinity_is_managed(d
)) {
1611 /* First try the NUMA node */
1612 if (node
!= NUMA_NO_NODE
) {
1614 * Try the intersection of the affinity mask and the
1615 * node mask (and the online mask, just to be safe).
1617 cpumask_and(tmpmask
, cpumask_of_node(node
), aff_mask
);
1618 cpumask_and(tmpmask
, tmpmask
, cpu_online_mask
);
1621 * Ideally, we would check if the mask is empty, and
1622 * try again on the full node here.
1624 * But it turns out that the way ACPI describes the
1625 * affinity for ITSs only deals about memory, and
1626 * not target CPUs, so it cannot describe a single
1627 * ITS placed next to two NUMA nodes.
1629 * Instead, just fallback on the online mask. This
1630 * diverges from Thomas' suggestion above.
1632 cpu
= cpumask_pick_least_loaded(d
, tmpmask
);
1633 if (cpu
< nr_cpu_ids
)
1636 /* If we can't cross sockets, give up */
1637 if ((its_dev
->its
->flags
& ITS_FLAGS_WORKAROUND_CAVIUM_23144
))
1640 /* If the above failed, expand the search */
1643 /* Try the intersection of the affinity and online masks */
1644 cpumask_and(tmpmask
, aff_mask
, cpu_online_mask
);
1646 /* If that doesn't fly, the online mask is the last resort */
1647 if (cpumask_empty(tmpmask
))
1648 cpumask_copy(tmpmask
, cpu_online_mask
);
1650 cpu
= cpumask_pick_least_loaded(d
, tmpmask
);
1652 cpumask_copy(tmpmask
, aff_mask
);
1654 /* If we cannot cross sockets, limit the search to that node */
1655 if ((its_dev
->its
->flags
& ITS_FLAGS_WORKAROUND_CAVIUM_23144
) &&
1656 node
!= NUMA_NO_NODE
)
1657 cpumask_and(tmpmask
, tmpmask
, cpumask_of_node(node
));
1659 cpu
= cpumask_pick_least_loaded(d
, tmpmask
);
1662 raw_spin_unlock_irqrestore(&tmpmask_lock
, flags
);
1664 pr_debug("IRQ%d -> %*pbl CPU%d\n", d
->irq
, cpumask_pr_args(aff_mask
), cpu
);
1668 static int its_set_affinity(struct irq_data
*d
, const struct cpumask
*mask_val
,
1671 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1672 struct its_collection
*target_col
;
1673 u32 id
= its_get_event_id(d
);
1676 /* A forwarded interrupt should use irq_set_vcpu_affinity */
1677 if (irqd_is_forwarded_to_vcpu(d
))
1680 prev_cpu
= its_dev
->event_map
.col_map
[id
];
1681 its_dec_lpi_count(d
, prev_cpu
);
1684 cpu
= its_select_cpu(d
, mask_val
);
1686 cpu
= cpumask_pick_least_loaded(d
, mask_val
);
1688 if (cpu
< 0 || cpu
>= nr_cpu_ids
)
1691 /* don't set the affinity when the target cpu is same as current one */
1692 if (cpu
!= prev_cpu
) {
1693 target_col
= &its_dev
->its
->collections
[cpu
];
1694 its_send_movi(its_dev
, target_col
, id
);
1695 its_dev
->event_map
.col_map
[id
] = cpu
;
1696 irq_data_update_effective_affinity(d
, cpumask_of(cpu
));
1699 its_inc_lpi_count(d
, cpu
);
1701 return IRQ_SET_MASK_OK_DONE
;
1704 its_inc_lpi_count(d
, prev_cpu
);
1708 static u64
its_irq_get_msi_base(struct its_device
*its_dev
)
1710 struct its_node
*its
= its_dev
->its
;
1712 return its
->phys_base
+ GITS_TRANSLATER
;
1715 static void its_irq_compose_msi_msg(struct irq_data
*d
, struct msi_msg
*msg
)
1717 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1718 struct its_node
*its
;
1722 addr
= its
->get_msi_base(its_dev
);
1724 msg
->address_lo
= lower_32_bits(addr
);
1725 msg
->address_hi
= upper_32_bits(addr
);
1726 msg
->data
= its_get_event_id(d
);
1728 iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d
), msg
);
1731 static int its_irq_set_irqchip_state(struct irq_data
*d
,
1732 enum irqchip_irq_state which
,
1735 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1736 u32 event
= its_get_event_id(d
);
1738 if (which
!= IRQCHIP_STATE_PENDING
)
1741 if (irqd_is_forwarded_to_vcpu(d
)) {
1743 its_send_vint(its_dev
, event
);
1745 its_send_vclear(its_dev
, event
);
1748 its_send_int(its_dev
, event
);
1750 its_send_clear(its_dev
, event
);
1756 static int its_irq_retrigger(struct irq_data
*d
)
1758 return !its_irq_set_irqchip_state(d
, IRQCHIP_STATE_PENDING
, true);
1762 * Two favourable cases:
1764 * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
1767 * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
1768 * and we're better off mapping all VPEs always
1770 * If neither (a) nor (b) is true, then we map vPEs on demand.
1773 static bool gic_requires_eager_mapping(void)
1775 if (!its_list_map
|| gic_rdists
->has_rvpeid
)
1781 static void its_map_vm(struct its_node
*its
, struct its_vm
*vm
)
1783 unsigned long flags
;
1785 if (gic_requires_eager_mapping())
1788 raw_spin_lock_irqsave(&vmovp_lock
, flags
);
1791 * If the VM wasn't mapped yet, iterate over the vpes and get
1794 vm
->vlpi_count
[its
->list_nr
]++;
1796 if (vm
->vlpi_count
[its
->list_nr
] == 1) {
1799 for (i
= 0; i
< vm
->nr_vpes
; i
++) {
1800 struct its_vpe
*vpe
= vm
->vpes
[i
];
1801 struct irq_data
*d
= irq_get_irq_data(vpe
->irq
);
1803 /* Map the VPE to the first possible CPU */
1804 vpe
->col_idx
= cpumask_first(cpu_online_mask
);
1805 its_send_vmapp(its
, vpe
, true);
1806 its_send_vinvall(its
, vpe
);
1807 irq_data_update_effective_affinity(d
, cpumask_of(vpe
->col_idx
));
1811 raw_spin_unlock_irqrestore(&vmovp_lock
, flags
);
1814 static void its_unmap_vm(struct its_node
*its
, struct its_vm
*vm
)
1816 unsigned long flags
;
1818 /* Not using the ITS list? Everything is always mapped. */
1819 if (gic_requires_eager_mapping())
1822 raw_spin_lock_irqsave(&vmovp_lock
, flags
);
1824 if (!--vm
->vlpi_count
[its
->list_nr
]) {
1827 for (i
= 0; i
< vm
->nr_vpes
; i
++)
1828 its_send_vmapp(its
, vm
->vpes
[i
], false);
1831 raw_spin_unlock_irqrestore(&vmovp_lock
, flags
);
1834 static int its_vlpi_map(struct irq_data
*d
, struct its_cmd_info
*info
)
1836 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1837 u32 event
= its_get_event_id(d
);
1843 raw_spin_lock(&its_dev
->event_map
.vlpi_lock
);
1845 if (!its_dev
->event_map
.vm
) {
1846 struct its_vlpi_map
*maps
;
1848 maps
= kcalloc(its_dev
->event_map
.nr_lpis
, sizeof(*maps
),
1855 its_dev
->event_map
.vm
= info
->map
->vm
;
1856 its_dev
->event_map
.vlpi_maps
= maps
;
1857 } else if (its_dev
->event_map
.vm
!= info
->map
->vm
) {
1862 /* Get our private copy of the mapping information */
1863 its_dev
->event_map
.vlpi_maps
[event
] = *info
->map
;
1865 if (irqd_is_forwarded_to_vcpu(d
)) {
1866 /* Already mapped, move it around */
1867 its_send_vmovi(its_dev
, event
);
1869 /* Ensure all the VPEs are mapped on this ITS */
1870 its_map_vm(its_dev
->its
, info
->map
->vm
);
1873 * Flag the interrupt as forwarded so that we can
1874 * start poking the virtual property table.
1876 irqd_set_forwarded_to_vcpu(d
);
1878 /* Write out the property to the prop table */
1879 lpi_write_config(d
, 0xff, info
->map
->properties
);
1881 /* Drop the physical mapping */
1882 its_send_discard(its_dev
, event
);
1884 /* and install the virtual one */
1885 its_send_vmapti(its_dev
, event
);
1887 /* Increment the number of VLPIs */
1888 its_dev
->event_map
.nr_vlpis
++;
1892 raw_spin_unlock(&its_dev
->event_map
.vlpi_lock
);
1896 static int its_vlpi_get(struct irq_data
*d
, struct its_cmd_info
*info
)
1898 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1899 struct its_vlpi_map
*map
;
1902 raw_spin_lock(&its_dev
->event_map
.vlpi_lock
);
1904 map
= get_vlpi_map(d
);
1906 if (!its_dev
->event_map
.vm
|| !map
) {
1911 /* Copy our mapping information to the incoming request */
1915 raw_spin_unlock(&its_dev
->event_map
.vlpi_lock
);
1919 static int its_vlpi_unmap(struct irq_data
*d
)
1921 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1922 u32 event
= its_get_event_id(d
);
1925 raw_spin_lock(&its_dev
->event_map
.vlpi_lock
);
1927 if (!its_dev
->event_map
.vm
|| !irqd_is_forwarded_to_vcpu(d
)) {
1932 /* Drop the virtual mapping */
1933 its_send_discard(its_dev
, event
);
1935 /* and restore the physical one */
1936 irqd_clr_forwarded_to_vcpu(d
);
1937 its_send_mapti(its_dev
, d
->hwirq
, event
);
1938 lpi_update_config(d
, 0xff, (LPI_PROP_DEFAULT_PRIO
|
1942 /* Potentially unmap the VM from this ITS */
1943 its_unmap_vm(its_dev
->its
, its_dev
->event_map
.vm
);
1946 * Drop the refcount and make the device available again if
1947 * this was the last VLPI.
1949 if (!--its_dev
->event_map
.nr_vlpis
) {
1950 its_dev
->event_map
.vm
= NULL
;
1951 kfree(its_dev
->event_map
.vlpi_maps
);
1955 raw_spin_unlock(&its_dev
->event_map
.vlpi_lock
);
1959 static int its_vlpi_prop_update(struct irq_data
*d
, struct its_cmd_info
*info
)
1961 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1963 if (!its_dev
->event_map
.vm
|| !irqd_is_forwarded_to_vcpu(d
))
1966 if (info
->cmd_type
== PROP_UPDATE_AND_INV_VLPI
)
1967 lpi_update_config(d
, 0xff, info
->config
);
1969 lpi_write_config(d
, 0xff, info
->config
);
1970 its_vlpi_set_doorbell(d
, !!(info
->config
& LPI_PROP_ENABLED
));
1975 static int its_irq_set_vcpu_affinity(struct irq_data
*d
, void *vcpu_info
)
1977 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1978 struct its_cmd_info
*info
= vcpu_info
;
1981 if (!is_v4(its_dev
->its
))
1984 /* Unmap request? */
1986 return its_vlpi_unmap(d
);
1988 switch (info
->cmd_type
) {
1990 return its_vlpi_map(d
, info
);
1993 return its_vlpi_get(d
, info
);
1995 case PROP_UPDATE_VLPI
:
1996 case PROP_UPDATE_AND_INV_VLPI
:
1997 return its_vlpi_prop_update(d
, info
);
2004 static struct irq_chip its_irq_chip
= {
2006 .irq_mask
= its_mask_irq
,
2007 .irq_unmask
= its_unmask_irq
,
2008 .irq_eoi
= irq_chip_eoi_parent
,
2009 .irq_set_affinity
= its_set_affinity
,
2010 .irq_compose_msi_msg
= its_irq_compose_msi_msg
,
2011 .irq_set_irqchip_state
= its_irq_set_irqchip_state
,
2012 .irq_retrigger
= its_irq_retrigger
,
2013 .irq_set_vcpu_affinity
= its_irq_set_vcpu_affinity
,
2018 * How we allocate LPIs:
2020 * lpi_range_list contains ranges of LPIs that are to available to
2021 * allocate from. To allocate LPIs, just pick the first range that
2022 * fits the required allocation, and reduce it by the required
2023 * amount. Once empty, remove the range from the list.
2025 * To free a range of LPIs, add a free range to the list, sort it and
2026 * merge the result if the new range happens to be adjacent to an
2027 * already free block.
2029 * The consequence of the above is that allocation is cost is low, but
2030 * freeing is expensive. We assumes that freeing rarely occurs.
2032 #define ITS_MAX_LPI_NRBITS 16 /* 64K LPIs */
2034 static DEFINE_MUTEX(lpi_range_lock
);
2035 static LIST_HEAD(lpi_range_list
);
2038 struct list_head entry
;
2043 static struct lpi_range
*mk_lpi_range(u32 base
, u32 span
)
2045 struct lpi_range
*range
;
2047 range
= kmalloc(sizeof(*range
), GFP_KERNEL
);
2049 range
->base_id
= base
;
2056 static int alloc_lpi_range(u32 nr_lpis
, u32
*base
)
2058 struct lpi_range
*range
, *tmp
;
2061 mutex_lock(&lpi_range_lock
);
2063 list_for_each_entry_safe(range
, tmp
, &lpi_range_list
, entry
) {
2064 if (range
->span
>= nr_lpis
) {
2065 *base
= range
->base_id
;
2066 range
->base_id
+= nr_lpis
;
2067 range
->span
-= nr_lpis
;
2069 if (range
->span
== 0) {
2070 list_del(&range
->entry
);
2079 mutex_unlock(&lpi_range_lock
);
2081 pr_debug("ITS: alloc %u:%u\n", *base
, nr_lpis
);
2085 static void merge_lpi_ranges(struct lpi_range
*a
, struct lpi_range
*b
)
2087 if (&a
->entry
== &lpi_range_list
|| &b
->entry
== &lpi_range_list
)
2089 if (a
->base_id
+ a
->span
!= b
->base_id
)
2091 b
->base_id
= a
->base_id
;
2093 list_del(&a
->entry
);
2097 static int free_lpi_range(u32 base
, u32 nr_lpis
)
2099 struct lpi_range
*new, *old
;
2101 new = mk_lpi_range(base
, nr_lpis
);
2105 mutex_lock(&lpi_range_lock
);
2107 list_for_each_entry_reverse(old
, &lpi_range_list
, entry
) {
2108 if (old
->base_id
< base
)
2112 * old is the last element with ->base_id smaller than base,
2113 * so new goes right after it. If there are no elements with
2114 * ->base_id smaller than base, &old->entry ends up pointing
2115 * at the head of the list, and inserting new it the start of
2116 * the list is the right thing to do in that case as well.
2118 list_add(&new->entry
, &old
->entry
);
2120 * Now check if we can merge with the preceding and/or
2123 merge_lpi_ranges(old
, new);
2124 merge_lpi_ranges(new, list_next_entry(new, entry
));
2126 mutex_unlock(&lpi_range_lock
);
2130 static int __init
its_lpi_init(u32 id_bits
)
2132 u32 lpis
= (1UL << id_bits
) - 8192;
2136 numlpis
= 1UL << GICD_TYPER_NUM_LPIS(gic_rdists
->gicd_typer
);
2138 if (numlpis
> 2 && !WARN_ON(numlpis
> lpis
)) {
2140 pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
2145 * Initializing the allocator is just the same as freeing the
2146 * full range of LPIs.
2148 err
= free_lpi_range(8192, lpis
);
2149 pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis
);
2153 static unsigned long *its_lpi_alloc(int nr_irqs
, u32
*base
, int *nr_ids
)
2155 unsigned long *bitmap
= NULL
;
2159 err
= alloc_lpi_range(nr_irqs
, base
);
2164 } while (nr_irqs
> 0);
2172 bitmap
= bitmap_zalloc(nr_irqs
, GFP_ATOMIC
);
2180 *base
= *nr_ids
= 0;
2185 static void its_lpi_free(unsigned long *bitmap
, u32 base
, u32 nr_ids
)
2187 WARN_ON(free_lpi_range(base
, nr_ids
));
2188 bitmap_free(bitmap
);
2191 static void gic_reset_prop_table(void *va
)
2193 /* Priority 0xa0, Group-1, disabled */
2194 memset(va
, LPI_PROP_DEFAULT_PRIO
| LPI_PROP_GROUP1
, LPI_PROPBASE_SZ
);
2196 /* Make sure the GIC will observe the written configuration */
2197 gic_flush_dcache_to_poc(va
, LPI_PROPBASE_SZ
);
2200 static struct page
*its_allocate_prop_table(gfp_t gfp_flags
)
2202 struct page
*prop_page
;
2204 prop_page
= alloc_pages(gfp_flags
, get_order(LPI_PROPBASE_SZ
));
2208 gic_reset_prop_table(page_address(prop_page
));
2213 static void its_free_prop_table(struct page
*prop_page
)
2215 free_pages((unsigned long)page_address(prop_page
),
2216 get_order(LPI_PROPBASE_SZ
));
2219 static bool gic_check_reserved_range(phys_addr_t addr
, unsigned long size
)
2221 phys_addr_t start
, end
, addr_end
;
2225 * We don't bother checking for a kdump kernel as by
2226 * construction, the LPI tables are out of this kernel's
2229 if (is_kdump_kernel())
2232 addr_end
= addr
+ size
- 1;
2234 for_each_reserved_mem_range(i
, &start
, &end
) {
2235 if (addr
>= start
&& addr_end
<= end
)
2239 /* Not found, not a good sign... */
2240 pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
2242 add_taint(TAINT_CRAP
, LOCKDEP_STILL_OK
);
2246 static int gic_reserve_range(phys_addr_t addr
, unsigned long size
)
2248 if (efi_enabled(EFI_CONFIG_TABLES
))
2249 return efi_mem_reserve_persistent(addr
, size
);
2254 static int __init
its_setup_lpi_prop_table(void)
2256 if (gic_rdists
->flags
& RDIST_FLAGS_RD_TABLES_PREALLOCATED
) {
2259 val
= gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER
);
2260 lpi_id_bits
= (val
& GICR_PROPBASER_IDBITS_MASK
) + 1;
2262 gic_rdists
->prop_table_pa
= val
& GENMASK_ULL(51, 12);
2263 gic_rdists
->prop_table_va
= memremap(gic_rdists
->prop_table_pa
,
2266 gic_reset_prop_table(gic_rdists
->prop_table_va
);
2270 lpi_id_bits
= min_t(u32
,
2271 GICD_TYPER_ID_BITS(gic_rdists
->gicd_typer
),
2272 ITS_MAX_LPI_NRBITS
);
2273 page
= its_allocate_prop_table(GFP_NOWAIT
);
2275 pr_err("Failed to allocate PROPBASE\n");
2279 gic_rdists
->prop_table_pa
= page_to_phys(page
);
2280 gic_rdists
->prop_table_va
= page_address(page
);
2281 WARN_ON(gic_reserve_range(gic_rdists
->prop_table_pa
,
2285 pr_info("GICv3: using LPI property table @%pa\n",
2286 &gic_rdists
->prop_table_pa
);
2288 return its_lpi_init(lpi_id_bits
);
2291 static const char *its_base_type_string
[] = {
2292 [GITS_BASER_TYPE_DEVICE
] = "Devices",
2293 [GITS_BASER_TYPE_VCPU
] = "Virtual CPUs",
2294 [GITS_BASER_TYPE_RESERVED3
] = "Reserved (3)",
2295 [GITS_BASER_TYPE_COLLECTION
] = "Interrupt Collections",
2296 [GITS_BASER_TYPE_RESERVED5
] = "Reserved (5)",
2297 [GITS_BASER_TYPE_RESERVED6
] = "Reserved (6)",
2298 [GITS_BASER_TYPE_RESERVED7
] = "Reserved (7)",
2301 static u64
its_read_baser(struct its_node
*its
, struct its_baser
*baser
)
2303 u32 idx
= baser
- its
->tables
;
2305 return gits_read_baser(its
->base
+ GITS_BASER
+ (idx
<< 3));
2308 static void its_write_baser(struct its_node
*its
, struct its_baser
*baser
,
2311 u32 idx
= baser
- its
->tables
;
2313 gits_write_baser(val
, its
->base
+ GITS_BASER
+ (idx
<< 3));
2314 baser
->val
= its_read_baser(its
, baser
);
2317 static int its_setup_baser(struct its_node
*its
, struct its_baser
*baser
,
2318 u64 cache
, u64 shr
, u32 order
, bool indirect
)
2320 u64 val
= its_read_baser(its
, baser
);
2321 u64 esz
= GITS_BASER_ENTRY_SIZE(val
);
2322 u64 type
= GITS_BASER_TYPE(val
);
2323 u64 baser_phys
, tmp
;
2324 u32 alloc_pages
, psz
;
2329 alloc_pages
= (PAGE_ORDER_TO_SIZE(order
) / psz
);
2330 if (alloc_pages
> GITS_BASER_PAGES_MAX
) {
2331 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
2332 &its
->phys_base
, its_base_type_string
[type
],
2333 alloc_pages
, GITS_BASER_PAGES_MAX
);
2334 alloc_pages
= GITS_BASER_PAGES_MAX
;
2335 order
= get_order(GITS_BASER_PAGES_MAX
* psz
);
2338 page
= alloc_pages_node(its
->numa_node
, GFP_KERNEL
| __GFP_ZERO
, order
);
2342 base
= (void *)page_address(page
);
2343 baser_phys
= virt_to_phys(base
);
2345 /* Check if the physical address of the memory is above 48bits */
2346 if (IS_ENABLED(CONFIG_ARM64_64K_PAGES
) && (baser_phys
>> 48)) {
2348 /* 52bit PA is supported only when PageSize=64K */
2349 if (psz
!= SZ_64K
) {
2350 pr_err("ITS: no 52bit PA support when psz=%d\n", psz
);
2351 free_pages((unsigned long)base
, order
);
2355 /* Convert 52bit PA to 48bit field */
2356 baser_phys
= GITS_BASER_PHYS_52_to_48(baser_phys
);
2361 (type
<< GITS_BASER_TYPE_SHIFT
) |
2362 ((esz
- 1) << GITS_BASER_ENTRY_SIZE_SHIFT
) |
2363 ((alloc_pages
- 1) << GITS_BASER_PAGES_SHIFT
) |
2368 val
|= indirect
? GITS_BASER_INDIRECT
: 0x0;
2372 val
|= GITS_BASER_PAGE_SIZE_4K
;
2375 val
|= GITS_BASER_PAGE_SIZE_16K
;
2378 val
|= GITS_BASER_PAGE_SIZE_64K
;
2383 gic_flush_dcache_to_poc(base
, PAGE_ORDER_TO_SIZE(order
));
2385 its_write_baser(its
, baser
, val
);
2388 if ((val
^ tmp
) & GITS_BASER_SHAREABILITY_MASK
) {
2390 * Shareability didn't stick. Just use
2391 * whatever the read reported, which is likely
2392 * to be the only thing this redistributor
2393 * supports. If that's zero, make it
2394 * non-cacheable as well.
2396 shr
= tmp
& GITS_BASER_SHAREABILITY_MASK
;
2398 cache
= GITS_BASER_nC
;
2404 pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
2405 &its
->phys_base
, its_base_type_string
[type
],
2407 free_pages((unsigned long)base
, order
);
2411 baser
->order
= order
;
2414 tmp
= indirect
? GITS_LVL1_ENTRY_SIZE
: esz
;
2416 pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
2417 &its
->phys_base
, (int)(PAGE_ORDER_TO_SIZE(order
) / (int)tmp
),
2418 its_base_type_string
[type
],
2419 (unsigned long)virt_to_phys(base
),
2420 indirect
? "indirect" : "flat", (int)esz
,
2421 psz
/ SZ_1K
, (int)shr
>> GITS_BASER_SHAREABILITY_SHIFT
);
2426 static bool its_parse_indirect_baser(struct its_node
*its
,
2427 struct its_baser
*baser
,
2428 u32
*order
, u32 ids
)
2430 u64 tmp
= its_read_baser(its
, baser
);
2431 u64 type
= GITS_BASER_TYPE(tmp
);
2432 u64 esz
= GITS_BASER_ENTRY_SIZE(tmp
);
2433 u64 val
= GITS_BASER_InnerShareable
| GITS_BASER_RaWaWb
;
2434 u32 new_order
= *order
;
2435 u32 psz
= baser
->psz
;
2436 bool indirect
= false;
2438 /* No need to enable Indirection if memory requirement < (psz*2)bytes */
2439 if ((esz
<< ids
) > (psz
* 2)) {
2441 * Find out whether hw supports a single or two-level table by
2442 * table by reading bit at offset '62' after writing '1' to it.
2444 its_write_baser(its
, baser
, val
| GITS_BASER_INDIRECT
);
2445 indirect
= !!(baser
->val
& GITS_BASER_INDIRECT
);
2449 * The size of the lvl2 table is equal to ITS page size
2450 * which is 'psz'. For computing lvl1 table size,
2451 * subtract ID bits that sparse lvl2 table from 'ids'
2452 * which is reported by ITS hardware times lvl1 table
2455 ids
-= ilog2(psz
/ (int)esz
);
2456 esz
= GITS_LVL1_ENTRY_SIZE
;
2461 * Allocate as many entries as required to fit the
2462 * range of device IDs that the ITS can grok... The ID
2463 * space being incredibly sparse, this results in a
2464 * massive waste of memory if two-level device table
2465 * feature is not supported by hardware.
2467 new_order
= max_t(u32
, get_order(esz
<< ids
), new_order
);
2468 if (new_order
> MAX_ORDER
) {
2469 new_order
= MAX_ORDER
;
2470 ids
= ilog2(PAGE_ORDER_TO_SIZE(new_order
) / (int)esz
);
2471 pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
2472 &its
->phys_base
, its_base_type_string
[type
],
2473 device_ids(its
), ids
);
2481 static u32
compute_common_aff(u64 val
)
2485 aff
= FIELD_GET(GICR_TYPER_AFFINITY
, val
);
2486 clpiaff
= FIELD_GET(GICR_TYPER_COMMON_LPI_AFF
, val
);
2488 return aff
& ~(GENMASK(31, 0) >> (clpiaff
* 8));
2491 static u32
compute_its_aff(struct its_node
*its
)
2497 * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
2498 * the resulting affinity. We then use that to see if this match
2501 svpet
= FIELD_GET(GITS_TYPER_SVPET
, its
->typer
);
2502 val
= FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF
, svpet
);
2503 val
|= FIELD_PREP(GICR_TYPER_AFFINITY
, its
->mpidr
);
2504 return compute_common_aff(val
);
2507 static struct its_node
*find_sibling_its(struct its_node
*cur_its
)
2509 struct its_node
*its
;
2512 if (!FIELD_GET(GITS_TYPER_SVPET
, cur_its
->typer
))
2515 aff
= compute_its_aff(cur_its
);
2517 list_for_each_entry(its
, &its_nodes
, entry
) {
2520 if (!is_v4_1(its
) || its
== cur_its
)
2523 if (!FIELD_GET(GITS_TYPER_SVPET
, its
->typer
))
2526 if (aff
!= compute_its_aff(its
))
2529 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2530 baser
= its
->tables
[2].val
;
2531 if (!(baser
& GITS_BASER_VALID
))
2540 static void its_free_tables(struct its_node
*its
)
2544 for (i
= 0; i
< GITS_BASER_NR_REGS
; i
++) {
2545 if (its
->tables
[i
].base
) {
2546 free_pages((unsigned long)its
->tables
[i
].base
,
2547 its
->tables
[i
].order
);
2548 its
->tables
[i
].base
= NULL
;
2553 static int its_probe_baser_psz(struct its_node
*its
, struct its_baser
*baser
)
2560 val
= its_read_baser(its
, baser
);
2561 val
&= ~GITS_BASER_PAGE_SIZE_MASK
;
2565 gpsz
= GITS_BASER_PAGE_SIZE_64K
;
2568 gpsz
= GITS_BASER_PAGE_SIZE_16K
;
2572 gpsz
= GITS_BASER_PAGE_SIZE_4K
;
2576 gpsz
>>= GITS_BASER_PAGE_SIZE_SHIFT
;
2578 val
|= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK
, gpsz
);
2579 its_write_baser(its
, baser
, val
);
2581 if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK
, baser
->val
) == gpsz
)
2601 static int its_alloc_tables(struct its_node
*its
)
2603 u64 shr
= GITS_BASER_InnerShareable
;
2604 u64 cache
= GITS_BASER_RaWaWb
;
2607 if (its
->flags
& ITS_FLAGS_WORKAROUND_CAVIUM_22375
)
2608 /* erratum 24313: ignore memory access type */
2609 cache
= GITS_BASER_nCnB
;
2611 if (its
->flags
& ITS_FLAGS_FORCE_NON_SHAREABLE
) {
2612 cache
= GITS_BASER_nC
;
2616 for (i
= 0; i
< GITS_BASER_NR_REGS
; i
++) {
2617 struct its_baser
*baser
= its
->tables
+ i
;
2618 u64 val
= its_read_baser(its
, baser
);
2619 u64 type
= GITS_BASER_TYPE(val
);
2620 bool indirect
= false;
2623 if (type
== GITS_BASER_TYPE_NONE
)
2626 if (its_probe_baser_psz(its
, baser
)) {
2627 its_free_tables(its
);
2631 order
= get_order(baser
->psz
);
2634 case GITS_BASER_TYPE_DEVICE
:
2635 indirect
= its_parse_indirect_baser(its
, baser
, &order
,
2639 case GITS_BASER_TYPE_VCPU
:
2641 struct its_node
*sibling
;
2644 if ((sibling
= find_sibling_its(its
))) {
2645 *baser
= sibling
->tables
[2];
2646 its_write_baser(its
, baser
, baser
->val
);
2651 indirect
= its_parse_indirect_baser(its
, baser
, &order
,
2652 ITS_MAX_VPEID_BITS
);
2656 err
= its_setup_baser(its
, baser
, cache
, shr
, order
, indirect
);
2658 its_free_tables(its
);
2662 /* Update settings which will be used for next BASERn */
2663 cache
= baser
->val
& GITS_BASER_CACHEABILITY_MASK
;
2664 shr
= baser
->val
& GITS_BASER_SHAREABILITY_MASK
;
2670 static u64
inherit_vpe_l1_table_from_its(void)
2672 struct its_node
*its
;
2676 val
= gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER
);
2677 aff
= compute_common_aff(val
);
2679 list_for_each_entry(its
, &its_nodes
, entry
) {
2685 if (!FIELD_GET(GITS_TYPER_SVPET
, its
->typer
))
2688 if (aff
!= compute_its_aff(its
))
2691 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2692 baser
= its
->tables
[2].val
;
2693 if (!(baser
& GITS_BASER_VALID
))
2696 /* We have a winner! */
2697 gic_data_rdist()->vpe_l1_base
= its
->tables
[2].base
;
2699 val
= GICR_VPROPBASER_4_1_VALID
;
2700 if (baser
& GITS_BASER_INDIRECT
)
2701 val
|= GICR_VPROPBASER_4_1_INDIRECT
;
2702 val
|= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE
,
2703 FIELD_GET(GITS_BASER_PAGE_SIZE_MASK
, baser
));
2704 switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK
, baser
)) {
2705 case GIC_PAGE_SIZE_64K
:
2706 addr
= GITS_BASER_ADDR_48_to_52(baser
);
2709 addr
= baser
& GENMASK_ULL(47, 12);
2712 val
|= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR
, addr
>> 12);
2713 val
|= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK
,
2714 FIELD_GET(GITS_BASER_SHAREABILITY_MASK
, baser
));
2715 val
|= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK
,
2716 FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK
, baser
));
2717 val
|= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE
, GITS_BASER_NR_PAGES(baser
) - 1);
2725 static u64
inherit_vpe_l1_table_from_rd(cpumask_t
**mask
)
2731 val
= gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER
);
2732 aff
= compute_common_aff(val
);
2734 for_each_possible_cpu(cpu
) {
2735 void __iomem
*base
= gic_data_rdist_cpu(cpu
)->rd_base
;
2737 if (!base
|| cpu
== smp_processor_id())
2740 val
= gic_read_typer(base
+ GICR_TYPER
);
2741 if (aff
!= compute_common_aff(val
))
2745 * At this point, we have a victim. This particular CPU
2746 * has already booted, and has an affinity that matches
2747 * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
2748 * Make sure we don't write the Z bit in that case.
2750 val
= gicr_read_vpropbaser(base
+ SZ_128K
+ GICR_VPROPBASER
);
2751 val
&= ~GICR_VPROPBASER_4_1_Z
;
2753 gic_data_rdist()->vpe_l1_base
= gic_data_rdist_cpu(cpu
)->vpe_l1_base
;
2754 *mask
= gic_data_rdist_cpu(cpu
)->vpe_table_mask
;
2762 static bool allocate_vpe_l2_table(int cpu
, u32 id
)
2764 void __iomem
*base
= gic_data_rdist_cpu(cpu
)->rd_base
;
2765 unsigned int psz
, esz
, idx
, npg
, gpsz
;
2770 if (!gic_rdists
->has_rvpeid
)
2773 /* Skip non-present CPUs */
2777 val
= gicr_read_vpropbaser(base
+ SZ_128K
+ GICR_VPROPBASER
);
2779 esz
= FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE
, val
) + 1;
2780 gpsz
= FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE
, val
);
2781 npg
= FIELD_GET(GICR_VPROPBASER_4_1_SIZE
, val
) + 1;
2787 case GIC_PAGE_SIZE_4K
:
2790 case GIC_PAGE_SIZE_16K
:
2793 case GIC_PAGE_SIZE_64K
:
2798 /* Don't allow vpe_id that exceeds single, flat table limit */
2799 if (!(val
& GICR_VPROPBASER_4_1_INDIRECT
))
2800 return (id
< (npg
* psz
/ (esz
* SZ_8
)));
2802 /* Compute 1st level table index & check if that exceeds table limit */
2803 idx
= id
>> ilog2(psz
/ (esz
* SZ_8
));
2804 if (idx
>= (npg
* psz
/ GITS_LVL1_ENTRY_SIZE
))
2807 table
= gic_data_rdist_cpu(cpu
)->vpe_l1_base
;
2809 /* Allocate memory for 2nd level table */
2811 page
= alloc_pages(GFP_KERNEL
| __GFP_ZERO
, get_order(psz
));
2815 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
2816 if (!(val
& GICR_VPROPBASER_SHAREABILITY_MASK
))
2817 gic_flush_dcache_to_poc(page_address(page
), psz
);
2819 table
[idx
] = cpu_to_le64(page_to_phys(page
) | GITS_BASER_VALID
);
2821 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
2822 if (!(val
& GICR_VPROPBASER_SHAREABILITY_MASK
))
2823 gic_flush_dcache_to_poc(table
+ idx
, GITS_LVL1_ENTRY_SIZE
);
2825 /* Ensure updated table contents are visible to RD hardware */
2832 static int allocate_vpe_l1_table(void)
2834 void __iomem
*vlpi_base
= gic_data_rdist_vlpi_base();
2835 u64 val
, gpsz
, npg
, pa
;
2836 unsigned int psz
= SZ_64K
;
2837 unsigned int np
, epp
, esz
;
2840 if (!gic_rdists
->has_rvpeid
)
2844 * if VPENDBASER.Valid is set, disable any previously programmed
2845 * VPE by setting PendingLast while clearing Valid. This has the
2846 * effect of making sure no doorbell will be generated and we can
2847 * then safely clear VPROPBASER.Valid.
2849 if (gicr_read_vpendbaser(vlpi_base
+ GICR_VPENDBASER
) & GICR_VPENDBASER_Valid
)
2850 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast
,
2851 vlpi_base
+ GICR_VPENDBASER
);
2854 * If we can inherit the configuration from another RD, let's do
2855 * so. Otherwise, we have to go through the allocation process. We
2856 * assume that all RDs have the exact same requirements, as
2857 * nothing will work otherwise.
2859 val
= inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask
);
2860 if (val
& GICR_VPROPBASER_4_1_VALID
)
2863 gic_data_rdist()->vpe_table_mask
= kzalloc(sizeof(cpumask_t
), GFP_ATOMIC
);
2864 if (!gic_data_rdist()->vpe_table_mask
)
2867 val
= inherit_vpe_l1_table_from_its();
2868 if (val
& GICR_VPROPBASER_4_1_VALID
)
2871 /* First probe the page size */
2872 val
= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE
, GIC_PAGE_SIZE_64K
);
2873 gicr_write_vpropbaser(val
, vlpi_base
+ GICR_VPROPBASER
);
2874 val
= gicr_read_vpropbaser(vlpi_base
+ GICR_VPROPBASER
);
2875 gpsz
= FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE
, val
);
2876 esz
= FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE
, val
);
2880 gpsz
= GIC_PAGE_SIZE_4K
;
2882 case GIC_PAGE_SIZE_4K
:
2885 case GIC_PAGE_SIZE_16K
:
2888 case GIC_PAGE_SIZE_64K
:
2894 * Start populating the register from scratch, including RO fields
2895 * (which we want to print in debug cases...)
2898 val
|= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE
, gpsz
);
2899 val
|= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE
, esz
);
2901 /* How many entries per GIC page? */
2903 epp
= psz
/ (esz
* SZ_8
);
2906 * If we need more than just a single L1 page, flag the table
2907 * as indirect and compute the number of required L1 pages.
2909 if (epp
< ITS_MAX_VPEID
) {
2912 val
|= GICR_VPROPBASER_4_1_INDIRECT
;
2914 /* Number of L2 pages required to cover the VPEID space */
2915 nl2
= DIV_ROUND_UP(ITS_MAX_VPEID
, epp
);
2917 /* Number of L1 pages to point to the L2 pages */
2918 npg
= DIV_ROUND_UP(nl2
* SZ_8
, psz
);
2923 val
|= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE
, npg
- 1);
2925 /* Right, that's the number of CPU pages we need for L1 */
2926 np
= DIV_ROUND_UP(npg
* psz
, PAGE_SIZE
);
2928 pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
2929 np
, npg
, psz
, epp
, esz
);
2930 page
= alloc_pages(GFP_ATOMIC
| __GFP_ZERO
, get_order(np
* PAGE_SIZE
));
2934 gic_data_rdist()->vpe_l1_base
= page_address(page
);
2935 pa
= virt_to_phys(page_address(page
));
2936 WARN_ON(!IS_ALIGNED(pa
, psz
));
2938 val
|= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR
, pa
>> 12);
2939 val
|= GICR_VPROPBASER_RaWb
;
2940 val
|= GICR_VPROPBASER_InnerShareable
;
2941 val
|= GICR_VPROPBASER_4_1_Z
;
2942 val
|= GICR_VPROPBASER_4_1_VALID
;
2945 gicr_write_vpropbaser(val
, vlpi_base
+ GICR_VPROPBASER
);
2946 cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask
);
2948 pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
2949 smp_processor_id(), val
,
2950 cpumask_pr_args(gic_data_rdist()->vpe_table_mask
));
2955 static int its_alloc_collections(struct its_node
*its
)
2959 its
->collections
= kcalloc(nr_cpu_ids
, sizeof(*its
->collections
),
2961 if (!its
->collections
)
2964 for (i
= 0; i
< nr_cpu_ids
; i
++)
2965 its
->collections
[i
].target_address
= ~0ULL;
2970 static struct page
*its_allocate_pending_table(gfp_t gfp_flags
)
2972 struct page
*pend_page
;
2974 pend_page
= alloc_pages(gfp_flags
| __GFP_ZERO
,
2975 get_order(LPI_PENDBASE_SZ
));
2979 /* Make sure the GIC will observe the zero-ed page */
2980 gic_flush_dcache_to_poc(page_address(pend_page
), LPI_PENDBASE_SZ
);
2985 static void its_free_pending_table(struct page
*pt
)
2987 free_pages((unsigned long)page_address(pt
), get_order(LPI_PENDBASE_SZ
));
2991 * Booting with kdump and LPIs enabled is generally fine. Any other
2992 * case is wrong in the absence of firmware/EFI support.
2994 static bool enabled_lpis_allowed(void)
2999 /* Check whether the property table is in a reserved region */
3000 val
= gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER
);
3001 addr
= val
& GENMASK_ULL(51, 12);
3003 return gic_check_reserved_range(addr
, LPI_PROPBASE_SZ
);
3006 static int __init
allocate_lpi_tables(void)
3012 * If LPIs are enabled while we run this from the boot CPU,
3013 * flag the RD tables as pre-allocated if the stars do align.
3015 val
= readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR
);
3016 if ((val
& GICR_CTLR_ENABLE_LPIS
) && enabled_lpis_allowed()) {
3017 gic_rdists
->flags
|= (RDIST_FLAGS_RD_TABLES_PREALLOCATED
|
3018 RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING
);
3019 pr_info("GICv3: Using preallocated redistributor tables\n");
3022 err
= its_setup_lpi_prop_table();
3027 * We allocate all the pending tables anyway, as we may have a
3028 * mix of RDs that have had LPIs enabled, and some that
3029 * don't. We'll free the unused ones as each CPU comes online.
3031 for_each_possible_cpu(cpu
) {
3032 struct page
*pend_page
;
3034 pend_page
= its_allocate_pending_table(GFP_NOWAIT
);
3036 pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu
);
3040 gic_data_rdist_cpu(cpu
)->pend_page
= pend_page
;
3046 static u64
read_vpend_dirty_clear(void __iomem
*vlpi_base
)
3048 u32 count
= 1000000; /* 1s! */
3053 val
= gicr_read_vpendbaser(vlpi_base
+ GICR_VPENDBASER
);
3054 clean
= !(val
& GICR_VPENDBASER_Dirty
);
3060 } while (!clean
&& count
);
3062 if (unlikely(!clean
))
3063 pr_err_ratelimited("ITS virtual pending table not cleaning\n");
3068 static u64
its_clear_vpend_valid(void __iomem
*vlpi_base
, u64 clr
, u64 set
)
3072 /* Make sure we wait until the RD is done with the initial scan */
3073 val
= read_vpend_dirty_clear(vlpi_base
);
3074 val
&= ~GICR_VPENDBASER_Valid
;
3077 gicr_write_vpendbaser(val
, vlpi_base
+ GICR_VPENDBASER
);
3079 val
= read_vpend_dirty_clear(vlpi_base
);
3080 if (unlikely(val
& GICR_VPENDBASER_Dirty
))
3081 val
|= GICR_VPENDBASER_PendingLast
;
3086 static void its_cpu_init_lpis(void)
3088 void __iomem
*rbase
= gic_data_rdist_rd_base();
3089 struct page
*pend_page
;
3093 if (gic_data_rdist()->flags
& RD_LOCAL_LPI_ENABLED
)
3096 val
= readl_relaxed(rbase
+ GICR_CTLR
);
3097 if ((gic_rdists
->flags
& RDIST_FLAGS_RD_TABLES_PREALLOCATED
) &&
3098 (val
& GICR_CTLR_ENABLE_LPIS
)) {
3100 * Check that we get the same property table on all
3101 * RDs. If we don't, this is hopeless.
3103 paddr
= gicr_read_propbaser(rbase
+ GICR_PROPBASER
);
3104 paddr
&= GENMASK_ULL(51, 12);
3105 if (WARN_ON(gic_rdists
->prop_table_pa
!= paddr
))
3106 add_taint(TAINT_CRAP
, LOCKDEP_STILL_OK
);
3108 paddr
= gicr_read_pendbaser(rbase
+ GICR_PENDBASER
);
3109 paddr
&= GENMASK_ULL(51, 16);
3111 WARN_ON(!gic_check_reserved_range(paddr
, LPI_PENDBASE_SZ
));
3112 gic_data_rdist()->flags
|= RD_LOCAL_PENDTABLE_PREALLOCATED
;
3117 pend_page
= gic_data_rdist()->pend_page
;
3118 paddr
= page_to_phys(pend_page
);
3121 val
= (gic_rdists
->prop_table_pa
|
3122 GICR_PROPBASER_InnerShareable
|
3123 GICR_PROPBASER_RaWaWb
|
3124 ((LPI_NRBITS
- 1) & GICR_PROPBASER_IDBITS_MASK
));
3126 gicr_write_propbaser(val
, rbase
+ GICR_PROPBASER
);
3127 tmp
= gicr_read_propbaser(rbase
+ GICR_PROPBASER
);
3129 if (gic_rdists
->flags
& RDIST_FLAGS_FORCE_NON_SHAREABLE
)
3130 tmp
&= ~GICR_PROPBASER_SHAREABILITY_MASK
;
3132 if ((tmp
^ val
) & GICR_PROPBASER_SHAREABILITY_MASK
) {
3133 if (!(tmp
& GICR_PROPBASER_SHAREABILITY_MASK
)) {
3135 * The HW reports non-shareable, we must
3136 * remove the cacheability attributes as
3139 val
&= ~(GICR_PROPBASER_SHAREABILITY_MASK
|
3140 GICR_PROPBASER_CACHEABILITY_MASK
);
3141 val
|= GICR_PROPBASER_nC
;
3142 gicr_write_propbaser(val
, rbase
+ GICR_PROPBASER
);
3144 pr_info_once("GIC: using cache flushing for LPI property table\n");
3145 gic_rdists
->flags
|= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING
;
3149 val
= (page_to_phys(pend_page
) |
3150 GICR_PENDBASER_InnerShareable
|
3151 GICR_PENDBASER_RaWaWb
);
3153 gicr_write_pendbaser(val
, rbase
+ GICR_PENDBASER
);
3154 tmp
= gicr_read_pendbaser(rbase
+ GICR_PENDBASER
);
3156 if (gic_rdists
->flags
& RDIST_FLAGS_FORCE_NON_SHAREABLE
)
3157 tmp
&= ~GICR_PENDBASER_SHAREABILITY_MASK
;
3159 if (!(tmp
& GICR_PENDBASER_SHAREABILITY_MASK
)) {
3161 * The HW reports non-shareable, we must remove the
3162 * cacheability attributes as well.
3164 val
&= ~(GICR_PENDBASER_SHAREABILITY_MASK
|
3165 GICR_PENDBASER_CACHEABILITY_MASK
);
3166 val
|= GICR_PENDBASER_nC
;
3167 gicr_write_pendbaser(val
, rbase
+ GICR_PENDBASER
);
3171 val
= readl_relaxed(rbase
+ GICR_CTLR
);
3172 val
|= GICR_CTLR_ENABLE_LPIS
;
3173 writel_relaxed(val
, rbase
+ GICR_CTLR
);
3175 if (gic_rdists
->has_vlpis
&& !gic_rdists
->has_rvpeid
) {
3176 void __iomem
*vlpi_base
= gic_data_rdist_vlpi_base();
3179 * It's possible for CPU to receive VLPIs before it is
3180 * scheduled as a vPE, especially for the first CPU, and the
3181 * VLPI with INTID larger than 2^(IDbits+1) will be considered
3182 * as out of range and dropped by GIC.
3183 * So we initialize IDbits to known value to avoid VLPI drop.
3185 val
= (LPI_NRBITS
- 1) & GICR_VPROPBASER_IDBITS_MASK
;
3186 pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
3187 smp_processor_id(), val
);
3188 gicr_write_vpropbaser(val
, vlpi_base
+ GICR_VPROPBASER
);
3191 * Also clear Valid bit of GICR_VPENDBASER, in case some
3192 * ancient programming gets left in and has possibility of
3193 * corrupting memory.
3195 val
= its_clear_vpend_valid(vlpi_base
, 0, 0);
3198 if (allocate_vpe_l1_table()) {
3200 * If the allocation has failed, we're in massive trouble.
3201 * Disable direct injection, and pray that no VM was
3202 * already running...
3204 gic_rdists
->has_rvpeid
= false;
3205 gic_rdists
->has_vlpis
= false;
3208 /* Make sure the GIC has seen the above */
3211 gic_data_rdist()->flags
|= RD_LOCAL_LPI_ENABLED
;
3212 pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
3214 gic_data_rdist()->flags
& RD_LOCAL_PENDTABLE_PREALLOCATED
?
3215 "reserved" : "allocated",
3219 static void its_cpu_init_collection(struct its_node
*its
)
3221 int cpu
= smp_processor_id();
3224 /* avoid cross node collections and its mapping */
3225 if (its
->flags
& ITS_FLAGS_WORKAROUND_CAVIUM_23144
) {
3226 struct device_node
*cpu_node
;
3228 cpu_node
= of_get_cpu_node(cpu
, NULL
);
3229 if (its
->numa_node
!= NUMA_NO_NODE
&&
3230 its
->numa_node
!= of_node_to_nid(cpu_node
))
3235 * We now have to bind each collection to its target
3238 if (gic_read_typer(its
->base
+ GITS_TYPER
) & GITS_TYPER_PTA
) {
3240 * This ITS wants the physical address of the
3243 target
= gic_data_rdist()->phys_base
;
3245 /* This ITS wants a linear CPU number. */
3246 target
= gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER
);
3247 target
= GICR_TYPER_CPU_NUMBER(target
) << 16;
3250 /* Perform collection mapping */
3251 its
->collections
[cpu
].target_address
= target
;
3252 its
->collections
[cpu
].col_id
= cpu
;
3254 its_send_mapc(its
, &its
->collections
[cpu
], 1);
3255 its_send_invall(its
, &its
->collections
[cpu
]);
3258 static void its_cpu_init_collections(void)
3260 struct its_node
*its
;
3262 raw_spin_lock(&its_lock
);
3264 list_for_each_entry(its
, &its_nodes
, entry
)
3265 its_cpu_init_collection(its
);
3267 raw_spin_unlock(&its_lock
);
3270 static struct its_device
*its_find_device(struct its_node
*its
, u32 dev_id
)
3272 struct its_device
*its_dev
= NULL
, *tmp
;
3273 unsigned long flags
;
3275 raw_spin_lock_irqsave(&its
->lock
, flags
);
3277 list_for_each_entry(tmp
, &its
->its_device_list
, entry
) {
3278 if (tmp
->device_id
== dev_id
) {
3284 raw_spin_unlock_irqrestore(&its
->lock
, flags
);
3289 static struct its_baser
*its_get_baser(struct its_node
*its
, u32 type
)
3293 for (i
= 0; i
< GITS_BASER_NR_REGS
; i
++) {
3294 if (GITS_BASER_TYPE(its
->tables
[i
].val
) == type
)
3295 return &its
->tables
[i
];
3301 static bool its_alloc_table_entry(struct its_node
*its
,
3302 struct its_baser
*baser
, u32 id
)
3308 /* Don't allow device id that exceeds single, flat table limit */
3309 esz
= GITS_BASER_ENTRY_SIZE(baser
->val
);
3310 if (!(baser
->val
& GITS_BASER_INDIRECT
))
3311 return (id
< (PAGE_ORDER_TO_SIZE(baser
->order
) / esz
));
3313 /* Compute 1st level table index & check if that exceeds table limit */
3314 idx
= id
>> ilog2(baser
->psz
/ esz
);
3315 if (idx
>= (PAGE_ORDER_TO_SIZE(baser
->order
) / GITS_LVL1_ENTRY_SIZE
))
3318 table
= baser
->base
;
3320 /* Allocate memory for 2nd level table */
3322 page
= alloc_pages_node(its
->numa_node
, GFP_KERNEL
| __GFP_ZERO
,
3323 get_order(baser
->psz
));
3327 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
3328 if (!(baser
->val
& GITS_BASER_SHAREABILITY_MASK
))
3329 gic_flush_dcache_to_poc(page_address(page
), baser
->psz
);
3331 table
[idx
] = cpu_to_le64(page_to_phys(page
) | GITS_BASER_VALID
);
3333 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
3334 if (!(baser
->val
& GITS_BASER_SHAREABILITY_MASK
))
3335 gic_flush_dcache_to_poc(table
+ idx
, GITS_LVL1_ENTRY_SIZE
);
3337 /* Ensure updated table contents are visible to ITS hardware */
3344 static bool its_alloc_device_table(struct its_node
*its
, u32 dev_id
)
3346 struct its_baser
*baser
;
3348 baser
= its_get_baser(its
, GITS_BASER_TYPE_DEVICE
);
3350 /* Don't allow device id that exceeds ITS hardware limit */
3352 return (ilog2(dev_id
) < device_ids(its
));
3354 return its_alloc_table_entry(its
, baser
, dev_id
);
3357 static bool its_alloc_vpe_table(u32 vpe_id
)
3359 struct its_node
*its
;
3363 * Make sure the L2 tables are allocated on *all* v4 ITSs. We
3364 * could try and only do it on ITSs corresponding to devices
3365 * that have interrupts targeted at this VPE, but the
3366 * complexity becomes crazy (and you have tons of memory
3369 list_for_each_entry(its
, &its_nodes
, entry
) {
3370 struct its_baser
*baser
;
3375 baser
= its_get_baser(its
, GITS_BASER_TYPE_VCPU
);
3379 if (!its_alloc_table_entry(its
, baser
, vpe_id
))
3383 /* Non v4.1? No need to iterate RDs and go back early. */
3384 if (!gic_rdists
->has_rvpeid
)
3388 * Make sure the L2 tables are allocated for all copies of
3389 * the L1 table on *all* v4.1 RDs.
3391 for_each_possible_cpu(cpu
) {
3392 if (!allocate_vpe_l2_table(cpu
, vpe_id
))
3399 static struct its_device
*its_create_device(struct its_node
*its
, u32 dev_id
,
3400 int nvecs
, bool alloc_lpis
)
3402 struct its_device
*dev
;
3403 unsigned long *lpi_map
= NULL
;
3404 unsigned long flags
;
3405 u16
*col_map
= NULL
;
3412 if (!its_alloc_device_table(its
, dev_id
))
3415 if (WARN_ON(!is_power_of_2(nvecs
)))
3416 nvecs
= roundup_pow_of_two(nvecs
);
3418 dev
= kzalloc(sizeof(*dev
), GFP_KERNEL
);
3420 * Even if the device wants a single LPI, the ITT must be
3421 * sized as a power of two (and you need at least one bit...).
3423 nr_ites
= max(2, nvecs
);
3424 sz
= nr_ites
* (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE
, its
->typer
) + 1);
3425 sz
= max(sz
, ITS_ITT_ALIGN
) + ITS_ITT_ALIGN
- 1;
3426 itt
= kzalloc_node(sz
, GFP_KERNEL
, its
->numa_node
);
3428 lpi_map
= its_lpi_alloc(nvecs
, &lpi_base
, &nr_lpis
);
3430 col_map
= kcalloc(nr_lpis
, sizeof(*col_map
),
3433 col_map
= kcalloc(nr_ites
, sizeof(*col_map
), GFP_KERNEL
);
3438 if (!dev
|| !itt
|| !col_map
|| (!lpi_map
&& alloc_lpis
)) {
3441 bitmap_free(lpi_map
);
3446 gic_flush_dcache_to_poc(itt
, sz
);
3450 dev
->nr_ites
= nr_ites
;
3451 dev
->event_map
.lpi_map
= lpi_map
;
3452 dev
->event_map
.col_map
= col_map
;
3453 dev
->event_map
.lpi_base
= lpi_base
;
3454 dev
->event_map
.nr_lpis
= nr_lpis
;
3455 raw_spin_lock_init(&dev
->event_map
.vlpi_lock
);
3456 dev
->device_id
= dev_id
;
3457 INIT_LIST_HEAD(&dev
->entry
);
3459 raw_spin_lock_irqsave(&its
->lock
, flags
);
3460 list_add(&dev
->entry
, &its
->its_device_list
);
3461 raw_spin_unlock_irqrestore(&its
->lock
, flags
);
3463 /* Map device to its ITT */
3464 its_send_mapd(dev
, 1);
3469 static void its_free_device(struct its_device
*its_dev
)
3471 unsigned long flags
;
3473 raw_spin_lock_irqsave(&its_dev
->its
->lock
, flags
);
3474 list_del(&its_dev
->entry
);
3475 raw_spin_unlock_irqrestore(&its_dev
->its
->lock
, flags
);
3476 kfree(its_dev
->event_map
.col_map
);
3477 kfree(its_dev
->itt
);
3481 static int its_alloc_device_irq(struct its_device
*dev
, int nvecs
, irq_hw_number_t
*hwirq
)
3485 /* Find a free LPI region in lpi_map and allocate them. */
3486 idx
= bitmap_find_free_region(dev
->event_map
.lpi_map
,
3487 dev
->event_map
.nr_lpis
,
3488 get_count_order(nvecs
));
3492 *hwirq
= dev
->event_map
.lpi_base
+ idx
;
3497 static int its_msi_prepare(struct irq_domain
*domain
, struct device
*dev
,
3498 int nvec
, msi_alloc_info_t
*info
)
3500 struct its_node
*its
;
3501 struct its_device
*its_dev
;
3502 struct msi_domain_info
*msi_info
;
3507 * We ignore "dev" entirely, and rely on the dev_id that has
3508 * been passed via the scratchpad. This limits this domain's
3509 * usefulness to upper layers that definitely know that they
3510 * are built on top of the ITS.
3512 dev_id
= info
->scratchpad
[0].ul
;
3514 msi_info
= msi_get_domain_info(domain
);
3515 its
= msi_info
->data
;
3517 if (!gic_rdists
->has_direct_lpi
&&
3519 vpe_proxy
.dev
->its
== its
&&
3520 dev_id
== vpe_proxy
.dev
->device_id
) {
3521 /* Bad luck. Get yourself a better implementation */
3522 WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
3527 mutex_lock(&its
->dev_alloc_lock
);
3528 its_dev
= its_find_device(its
, dev_id
);
3531 * We already have seen this ID, probably through
3532 * another alias (PCI bridge of some sort). No need to
3533 * create the device.
3535 its_dev
->shared
= true;
3536 pr_debug("Reusing ITT for devID %x\n", dev_id
);
3540 its_dev
= its_create_device(its
, dev_id
, nvec
, true);
3546 if (info
->flags
& MSI_ALLOC_FLAGS_PROXY_DEVICE
)
3547 its_dev
->shared
= true;
3549 pr_debug("ITT %d entries, %d bits\n", nvec
, ilog2(nvec
));
3551 mutex_unlock(&its
->dev_alloc_lock
);
3552 info
->scratchpad
[0].ptr
= its_dev
;
3556 static struct msi_domain_ops its_msi_domain_ops
= {
3557 .msi_prepare
= its_msi_prepare
,
3560 static int its_irq_gic_domain_alloc(struct irq_domain
*domain
,
3562 irq_hw_number_t hwirq
)
3564 struct irq_fwspec fwspec
;
3566 if (irq_domain_get_of_node(domain
->parent
)) {
3567 fwspec
.fwnode
= domain
->parent
->fwnode
;
3568 fwspec
.param_count
= 3;
3569 fwspec
.param
[0] = GIC_IRQ_TYPE_LPI
;
3570 fwspec
.param
[1] = hwirq
;
3571 fwspec
.param
[2] = IRQ_TYPE_EDGE_RISING
;
3572 } else if (is_fwnode_irqchip(domain
->parent
->fwnode
)) {
3573 fwspec
.fwnode
= domain
->parent
->fwnode
;
3574 fwspec
.param_count
= 2;
3575 fwspec
.param
[0] = hwirq
;
3576 fwspec
.param
[1] = IRQ_TYPE_EDGE_RISING
;
3581 return irq_domain_alloc_irqs_parent(domain
, virq
, 1, &fwspec
);
3584 static int its_irq_domain_alloc(struct irq_domain
*domain
, unsigned int virq
,
3585 unsigned int nr_irqs
, void *args
)
3587 msi_alloc_info_t
*info
= args
;
3588 struct its_device
*its_dev
= info
->scratchpad
[0].ptr
;
3589 struct its_node
*its
= its_dev
->its
;
3590 struct irq_data
*irqd
;
3591 irq_hw_number_t hwirq
;
3595 err
= its_alloc_device_irq(its_dev
, nr_irqs
, &hwirq
);
3599 err
= iommu_dma_prepare_msi(info
->desc
, its
->get_msi_base(its_dev
));
3603 for (i
= 0; i
< nr_irqs
; i
++) {
3604 err
= its_irq_gic_domain_alloc(domain
, virq
+ i
, hwirq
+ i
);
3608 irq_domain_set_hwirq_and_chip(domain
, virq
+ i
,
3609 hwirq
+ i
, &its_irq_chip
, its_dev
);
3610 irqd
= irq_get_irq_data(virq
+ i
);
3611 irqd_set_single_target(irqd
);
3612 irqd_set_affinity_on_activate(irqd
);
3613 irqd_set_resend_when_in_progress(irqd
);
3614 pr_debug("ID:%d pID:%d vID:%d\n",
3615 (int)(hwirq
+ i
- its_dev
->event_map
.lpi_base
),
3616 (int)(hwirq
+ i
), virq
+ i
);
3622 static int its_irq_domain_activate(struct irq_domain
*domain
,
3623 struct irq_data
*d
, bool reserve
)
3625 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
3626 u32 event
= its_get_event_id(d
);
3629 cpu
= its_select_cpu(d
, cpu_online_mask
);
3630 if (cpu
< 0 || cpu
>= nr_cpu_ids
)
3633 its_inc_lpi_count(d
, cpu
);
3634 its_dev
->event_map
.col_map
[event
] = cpu
;
3635 irq_data_update_effective_affinity(d
, cpumask_of(cpu
));
3637 /* Map the GIC IRQ and event to the device */
3638 its_send_mapti(its_dev
, d
->hwirq
, event
);
3642 static void its_irq_domain_deactivate(struct irq_domain
*domain
,
3645 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
3646 u32 event
= its_get_event_id(d
);
3648 its_dec_lpi_count(d
, its_dev
->event_map
.col_map
[event
]);
3649 /* Stop the delivery of interrupts */
3650 its_send_discard(its_dev
, event
);
3653 static void its_irq_domain_free(struct irq_domain
*domain
, unsigned int virq
,
3654 unsigned int nr_irqs
)
3656 struct irq_data
*d
= irq_domain_get_irq_data(domain
, virq
);
3657 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
3658 struct its_node
*its
= its_dev
->its
;
3661 bitmap_release_region(its_dev
->event_map
.lpi_map
,
3662 its_get_event_id(irq_domain_get_irq_data(domain
, virq
)),
3663 get_count_order(nr_irqs
));
3665 for (i
= 0; i
< nr_irqs
; i
++) {
3666 struct irq_data
*data
= irq_domain_get_irq_data(domain
,
3668 /* Nuke the entry in the domain */
3669 irq_domain_reset_irq_data(data
);
3672 mutex_lock(&its
->dev_alloc_lock
);
3675 * If all interrupts have been freed, start mopping the
3676 * floor. This is conditioned on the device not being shared.
3678 if (!its_dev
->shared
&&
3679 bitmap_empty(its_dev
->event_map
.lpi_map
,
3680 its_dev
->event_map
.nr_lpis
)) {
3681 its_lpi_free(its_dev
->event_map
.lpi_map
,
3682 its_dev
->event_map
.lpi_base
,
3683 its_dev
->event_map
.nr_lpis
);
3685 /* Unmap device/itt */
3686 its_send_mapd(its_dev
, 0);
3687 its_free_device(its_dev
);
3690 mutex_unlock(&its
->dev_alloc_lock
);
3692 irq_domain_free_irqs_parent(domain
, virq
, nr_irqs
);
3695 static const struct irq_domain_ops its_domain_ops
= {
3696 .alloc
= its_irq_domain_alloc
,
3697 .free
= its_irq_domain_free
,
3698 .activate
= its_irq_domain_activate
,
3699 .deactivate
= its_irq_domain_deactivate
,
3705 * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
3706 * likely), the only way to perform an invalidate is to use a fake
3707 * device to issue an INV command, implying that the LPI has first
3708 * been mapped to some event on that device. Since this is not exactly
3709 * cheap, we try to keep that mapping around as long as possible, and
3710 * only issue an UNMAP if we're short on available slots.
3712 * Broken by design(tm).
3714 * GICv4.1, on the other hand, mandates that we're able to invalidate
3715 * by writing to a MMIO register. It doesn't implement the whole of
3716 * DirectLPI, but that's good enough. And most of the time, we don't
3717 * even have to invalidate anything, as the redistributor can be told
3718 * whether to generate a doorbell or not (we thus leave it enabled,
3721 static void its_vpe_db_proxy_unmap_locked(struct its_vpe
*vpe
)
3723 /* GICv4.1 doesn't use a proxy, so nothing to do here */
3724 if (gic_rdists
->has_rvpeid
)
3727 /* Already unmapped? */
3728 if (vpe
->vpe_proxy_event
== -1)
3731 its_send_discard(vpe_proxy
.dev
, vpe
->vpe_proxy_event
);
3732 vpe_proxy
.vpes
[vpe
->vpe_proxy_event
] = NULL
;
3735 * We don't track empty slots at all, so let's move the
3736 * next_victim pointer if we can quickly reuse that slot
3737 * instead of nuking an existing entry. Not clear that this is
3738 * always a win though, and this might just generate a ripple
3739 * effect... Let's just hope VPEs don't migrate too often.
3741 if (vpe_proxy
.vpes
[vpe_proxy
.next_victim
])
3742 vpe_proxy
.next_victim
= vpe
->vpe_proxy_event
;
3744 vpe
->vpe_proxy_event
= -1;
3747 static void its_vpe_db_proxy_unmap(struct its_vpe
*vpe
)
3749 /* GICv4.1 doesn't use a proxy, so nothing to do here */
3750 if (gic_rdists
->has_rvpeid
)
3753 if (!gic_rdists
->has_direct_lpi
) {
3754 unsigned long flags
;
3756 raw_spin_lock_irqsave(&vpe_proxy
.lock
, flags
);
3757 its_vpe_db_proxy_unmap_locked(vpe
);
3758 raw_spin_unlock_irqrestore(&vpe_proxy
.lock
, flags
);
3762 static void its_vpe_db_proxy_map_locked(struct its_vpe
*vpe
)
3764 /* GICv4.1 doesn't use a proxy, so nothing to do here */
3765 if (gic_rdists
->has_rvpeid
)
3768 /* Already mapped? */
3769 if (vpe
->vpe_proxy_event
!= -1)
3772 /* This slot was already allocated. Kick the other VPE out. */
3773 if (vpe_proxy
.vpes
[vpe_proxy
.next_victim
])
3774 its_vpe_db_proxy_unmap_locked(vpe_proxy
.vpes
[vpe_proxy
.next_victim
]);
3776 /* Map the new VPE instead */
3777 vpe_proxy
.vpes
[vpe_proxy
.next_victim
] = vpe
;
3778 vpe
->vpe_proxy_event
= vpe_proxy
.next_victim
;
3779 vpe_proxy
.next_victim
= (vpe_proxy
.next_victim
+ 1) % vpe_proxy
.dev
->nr_ites
;
3781 vpe_proxy
.dev
->event_map
.col_map
[vpe
->vpe_proxy_event
] = vpe
->col_idx
;
3782 its_send_mapti(vpe_proxy
.dev
, vpe
->vpe_db_lpi
, vpe
->vpe_proxy_event
);
3785 static void its_vpe_db_proxy_move(struct its_vpe
*vpe
, int from
, int to
)
3787 unsigned long flags
;
3788 struct its_collection
*target_col
;
3790 /* GICv4.1 doesn't use a proxy, so nothing to do here */
3791 if (gic_rdists
->has_rvpeid
)
3794 if (gic_rdists
->has_direct_lpi
) {
3795 void __iomem
*rdbase
;
3797 rdbase
= per_cpu_ptr(gic_rdists
->rdist
, from
)->rd_base
;
3798 gic_write_lpir(vpe
->vpe_db_lpi
, rdbase
+ GICR_CLRLPIR
);
3799 wait_for_syncr(rdbase
);
3804 raw_spin_lock_irqsave(&vpe_proxy
.lock
, flags
);
3806 its_vpe_db_proxy_map_locked(vpe
);
3808 target_col
= &vpe_proxy
.dev
->its
->collections
[to
];
3809 its_send_movi(vpe_proxy
.dev
, target_col
, vpe
->vpe_proxy_event
);
3810 vpe_proxy
.dev
->event_map
.col_map
[vpe
->vpe_proxy_event
] = to
;
3812 raw_spin_unlock_irqrestore(&vpe_proxy
.lock
, flags
);
3815 static int its_vpe_set_affinity(struct irq_data
*d
,
3816 const struct cpumask
*mask_val
,
3819 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
3820 int from
, cpu
= cpumask_first(mask_val
);
3821 unsigned long flags
;
3824 * Changing affinity is mega expensive, so let's be as lazy as
3825 * we can and only do it if we really have to. Also, if mapped
3826 * into the proxy device, we need to move the doorbell
3827 * interrupt to its new location.
3829 * Another thing is that changing the affinity of a vPE affects
3830 * *other interrupts* such as all the vLPIs that are routed to
3831 * this vPE. This means that the irq_desc lock is not enough to
3832 * protect us, and that we must ensure nobody samples vpe->col_idx
3833 * during the update, hence the lock below which must also be
3834 * taken on any vLPI handling path that evaluates vpe->col_idx.
3836 from
= vpe_to_cpuid_lock(vpe
, &flags
);
3843 * GICv4.1 allows us to skip VMOVP if moving to a cpu whose RD
3844 * is sharing its VPE table with the current one.
3846 if (gic_data_rdist_cpu(cpu
)->vpe_table_mask
&&
3847 cpumask_test_cpu(from
, gic_data_rdist_cpu(cpu
)->vpe_table_mask
))
3850 its_send_vmovp(vpe
);
3851 its_vpe_db_proxy_move(vpe
, from
, cpu
);
3854 irq_data_update_effective_affinity(d
, cpumask_of(cpu
));
3855 vpe_to_cpuid_unlock(vpe
, flags
);
3857 return IRQ_SET_MASK_OK_DONE
;
3860 static void its_wait_vpt_parse_complete(void)
3862 void __iomem
*vlpi_base
= gic_data_rdist_vlpi_base();
3865 if (!gic_rdists
->has_vpend_valid_dirty
)
3868 WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base
+ GICR_VPENDBASER
,
3870 !(val
& GICR_VPENDBASER_Dirty
),
3874 static void its_vpe_schedule(struct its_vpe
*vpe
)
3876 void __iomem
*vlpi_base
= gic_data_rdist_vlpi_base();
3879 /* Schedule the VPE */
3880 val
= virt_to_phys(page_address(vpe
->its_vm
->vprop_page
)) &
3881 GENMASK_ULL(51, 12);
3882 val
|= (LPI_NRBITS
- 1) & GICR_VPROPBASER_IDBITS_MASK
;
3883 val
|= GICR_VPROPBASER_RaWb
;
3884 val
|= GICR_VPROPBASER_InnerShareable
;
3885 gicr_write_vpropbaser(val
, vlpi_base
+ GICR_VPROPBASER
);
3887 val
= virt_to_phys(page_address(vpe
->vpt_page
)) &
3888 GENMASK_ULL(51, 16);
3889 val
|= GICR_VPENDBASER_RaWaWb
;
3890 val
|= GICR_VPENDBASER_InnerShareable
;
3892 * There is no good way of finding out if the pending table is
3893 * empty as we can race against the doorbell interrupt very
3894 * easily. So in the end, vpe->pending_last is only an
3895 * indication that the vcpu has something pending, not one
3896 * that the pending table is empty. A good implementation
3897 * would be able to read its coarse map pretty quickly anyway,
3898 * making this a tolerable issue.
3900 val
|= GICR_VPENDBASER_PendingLast
;
3901 val
|= vpe
->idai
? GICR_VPENDBASER_IDAI
: 0;
3902 val
|= GICR_VPENDBASER_Valid
;
3903 gicr_write_vpendbaser(val
, vlpi_base
+ GICR_VPENDBASER
);
3906 static void its_vpe_deschedule(struct its_vpe
*vpe
)
3908 void __iomem
*vlpi_base
= gic_data_rdist_vlpi_base();
3911 val
= its_clear_vpend_valid(vlpi_base
, 0, 0);
3913 vpe
->idai
= !!(val
& GICR_VPENDBASER_IDAI
);
3914 vpe
->pending_last
= !!(val
& GICR_VPENDBASER_PendingLast
);
3917 static void its_vpe_invall(struct its_vpe
*vpe
)
3919 struct its_node
*its
;
3921 list_for_each_entry(its
, &its_nodes
, entry
) {
3925 if (its_list_map
&& !vpe
->its_vm
->vlpi_count
[its
->list_nr
])
3929 * Sending a VINVALL to a single ITS is enough, as all
3930 * we need is to reach the redistributors.
3932 its_send_vinvall(its
, vpe
);
3937 static int its_vpe_set_vcpu_affinity(struct irq_data
*d
, void *vcpu_info
)
3939 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
3940 struct its_cmd_info
*info
= vcpu_info
;
3942 switch (info
->cmd_type
) {
3944 its_vpe_schedule(vpe
);
3947 case DESCHEDULE_VPE
:
3948 its_vpe_deschedule(vpe
);
3952 its_wait_vpt_parse_complete();
3956 its_vpe_invall(vpe
);
3964 static void its_vpe_send_cmd(struct its_vpe
*vpe
,
3965 void (*cmd
)(struct its_device
*, u32
))
3967 unsigned long flags
;
3969 raw_spin_lock_irqsave(&vpe_proxy
.lock
, flags
);
3971 its_vpe_db_proxy_map_locked(vpe
);
3972 cmd(vpe_proxy
.dev
, vpe
->vpe_proxy_event
);
3974 raw_spin_unlock_irqrestore(&vpe_proxy
.lock
, flags
);
3977 static void its_vpe_send_inv(struct irq_data
*d
)
3979 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
3981 if (gic_rdists
->has_direct_lpi
)
3982 __direct_lpi_inv(d
, d
->parent_data
->hwirq
);
3984 its_vpe_send_cmd(vpe
, its_send_inv
);
3987 static void its_vpe_mask_irq(struct irq_data
*d
)
3990 * We need to unmask the LPI, which is described by the parent
3991 * irq_data. Instead of calling into the parent (which won't
3992 * exactly do the right thing, let's simply use the
3993 * parent_data pointer. Yes, I'm naughty.
3995 lpi_write_config(d
->parent_data
, LPI_PROP_ENABLED
, 0);
3996 its_vpe_send_inv(d
);
3999 static void its_vpe_unmask_irq(struct irq_data
*d
)
4001 /* Same hack as above... */
4002 lpi_write_config(d
->parent_data
, 0, LPI_PROP_ENABLED
);
4003 its_vpe_send_inv(d
);
4006 static int its_vpe_set_irqchip_state(struct irq_data
*d
,
4007 enum irqchip_irq_state which
,
4010 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
4012 if (which
!= IRQCHIP_STATE_PENDING
)
4015 if (gic_rdists
->has_direct_lpi
) {
4016 void __iomem
*rdbase
;
4018 rdbase
= per_cpu_ptr(gic_rdists
->rdist
, vpe
->col_idx
)->rd_base
;
4020 gic_write_lpir(vpe
->vpe_db_lpi
, rdbase
+ GICR_SETLPIR
);
4022 gic_write_lpir(vpe
->vpe_db_lpi
, rdbase
+ GICR_CLRLPIR
);
4023 wait_for_syncr(rdbase
);
4027 its_vpe_send_cmd(vpe
, its_send_int
);
4029 its_vpe_send_cmd(vpe
, its_send_clear
);
4035 static int its_vpe_retrigger(struct irq_data
*d
)
4037 return !its_vpe_set_irqchip_state(d
, IRQCHIP_STATE_PENDING
, true);
4040 static struct irq_chip its_vpe_irq_chip
= {
4041 .name
= "GICv4-vpe",
4042 .irq_mask
= its_vpe_mask_irq
,
4043 .irq_unmask
= its_vpe_unmask_irq
,
4044 .irq_eoi
= irq_chip_eoi_parent
,
4045 .irq_set_affinity
= its_vpe_set_affinity
,
4046 .irq_retrigger
= its_vpe_retrigger
,
4047 .irq_set_irqchip_state
= its_vpe_set_irqchip_state
,
4048 .irq_set_vcpu_affinity
= its_vpe_set_vcpu_affinity
,
4051 static struct its_node
*find_4_1_its(void)
4053 static struct its_node
*its
= NULL
;
4056 list_for_each_entry(its
, &its_nodes
, entry
) {
4068 static void its_vpe_4_1_send_inv(struct irq_data
*d
)
4070 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
4071 struct its_node
*its
;
4074 * GICv4.1 wants doorbells to be invalidated using the
4075 * INVDB command in order to be broadcast to all RDs. Send
4076 * it to the first valid ITS, and let the HW do its magic.
4078 its
= find_4_1_its();
4080 its_send_invdb(its
, vpe
);
4083 static void its_vpe_4_1_mask_irq(struct irq_data
*d
)
4085 lpi_write_config(d
->parent_data
, LPI_PROP_ENABLED
, 0);
4086 its_vpe_4_1_send_inv(d
);
4089 static void its_vpe_4_1_unmask_irq(struct irq_data
*d
)
4091 lpi_write_config(d
->parent_data
, 0, LPI_PROP_ENABLED
);
4092 its_vpe_4_1_send_inv(d
);
4095 static void its_vpe_4_1_schedule(struct its_vpe
*vpe
,
4096 struct its_cmd_info
*info
)
4098 void __iomem
*vlpi_base
= gic_data_rdist_vlpi_base();
4101 /* Schedule the VPE */
4102 val
|= GICR_VPENDBASER_Valid
;
4103 val
|= info
->g0en
? GICR_VPENDBASER_4_1_VGRP0EN
: 0;
4104 val
|= info
->g1en
? GICR_VPENDBASER_4_1_VGRP1EN
: 0;
4105 val
|= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID
, vpe
->vpe_id
);
4107 gicr_write_vpendbaser(val
, vlpi_base
+ GICR_VPENDBASER
);
4110 static void its_vpe_4_1_deschedule(struct its_vpe
*vpe
,
4111 struct its_cmd_info
*info
)
4113 void __iomem
*vlpi_base
= gic_data_rdist_vlpi_base();
4117 unsigned long flags
;
4120 * vPE is going to block: make the vPE non-resident with
4121 * PendingLast clear and DB set. The GIC guarantees that if
4122 * we read-back PendingLast clear, then a doorbell will be
4123 * delivered when an interrupt comes.
4125 * Note the locking to deal with the concurrent update of
4126 * pending_last from the doorbell interrupt handler that can
4129 raw_spin_lock_irqsave(&vpe
->vpe_lock
, flags
);
4130 val
= its_clear_vpend_valid(vlpi_base
,
4131 GICR_VPENDBASER_PendingLast
,
4132 GICR_VPENDBASER_4_1_DB
);
4133 vpe
->pending_last
= !!(val
& GICR_VPENDBASER_PendingLast
);
4134 raw_spin_unlock_irqrestore(&vpe
->vpe_lock
, flags
);
4137 * We're not blocking, so just make the vPE non-resident
4138 * with PendingLast set, indicating that we'll be back.
4140 val
= its_clear_vpend_valid(vlpi_base
,
4142 GICR_VPENDBASER_PendingLast
);
4143 vpe
->pending_last
= true;
4147 static void its_vpe_4_1_invall(struct its_vpe
*vpe
)
4149 void __iomem
*rdbase
;
4150 unsigned long flags
;
4154 val
= GICR_INVALLR_V
;
4155 val
|= FIELD_PREP(GICR_INVALLR_VPEID
, vpe
->vpe_id
);
4157 /* Target the redistributor this vPE is currently known on */
4158 cpu
= vpe_to_cpuid_lock(vpe
, &flags
);
4159 raw_spin_lock(&gic_data_rdist_cpu(cpu
)->rd_lock
);
4160 rdbase
= per_cpu_ptr(gic_rdists
->rdist
, cpu
)->rd_base
;
4161 gic_write_lpir(val
, rdbase
+ GICR_INVALLR
);
4163 wait_for_syncr(rdbase
);
4164 raw_spin_unlock(&gic_data_rdist_cpu(cpu
)->rd_lock
);
4165 vpe_to_cpuid_unlock(vpe
, flags
);
4168 static int its_vpe_4_1_set_vcpu_affinity(struct irq_data
*d
, void *vcpu_info
)
4170 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
4171 struct its_cmd_info
*info
= vcpu_info
;
4173 switch (info
->cmd_type
) {
4175 its_vpe_4_1_schedule(vpe
, info
);
4178 case DESCHEDULE_VPE
:
4179 its_vpe_4_1_deschedule(vpe
, info
);
4183 its_wait_vpt_parse_complete();
4187 its_vpe_4_1_invall(vpe
);
4195 static struct irq_chip its_vpe_4_1_irq_chip
= {
4196 .name
= "GICv4.1-vpe",
4197 .irq_mask
= its_vpe_4_1_mask_irq
,
4198 .irq_unmask
= its_vpe_4_1_unmask_irq
,
4199 .irq_eoi
= irq_chip_eoi_parent
,
4200 .irq_set_affinity
= its_vpe_set_affinity
,
4201 .irq_set_vcpu_affinity
= its_vpe_4_1_set_vcpu_affinity
,
4204 static void its_configure_sgi(struct irq_data
*d
, bool clear
)
4206 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
4207 struct its_cmd_desc desc
;
4209 desc
.its_vsgi_cmd
.vpe
= vpe
;
4210 desc
.its_vsgi_cmd
.sgi
= d
->hwirq
;
4211 desc
.its_vsgi_cmd
.priority
= vpe
->sgi_config
[d
->hwirq
].priority
;
4212 desc
.its_vsgi_cmd
.enable
= vpe
->sgi_config
[d
->hwirq
].enabled
;
4213 desc
.its_vsgi_cmd
.group
= vpe
->sgi_config
[d
->hwirq
].group
;
4214 desc
.its_vsgi_cmd
.clear
= clear
;
4217 * GICv4.1 allows us to send VSGI commands to any ITS as long as the
4218 * destination VPE is mapped there. Since we map them eagerly at
4219 * activation time, we're pretty sure the first GICv4.1 ITS will do.
4221 its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd
, &desc
);
4224 static void its_sgi_mask_irq(struct irq_data
*d
)
4226 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
4228 vpe
->sgi_config
[d
->hwirq
].enabled
= false;
4229 its_configure_sgi(d
, false);
4232 static void its_sgi_unmask_irq(struct irq_data
*d
)
4234 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
4236 vpe
->sgi_config
[d
->hwirq
].enabled
= true;
4237 its_configure_sgi(d
, false);
4240 static int its_sgi_set_affinity(struct irq_data
*d
,
4241 const struct cpumask
*mask_val
,
4245 * There is no notion of affinity for virtual SGIs, at least
4246 * not on the host (since they can only be targeting a vPE).
4247 * Tell the kernel we've done whatever it asked for.
4249 irq_data_update_effective_affinity(d
, mask_val
);
4250 return IRQ_SET_MASK_OK
;
4253 static int its_sgi_set_irqchip_state(struct irq_data
*d
,
4254 enum irqchip_irq_state which
,
4257 if (which
!= IRQCHIP_STATE_PENDING
)
4261 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
4262 struct its_node
*its
= find_4_1_its();
4265 val
= FIELD_PREP(GITS_SGIR_VPEID
, vpe
->vpe_id
);
4266 val
|= FIELD_PREP(GITS_SGIR_VINTID
, d
->hwirq
);
4267 writeq_relaxed(val
, its
->sgir_base
+ GITS_SGIR
- SZ_128K
);
4269 its_configure_sgi(d
, true);
4275 static int its_sgi_get_irqchip_state(struct irq_data
*d
,
4276 enum irqchip_irq_state which
, bool *val
)
4278 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
4280 unsigned long flags
;
4281 u32 count
= 1000000; /* 1s! */
4285 if (which
!= IRQCHIP_STATE_PENDING
)
4289 * Locking galore! We can race against two different events:
4291 * - Concurrent vPE affinity change: we must make sure it cannot
4292 * happen, or we'll talk to the wrong redistributor. This is
4293 * identical to what happens with vLPIs.
4295 * - Concurrent VSGIPENDR access: As it involves accessing two
4296 * MMIO registers, this must be made atomic one way or another.
4298 cpu
= vpe_to_cpuid_lock(vpe
, &flags
);
4299 raw_spin_lock(&gic_data_rdist_cpu(cpu
)->rd_lock
);
4300 base
= gic_data_rdist_cpu(cpu
)->rd_base
+ SZ_128K
;
4301 writel_relaxed(vpe
->vpe_id
, base
+ GICR_VSGIR
);
4303 status
= readl_relaxed(base
+ GICR_VSGIPENDR
);
4304 if (!(status
& GICR_VSGIPENDR_BUSY
))
4309 pr_err_ratelimited("Unable to get SGI status\n");
4317 raw_spin_unlock(&gic_data_rdist_cpu(cpu
)->rd_lock
);
4318 vpe_to_cpuid_unlock(vpe
, flags
);
4323 *val
= !!(status
& (1 << d
->hwirq
));
4328 static int its_sgi_set_vcpu_affinity(struct irq_data
*d
, void *vcpu_info
)
4330 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
4331 struct its_cmd_info
*info
= vcpu_info
;
4333 switch (info
->cmd_type
) {
4334 case PROP_UPDATE_VSGI
:
4335 vpe
->sgi_config
[d
->hwirq
].priority
= info
->priority
;
4336 vpe
->sgi_config
[d
->hwirq
].group
= info
->group
;
4337 its_configure_sgi(d
, false);
4345 static struct irq_chip its_sgi_irq_chip
= {
4346 .name
= "GICv4.1-sgi",
4347 .irq_mask
= its_sgi_mask_irq
,
4348 .irq_unmask
= its_sgi_unmask_irq
,
4349 .irq_set_affinity
= its_sgi_set_affinity
,
4350 .irq_set_irqchip_state
= its_sgi_set_irqchip_state
,
4351 .irq_get_irqchip_state
= its_sgi_get_irqchip_state
,
4352 .irq_set_vcpu_affinity
= its_sgi_set_vcpu_affinity
,
4355 static int its_sgi_irq_domain_alloc(struct irq_domain
*domain
,
4356 unsigned int virq
, unsigned int nr_irqs
,
4359 struct its_vpe
*vpe
= args
;
4362 /* Yes, we do want 16 SGIs */
4363 WARN_ON(nr_irqs
!= 16);
4365 for (i
= 0; i
< 16; i
++) {
4366 vpe
->sgi_config
[i
].priority
= 0;
4367 vpe
->sgi_config
[i
].enabled
= false;
4368 vpe
->sgi_config
[i
].group
= false;
4370 irq_domain_set_hwirq_and_chip(domain
, virq
+ i
, i
,
4371 &its_sgi_irq_chip
, vpe
);
4372 irq_set_status_flags(virq
+ i
, IRQ_DISABLE_UNLAZY
);
4378 static void its_sgi_irq_domain_free(struct irq_domain
*domain
,
4380 unsigned int nr_irqs
)
4385 static int its_sgi_irq_domain_activate(struct irq_domain
*domain
,
4386 struct irq_data
*d
, bool reserve
)
4388 /* Write out the initial SGI configuration */
4389 its_configure_sgi(d
, false);
4393 static void its_sgi_irq_domain_deactivate(struct irq_domain
*domain
,
4396 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
4399 * The VSGI command is awkward:
4401 * - To change the configuration, CLEAR must be set to false,
4402 * leaving the pending bit unchanged.
4403 * - To clear the pending bit, CLEAR must be set to true, leaving
4404 * the configuration unchanged.
4406 * You just can't do both at once, hence the two commands below.
4408 vpe
->sgi_config
[d
->hwirq
].enabled
= false;
4409 its_configure_sgi(d
, false);
4410 its_configure_sgi(d
, true);
4413 static const struct irq_domain_ops its_sgi_domain_ops
= {
4414 .alloc
= its_sgi_irq_domain_alloc
,
4415 .free
= its_sgi_irq_domain_free
,
4416 .activate
= its_sgi_irq_domain_activate
,
4417 .deactivate
= its_sgi_irq_domain_deactivate
,
4420 static int its_vpe_id_alloc(void)
4422 return ida_simple_get(&its_vpeid_ida
, 0, ITS_MAX_VPEID
, GFP_KERNEL
);
4425 static void its_vpe_id_free(u16 id
)
4427 ida_simple_remove(&its_vpeid_ida
, id
);
4430 static int its_vpe_init(struct its_vpe
*vpe
)
4432 struct page
*vpt_page
;
4435 /* Allocate vpe_id */
4436 vpe_id
= its_vpe_id_alloc();
4441 vpt_page
= its_allocate_pending_table(GFP_KERNEL
);
4443 its_vpe_id_free(vpe_id
);
4447 if (!its_alloc_vpe_table(vpe_id
)) {
4448 its_vpe_id_free(vpe_id
);
4449 its_free_pending_table(vpt_page
);
4453 raw_spin_lock_init(&vpe
->vpe_lock
);
4454 vpe
->vpe_id
= vpe_id
;
4455 vpe
->vpt_page
= vpt_page
;
4456 if (gic_rdists
->has_rvpeid
)
4457 atomic_set(&vpe
->vmapp_count
, 0);
4459 vpe
->vpe_proxy_event
= -1;
4464 static void its_vpe_teardown(struct its_vpe
*vpe
)
4466 its_vpe_db_proxy_unmap(vpe
);
4467 its_vpe_id_free(vpe
->vpe_id
);
4468 its_free_pending_table(vpe
->vpt_page
);
4471 static void its_vpe_irq_domain_free(struct irq_domain
*domain
,
4473 unsigned int nr_irqs
)
4475 struct its_vm
*vm
= domain
->host_data
;
4478 irq_domain_free_irqs_parent(domain
, virq
, nr_irqs
);
4480 for (i
= 0; i
< nr_irqs
; i
++) {
4481 struct irq_data
*data
= irq_domain_get_irq_data(domain
,
4483 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(data
);
4485 BUG_ON(vm
!= vpe
->its_vm
);
4487 clear_bit(data
->hwirq
, vm
->db_bitmap
);
4488 its_vpe_teardown(vpe
);
4489 irq_domain_reset_irq_data(data
);
4492 if (bitmap_empty(vm
->db_bitmap
, vm
->nr_db_lpis
)) {
4493 its_lpi_free(vm
->db_bitmap
, vm
->db_lpi_base
, vm
->nr_db_lpis
);
4494 its_free_prop_table(vm
->vprop_page
);
4498 static int its_vpe_irq_domain_alloc(struct irq_domain
*domain
, unsigned int virq
,
4499 unsigned int nr_irqs
, void *args
)
4501 struct irq_chip
*irqchip
= &its_vpe_irq_chip
;
4502 struct its_vm
*vm
= args
;
4503 unsigned long *bitmap
;
4504 struct page
*vprop_page
;
4505 int base
, nr_ids
, i
, err
= 0;
4509 bitmap
= its_lpi_alloc(roundup_pow_of_two(nr_irqs
), &base
, &nr_ids
);
4513 if (nr_ids
< nr_irqs
) {
4514 its_lpi_free(bitmap
, base
, nr_ids
);
4518 vprop_page
= its_allocate_prop_table(GFP_KERNEL
);
4520 its_lpi_free(bitmap
, base
, nr_ids
);
4524 vm
->db_bitmap
= bitmap
;
4525 vm
->db_lpi_base
= base
;
4526 vm
->nr_db_lpis
= nr_ids
;
4527 vm
->vprop_page
= vprop_page
;
4529 if (gic_rdists
->has_rvpeid
)
4530 irqchip
= &its_vpe_4_1_irq_chip
;
4532 for (i
= 0; i
< nr_irqs
; i
++) {
4533 vm
->vpes
[i
]->vpe_db_lpi
= base
+ i
;
4534 err
= its_vpe_init(vm
->vpes
[i
]);
4537 err
= its_irq_gic_domain_alloc(domain
, virq
+ i
,
4538 vm
->vpes
[i
]->vpe_db_lpi
);
4541 irq_domain_set_hwirq_and_chip(domain
, virq
+ i
, i
,
4542 irqchip
, vm
->vpes
[i
]);
4544 irqd_set_resend_when_in_progress(irq_get_irq_data(virq
+ i
));
4549 its_vpe_irq_domain_free(domain
, virq
, i
);
4551 its_lpi_free(bitmap
, base
, nr_ids
);
4552 its_free_prop_table(vprop_page
);
4558 static int its_vpe_irq_domain_activate(struct irq_domain
*domain
,
4559 struct irq_data
*d
, bool reserve
)
4561 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
4562 struct its_node
*its
;
4565 * If we use the list map, we issue VMAPP on demand... Unless
4566 * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
4567 * so that VSGIs can work.
4569 if (!gic_requires_eager_mapping())
4572 /* Map the VPE to the first possible CPU */
4573 vpe
->col_idx
= cpumask_first(cpu_online_mask
);
4575 list_for_each_entry(its
, &its_nodes
, entry
) {
4579 its_send_vmapp(its
, vpe
, true);
4580 its_send_vinvall(its
, vpe
);
4583 irq_data_update_effective_affinity(d
, cpumask_of(vpe
->col_idx
));
4588 static void its_vpe_irq_domain_deactivate(struct irq_domain
*domain
,
4591 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
4592 struct its_node
*its
;
4595 * If we use the list map on GICv4.0, we unmap the VPE once no
4596 * VLPIs are associated with the VM.
4598 if (!gic_requires_eager_mapping())
4601 list_for_each_entry(its
, &its_nodes
, entry
) {
4605 its_send_vmapp(its
, vpe
, false);
4609 * There may be a direct read to the VPT after unmapping the
4610 * vPE, to guarantee the validity of this, we make the VPT
4611 * memory coherent with the CPU caches here.
4613 if (find_4_1_its() && !atomic_read(&vpe
->vmapp_count
))
4614 gic_flush_dcache_to_poc(page_address(vpe
->vpt_page
),
4618 static const struct irq_domain_ops its_vpe_domain_ops
= {
4619 .alloc
= its_vpe_irq_domain_alloc
,
4620 .free
= its_vpe_irq_domain_free
,
4621 .activate
= its_vpe_irq_domain_activate
,
4622 .deactivate
= its_vpe_irq_domain_deactivate
,
4625 static int its_force_quiescent(void __iomem
*base
)
4627 u32 count
= 1000000; /* 1s */
4630 val
= readl_relaxed(base
+ GITS_CTLR
);
4632 * GIC architecture specification requires the ITS to be both
4633 * disabled and quiescent for writes to GITS_BASER<n> or
4634 * GITS_CBASER to not have UNPREDICTABLE results.
4636 if ((val
& GITS_CTLR_QUIESCENT
) && !(val
& GITS_CTLR_ENABLE
))
4639 /* Disable the generation of all interrupts to this ITS */
4640 val
&= ~(GITS_CTLR_ENABLE
| GITS_CTLR_ImDe
);
4641 writel_relaxed(val
, base
+ GITS_CTLR
);
4643 /* Poll GITS_CTLR and wait until ITS becomes quiescent */
4645 val
= readl_relaxed(base
+ GITS_CTLR
);
4646 if (val
& GITS_CTLR_QUIESCENT
)
4658 static bool __maybe_unused
its_enable_quirk_cavium_22375(void *data
)
4660 struct its_node
*its
= data
;
4662 /* erratum 22375: only alloc 8MB table size (20 bits) */
4663 its
->typer
&= ~GITS_TYPER_DEVBITS
;
4664 its
->typer
|= FIELD_PREP(GITS_TYPER_DEVBITS
, 20 - 1);
4665 its
->flags
|= ITS_FLAGS_WORKAROUND_CAVIUM_22375
;
4670 static bool __maybe_unused
its_enable_quirk_cavium_23144(void *data
)
4672 struct its_node
*its
= data
;
4674 its
->flags
|= ITS_FLAGS_WORKAROUND_CAVIUM_23144
;
4679 static bool __maybe_unused
its_enable_quirk_qdf2400_e0065(void *data
)
4681 struct its_node
*its
= data
;
4683 /* On QDF2400, the size of the ITE is 16Bytes */
4684 its
->typer
&= ~GITS_TYPER_ITT_ENTRY_SIZE
;
4685 its
->typer
|= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE
, 16 - 1);
4690 static u64
its_irq_get_msi_base_pre_its(struct its_device
*its_dev
)
4692 struct its_node
*its
= its_dev
->its
;
4695 * The Socionext Synquacer SoC has a so-called 'pre-ITS',
4696 * which maps 32-bit writes targeted at a separate window of
4697 * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
4698 * with device ID taken from bits [device_id_bits + 1:2] of
4699 * the window offset.
4701 return its
->pre_its_base
+ (its_dev
->device_id
<< 2);
4704 static bool __maybe_unused
its_enable_quirk_socionext_synquacer(void *data
)
4706 struct its_node
*its
= data
;
4707 u32 pre_its_window
[2];
4710 if (!fwnode_property_read_u32_array(its
->fwnode_handle
,
4711 "socionext,synquacer-pre-its",
4713 ARRAY_SIZE(pre_its_window
))) {
4715 its
->pre_its_base
= pre_its_window
[0];
4716 its
->get_msi_base
= its_irq_get_msi_base_pre_its
;
4718 ids
= ilog2(pre_its_window
[1]) - 2;
4719 if (device_ids(its
) > ids
) {
4720 its
->typer
&= ~GITS_TYPER_DEVBITS
;
4721 its
->typer
|= FIELD_PREP(GITS_TYPER_DEVBITS
, ids
- 1);
4724 /* the pre-ITS breaks isolation, so disable MSI remapping */
4725 its
->msi_domain_flags
&= ~IRQ_DOMAIN_FLAG_ISOLATED_MSI
;
4731 static bool __maybe_unused
its_enable_quirk_hip07_161600802(void *data
)
4733 struct its_node
*its
= data
;
4736 * Hip07 insists on using the wrong address for the VLPI
4737 * page. Trick it into doing the right thing...
4739 its
->vlpi_redist_offset
= SZ_128K
;
4743 static bool __maybe_unused
its_enable_rk3588001(void *data
)
4745 struct its_node
*its
= data
;
4747 if (!of_machine_is_compatible("rockchip,rk3588") &&
4748 !of_machine_is_compatible("rockchip,rk3588s"))
4751 its
->flags
|= ITS_FLAGS_FORCE_NON_SHAREABLE
;
4752 gic_rdists
->flags
|= RDIST_FLAGS_FORCE_NON_SHAREABLE
;
4757 static bool its_set_non_coherent(void *data
)
4759 struct its_node
*its
= data
;
4761 its
->flags
|= ITS_FLAGS_FORCE_NON_SHAREABLE
;
4765 static const struct gic_quirk its_quirks
[] = {
4766 #ifdef CONFIG_CAVIUM_ERRATUM_22375
4768 .desc
= "ITS: Cavium errata 22375, 24313",
4769 .iidr
= 0xa100034c, /* ThunderX pass 1.x */
4771 .init
= its_enable_quirk_cavium_22375
,
4774 #ifdef CONFIG_CAVIUM_ERRATUM_23144
4776 .desc
= "ITS: Cavium erratum 23144",
4777 .iidr
= 0xa100034c, /* ThunderX pass 1.x */
4779 .init
= its_enable_quirk_cavium_23144
,
4782 #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
4784 .desc
= "ITS: QDF2400 erratum 0065",
4785 .iidr
= 0x00001070, /* QDF2400 ITS rev 1.x */
4787 .init
= its_enable_quirk_qdf2400_e0065
,
4790 #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
4793 * The Socionext Synquacer SoC incorporates ARM's own GIC-500
4794 * implementation, but with a 'pre-ITS' added that requires
4795 * special handling in software.
4797 .desc
= "ITS: Socionext Synquacer pre-ITS",
4800 .init
= its_enable_quirk_socionext_synquacer
,
4803 #ifdef CONFIG_HISILICON_ERRATUM_161600802
4805 .desc
= "ITS: Hip07 erratum 161600802",
4808 .init
= its_enable_quirk_hip07_161600802
,
4811 #ifdef CONFIG_ROCKCHIP_ERRATUM_3588001
4813 .desc
= "ITS: Rockchip erratum RK3588001",
4816 .init
= its_enable_rk3588001
,
4820 .desc
= "ITS: non-coherent attribute",
4821 .property
= "dma-noncoherent",
4822 .init
= its_set_non_coherent
,
4828 static void its_enable_quirks(struct its_node
*its
)
4830 u32 iidr
= readl_relaxed(its
->base
+ GITS_IIDR
);
4832 gic_enable_quirks(iidr
, its_quirks
, its
);
4834 if (is_of_node(its
->fwnode_handle
))
4835 gic_enable_of_quirks(to_of_node(its
->fwnode_handle
),
4839 static int its_save_disable(void)
4841 struct its_node
*its
;
4844 raw_spin_lock(&its_lock
);
4845 list_for_each_entry(its
, &its_nodes
, entry
) {
4849 its
->ctlr_save
= readl_relaxed(base
+ GITS_CTLR
);
4850 err
= its_force_quiescent(base
);
4852 pr_err("ITS@%pa: failed to quiesce: %d\n",
4853 &its
->phys_base
, err
);
4854 writel_relaxed(its
->ctlr_save
, base
+ GITS_CTLR
);
4858 its
->cbaser_save
= gits_read_cbaser(base
+ GITS_CBASER
);
4863 list_for_each_entry_continue_reverse(its
, &its_nodes
, entry
) {
4867 writel_relaxed(its
->ctlr_save
, base
+ GITS_CTLR
);
4870 raw_spin_unlock(&its_lock
);
4875 static void its_restore_enable(void)
4877 struct its_node
*its
;
4880 raw_spin_lock(&its_lock
);
4881 list_for_each_entry(its
, &its_nodes
, entry
) {
4888 * Make sure that the ITS is disabled. If it fails to quiesce,
4889 * don't restore it since writing to CBASER or BASER<n>
4890 * registers is undefined according to the GIC v3 ITS
4893 * Firmware resuming with the ITS enabled is terminally broken.
4895 WARN_ON(readl_relaxed(base
+ GITS_CTLR
) & GITS_CTLR_ENABLE
);
4896 ret
= its_force_quiescent(base
);
4898 pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
4899 &its
->phys_base
, ret
);
4903 gits_write_cbaser(its
->cbaser_save
, base
+ GITS_CBASER
);
4906 * Writing CBASER resets CREADR to 0, so make CWRITER and
4907 * cmd_write line up with it.
4909 its
->cmd_write
= its
->cmd_base
;
4910 gits_write_cwriter(0, base
+ GITS_CWRITER
);
4912 /* Restore GITS_BASER from the value cache. */
4913 for (i
= 0; i
< GITS_BASER_NR_REGS
; i
++) {
4914 struct its_baser
*baser
= &its
->tables
[i
];
4916 if (!(baser
->val
& GITS_BASER_VALID
))
4919 its_write_baser(its
, baser
, baser
->val
);
4921 writel_relaxed(its
->ctlr_save
, base
+ GITS_CTLR
);
4924 * Reinit the collection if it's stored in the ITS. This is
4925 * indicated by the col_id being less than the HCC field.
4926 * CID < HCC as specified in the GIC v3 Documentation.
4928 if (its
->collections
[smp_processor_id()].col_id
<
4929 GITS_TYPER_HCC(gic_read_typer(base
+ GITS_TYPER
)))
4930 its_cpu_init_collection(its
);
4932 raw_spin_unlock(&its_lock
);
4935 static struct syscore_ops its_syscore_ops
= {
4936 .suspend
= its_save_disable
,
4937 .resume
= its_restore_enable
,
4940 static void __init __iomem
*its_map_one(struct resource
*res
, int *err
)
4942 void __iomem
*its_base
;
4945 its_base
= ioremap(res
->start
, SZ_64K
);
4947 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res
->start
);
4952 val
= readl_relaxed(its_base
+ GITS_PIDR2
) & GIC_PIDR2_ARCH_MASK
;
4953 if (val
!= 0x30 && val
!= 0x40) {
4954 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res
->start
);
4959 *err
= its_force_quiescent(its_base
);
4961 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res
->start
);
4972 static int its_init_domain(struct its_node
*its
)
4974 struct irq_domain
*inner_domain
;
4975 struct msi_domain_info
*info
;
4977 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
4981 info
->ops
= &its_msi_domain_ops
;
4984 inner_domain
= irq_domain_create_hierarchy(its_parent
,
4985 its
->msi_domain_flags
, 0,
4986 its
->fwnode_handle
, &its_domain_ops
,
4988 if (!inner_domain
) {
4993 irq_domain_update_bus_token(inner_domain
, DOMAIN_BUS_NEXUS
);
4998 static int its_init_vpe_domain(void)
5000 struct its_node
*its
;
5004 if (gic_rdists
->has_direct_lpi
) {
5005 pr_info("ITS: Using DirectLPI for VPE invalidation\n");
5009 /* Any ITS will do, even if not v4 */
5010 its
= list_first_entry(&its_nodes
, struct its_node
, entry
);
5012 entries
= roundup_pow_of_two(nr_cpu_ids
);
5013 vpe_proxy
.vpes
= kcalloc(entries
, sizeof(*vpe_proxy
.vpes
),
5015 if (!vpe_proxy
.vpes
)
5018 /* Use the last possible DevID */
5019 devid
= GENMASK(device_ids(its
) - 1, 0);
5020 vpe_proxy
.dev
= its_create_device(its
, devid
, entries
, false);
5021 if (!vpe_proxy
.dev
) {
5022 kfree(vpe_proxy
.vpes
);
5023 pr_err("ITS: Can't allocate GICv4 proxy device\n");
5027 BUG_ON(entries
> vpe_proxy
.dev
->nr_ites
);
5029 raw_spin_lock_init(&vpe_proxy
.lock
);
5030 vpe_proxy
.next_victim
= 0;
5031 pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
5032 devid
, vpe_proxy
.dev
->nr_ites
);
5037 static int __init
its_compute_its_list_map(struct its_node
*its
)
5043 * This is assumed to be done early enough that we're
5044 * guaranteed to be single-threaded, hence no
5045 * locking. Should this change, we should address
5048 its_number
= find_first_zero_bit(&its_list_map
, GICv4_ITS_LIST_MAX
);
5049 if (its_number
>= GICv4_ITS_LIST_MAX
) {
5050 pr_err("ITS@%pa: No ITSList entry available!\n",
5055 ctlr
= readl_relaxed(its
->base
+ GITS_CTLR
);
5056 ctlr
&= ~GITS_CTLR_ITS_NUMBER
;
5057 ctlr
|= its_number
<< GITS_CTLR_ITS_NUMBER_SHIFT
;
5058 writel_relaxed(ctlr
, its
->base
+ GITS_CTLR
);
5059 ctlr
= readl_relaxed(its
->base
+ GITS_CTLR
);
5060 if ((ctlr
& GITS_CTLR_ITS_NUMBER
) != (its_number
<< GITS_CTLR_ITS_NUMBER_SHIFT
)) {
5061 its_number
= ctlr
& GITS_CTLR_ITS_NUMBER
;
5062 its_number
>>= GITS_CTLR_ITS_NUMBER_SHIFT
;
5065 if (test_and_set_bit(its_number
, &its_list_map
)) {
5066 pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
5067 &its
->phys_base
, its_number
);
5074 static int __init
its_probe_one(struct its_node
*its
)
5082 if (!(its
->typer
& GITS_TYPER_VMOVP
)) {
5083 err
= its_compute_its_list_map(its
);
5089 pr_info("ITS@%pa: Using ITS number %d\n",
5090 &its
->phys_base
, err
);
5092 pr_info("ITS@%pa: Single VMOVP capable\n", &its
->phys_base
);
5096 u32 svpet
= FIELD_GET(GITS_TYPER_SVPET
, its
->typer
);
5098 its
->sgir_base
= ioremap(its
->phys_base
+ SZ_128K
, SZ_64K
);
5099 if (!its
->sgir_base
) {
5104 its
->mpidr
= readl_relaxed(its
->base
+ GITS_MPIDR
);
5106 pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
5107 &its
->phys_base
, its
->mpidr
, svpet
);
5111 page
= alloc_pages_node(its
->numa_node
, GFP_KERNEL
| __GFP_ZERO
,
5112 get_order(ITS_CMD_QUEUE_SZ
));
5115 goto out_unmap_sgir
;
5117 its
->cmd_base
= (void *)page_address(page
);
5118 its
->cmd_write
= its
->cmd_base
;
5120 err
= its_alloc_tables(its
);
5124 err
= its_alloc_collections(its
);
5126 goto out_free_tables
;
5128 baser
= (virt_to_phys(its
->cmd_base
) |
5129 GITS_CBASER_RaWaWb
|
5130 GITS_CBASER_InnerShareable
|
5131 (ITS_CMD_QUEUE_SZ
/ SZ_4K
- 1) |
5134 gits_write_cbaser(baser
, its
->base
+ GITS_CBASER
);
5135 tmp
= gits_read_cbaser(its
->base
+ GITS_CBASER
);
5137 if (its
->flags
& ITS_FLAGS_FORCE_NON_SHAREABLE
)
5138 tmp
&= ~GITS_CBASER_SHAREABILITY_MASK
;
5140 if ((tmp
^ baser
) & GITS_CBASER_SHAREABILITY_MASK
) {
5141 if (!(tmp
& GITS_CBASER_SHAREABILITY_MASK
)) {
5143 * The HW reports non-shareable, we must
5144 * remove the cacheability attributes as
5147 baser
&= ~(GITS_CBASER_SHAREABILITY_MASK
|
5148 GITS_CBASER_CACHEABILITY_MASK
);
5149 baser
|= GITS_CBASER_nC
;
5150 gits_write_cbaser(baser
, its
->base
+ GITS_CBASER
);
5152 pr_info("ITS: using cache flushing for cmd queue\n");
5153 its
->flags
|= ITS_FLAGS_CMDQ_NEEDS_FLUSHING
;
5156 gits_write_cwriter(0, its
->base
+ GITS_CWRITER
);
5157 ctlr
= readl_relaxed(its
->base
+ GITS_CTLR
);
5158 ctlr
|= GITS_CTLR_ENABLE
;
5160 ctlr
|= GITS_CTLR_ImDe
;
5161 writel_relaxed(ctlr
, its
->base
+ GITS_CTLR
);
5163 err
= its_init_domain(its
);
5165 goto out_free_tables
;
5167 raw_spin_lock(&its_lock
);
5168 list_add(&its
->entry
, &its_nodes
);
5169 raw_spin_unlock(&its_lock
);
5174 its_free_tables(its
);
5176 free_pages((unsigned long)its
->cmd_base
, get_order(ITS_CMD_QUEUE_SZ
));
5179 iounmap(its
->sgir_base
);
5181 pr_err("ITS@%pa: failed probing (%d)\n", &its
->phys_base
, err
);
5185 static bool gic_rdists_supports_plpis(void)
5187 return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER
) & GICR_TYPER_PLPIS
);
5190 static int redist_disable_lpis(void)
5192 void __iomem
*rbase
= gic_data_rdist_rd_base();
5193 u64 timeout
= USEC_PER_SEC
;
5196 if (!gic_rdists_supports_plpis()) {
5197 pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
5201 val
= readl_relaxed(rbase
+ GICR_CTLR
);
5202 if (!(val
& GICR_CTLR_ENABLE_LPIS
))
5206 * If coming via a CPU hotplug event, we don't need to disable
5207 * LPIs before trying to re-enable them. They are already
5208 * configured and all is well in the world.
5210 * If running with preallocated tables, there is nothing to do.
5212 if ((gic_data_rdist()->flags
& RD_LOCAL_LPI_ENABLED
) ||
5213 (gic_rdists
->flags
& RDIST_FLAGS_RD_TABLES_PREALLOCATED
))
5217 * From that point on, we only try to do some damage control.
5219 pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
5220 smp_processor_id());
5221 add_taint(TAINT_CRAP
, LOCKDEP_STILL_OK
);
5224 val
&= ~GICR_CTLR_ENABLE_LPIS
;
5225 writel_relaxed(val
, rbase
+ GICR_CTLR
);
5227 /* Make sure any change to GICR_CTLR is observable by the GIC */
5231 * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
5232 * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
5233 * Error out if we time out waiting for RWP to clear.
5235 while (readl_relaxed(rbase
+ GICR_CTLR
) & GICR_CTLR_RWP
) {
5237 pr_err("CPU%d: Timeout while disabling LPIs\n",
5238 smp_processor_id());
5246 * After it has been written to 1, it is IMPLEMENTATION
5247 * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
5248 * cleared to 0. Error out if clearing the bit failed.
5250 if (readl_relaxed(rbase
+ GICR_CTLR
) & GICR_CTLR_ENABLE_LPIS
) {
5251 pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
5258 int its_cpu_init(void)
5260 if (!list_empty(&its_nodes
)) {
5263 ret
= redist_disable_lpis();
5267 its_cpu_init_lpis();
5268 its_cpu_init_collections();
5274 static void rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct
*work
)
5276 cpuhp_remove_state_nocalls(gic_rdists
->cpuhp_memreserve_state
);
5277 gic_rdists
->cpuhp_memreserve_state
= CPUHP_INVALID
;
5280 static DECLARE_WORK(rdist_memreserve_cpuhp_cleanup_work
,
5281 rdist_memreserve_cpuhp_cleanup_workfn
);
5283 static int its_cpu_memreserve_lpi(unsigned int cpu
)
5285 struct page
*pend_page
;
5288 /* This gets to run exactly once per CPU */
5289 if (gic_data_rdist()->flags
& RD_LOCAL_MEMRESERVE_DONE
)
5292 pend_page
= gic_data_rdist()->pend_page
;
5293 if (WARN_ON(!pend_page
)) {
5298 * If the pending table was pre-programmed, free the memory we
5299 * preemptively allocated. Otherwise, reserve that memory for
5302 if (gic_data_rdist()->flags
& RD_LOCAL_PENDTABLE_PREALLOCATED
) {
5303 its_free_pending_table(pend_page
);
5304 gic_data_rdist()->pend_page
= NULL
;
5306 phys_addr_t paddr
= page_to_phys(pend_page
);
5307 WARN_ON(gic_reserve_range(paddr
, LPI_PENDBASE_SZ
));
5311 /* Last CPU being brought up gets to issue the cleanup */
5312 if (!IS_ENABLED(CONFIG_SMP
) ||
5313 cpumask_equal(&cpus_booted_once_mask
, cpu_possible_mask
))
5314 schedule_work(&rdist_memreserve_cpuhp_cleanup_work
);
5316 gic_data_rdist()->flags
|= RD_LOCAL_MEMRESERVE_DONE
;
5320 /* Mark all the BASER registers as invalid before they get reprogrammed */
5321 static int __init
its_reset_one(struct resource
*res
)
5323 void __iomem
*its_base
;
5326 its_base
= its_map_one(res
, &err
);
5330 for (i
= 0; i
< GITS_BASER_NR_REGS
; i
++)
5331 gits_write_baser(0, its_base
+ GITS_BASER
+ (i
<< 3));
5337 static const struct of_device_id its_device_id
[] = {
5338 { .compatible
= "arm,gic-v3-its", },
5342 static struct its_node __init
*its_node_init(struct resource
*res
,
5343 struct fwnode_handle
*handle
, int numa_node
)
5345 void __iomem
*its_base
;
5346 struct its_node
*its
;
5349 its_base
= its_map_one(res
, &err
);
5353 pr_info("ITS %pR\n", res
);
5355 its
= kzalloc(sizeof(*its
), GFP_KERNEL
);
5359 raw_spin_lock_init(&its
->lock
);
5360 mutex_init(&its
->dev_alloc_lock
);
5361 INIT_LIST_HEAD(&its
->entry
);
5362 INIT_LIST_HEAD(&its
->its_device_list
);
5364 its
->typer
= gic_read_typer(its_base
+ GITS_TYPER
);
5365 its
->base
= its_base
;
5366 its
->phys_base
= res
->start
;
5367 its
->get_msi_base
= its_irq_get_msi_base
;
5368 its
->msi_domain_flags
= IRQ_DOMAIN_FLAG_ISOLATED_MSI
;
5370 its
->numa_node
= numa_node
;
5371 its
->fwnode_handle
= handle
;
5380 static void its_node_destroy(struct its_node
*its
)
5386 static int __init
its_of_probe(struct device_node
*node
)
5388 struct device_node
*np
;
5389 struct resource res
;
5393 * Make sure *all* the ITS are reset before we probe any, as
5394 * they may be sharing memory. If any of the ITS fails to
5395 * reset, don't even try to go any further, as this could
5396 * result in something even worse.
5398 for (np
= of_find_matching_node(node
, its_device_id
); np
;
5399 np
= of_find_matching_node(np
, its_device_id
)) {
5400 if (!of_device_is_available(np
) ||
5401 !of_property_read_bool(np
, "msi-controller") ||
5402 of_address_to_resource(np
, 0, &res
))
5405 err
= its_reset_one(&res
);
5410 for (np
= of_find_matching_node(node
, its_device_id
); np
;
5411 np
= of_find_matching_node(np
, its_device_id
)) {
5412 struct its_node
*its
;
5414 if (!of_device_is_available(np
))
5416 if (!of_property_read_bool(np
, "msi-controller")) {
5417 pr_warn("%pOF: no msi-controller property, ITS ignored\n",
5422 if (of_address_to_resource(np
, 0, &res
)) {
5423 pr_warn("%pOF: no regs?\n", np
);
5428 its
= its_node_init(&res
, &np
->fwnode
, of_node_to_nid(np
));
5432 its_enable_quirks(its
);
5433 err
= its_probe_one(its
);
5435 its_node_destroy(its
);
5444 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
5446 #ifdef CONFIG_ACPI_NUMA
5447 struct its_srat_map
{
5454 static struct its_srat_map
*its_srat_maps __initdata
;
5455 static int its_in_srat __initdata
;
5457 static int __init
acpi_get_its_numa_node(u32 its_id
)
5461 for (i
= 0; i
< its_in_srat
; i
++) {
5462 if (its_id
== its_srat_maps
[i
].its_id
)
5463 return its_srat_maps
[i
].numa_node
;
5465 return NUMA_NO_NODE
;
5468 static int __init
gic_acpi_match_srat_its(union acpi_subtable_headers
*header
,
5469 const unsigned long end
)
5474 static int __init
gic_acpi_parse_srat_its(union acpi_subtable_headers
*header
,
5475 const unsigned long end
)
5478 struct acpi_srat_gic_its_affinity
*its_affinity
;
5480 its_affinity
= (struct acpi_srat_gic_its_affinity
*)header
;
5484 if (its_affinity
->header
.length
< sizeof(*its_affinity
)) {
5485 pr_err("SRAT: Invalid header length %d in ITS affinity\n",
5486 its_affinity
->header
.length
);
5491 * Note that in theory a new proximity node could be created by this
5492 * entry as it is an SRAT resource allocation structure.
5493 * We do not currently support doing so.
5495 node
= pxm_to_node(its_affinity
->proximity_domain
);
5497 if (node
== NUMA_NO_NODE
|| node
>= MAX_NUMNODES
) {
5498 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node
);
5502 its_srat_maps
[its_in_srat
].numa_node
= node
;
5503 its_srat_maps
[its_in_srat
].its_id
= its_affinity
->its_id
;
5505 pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
5506 its_affinity
->proximity_domain
, its_affinity
->its_id
, node
);
5511 static void __init
acpi_table_parse_srat_its(void)
5515 count
= acpi_table_parse_entries(ACPI_SIG_SRAT
,
5516 sizeof(struct acpi_table_srat
),
5517 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY
,
5518 gic_acpi_match_srat_its
, 0);
5522 its_srat_maps
= kmalloc_array(count
, sizeof(struct its_srat_map
),
5527 acpi_table_parse_entries(ACPI_SIG_SRAT
,
5528 sizeof(struct acpi_table_srat
),
5529 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY
,
5530 gic_acpi_parse_srat_its
, 0);
5533 /* free the its_srat_maps after ITS probing */
5534 static void __init
acpi_its_srat_maps_free(void)
5536 kfree(its_srat_maps
);
5539 static void __init
acpi_table_parse_srat_its(void) { }
5540 static int __init
acpi_get_its_numa_node(u32 its_id
) { return NUMA_NO_NODE
; }
5541 static void __init
acpi_its_srat_maps_free(void) { }
5544 static int __init
gic_acpi_parse_madt_its(union acpi_subtable_headers
*header
,
5545 const unsigned long end
)
5547 struct acpi_madt_generic_translator
*its_entry
;
5548 struct fwnode_handle
*dom_handle
;
5549 struct its_node
*its
;
5550 struct resource res
;
5553 its_entry
= (struct acpi_madt_generic_translator
*)header
;
5554 memset(&res
, 0, sizeof(res
));
5555 res
.start
= its_entry
->base_address
;
5556 res
.end
= its_entry
->base_address
+ ACPI_GICV3_ITS_MEM_SIZE
- 1;
5557 res
.flags
= IORESOURCE_MEM
;
5559 dom_handle
= irq_domain_alloc_fwnode(&res
.start
);
5561 pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
5566 err
= iort_register_domain_token(its_entry
->translation_id
, res
.start
,
5569 pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
5570 &res
.start
, its_entry
->translation_id
);
5574 its
= its_node_init(&res
, dom_handle
,
5575 acpi_get_its_numa_node(its_entry
->translation_id
));
5581 err
= its_probe_one(its
);
5586 iort_deregister_domain_token(its_entry
->translation_id
);
5588 irq_domain_free_fwnode(dom_handle
);
5592 static int __init
its_acpi_reset(union acpi_subtable_headers
*header
,
5593 const unsigned long end
)
5595 struct acpi_madt_generic_translator
*its_entry
;
5596 struct resource res
;
5598 its_entry
= (struct acpi_madt_generic_translator
*)header
;
5599 res
= (struct resource
) {
5600 .start
= its_entry
->base_address
,
5601 .end
= its_entry
->base_address
+ ACPI_GICV3_ITS_MEM_SIZE
- 1,
5602 .flags
= IORESOURCE_MEM
,
5605 return its_reset_one(&res
);
5608 static void __init
its_acpi_probe(void)
5610 acpi_table_parse_srat_its();
5612 * Make sure *all* the ITS are reset before we probe any, as
5613 * they may be sharing memory. If any of the ITS fails to
5614 * reset, don't even try to go any further, as this could
5615 * result in something even worse.
5617 if (acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR
,
5618 its_acpi_reset
, 0) > 0)
5619 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR
,
5620 gic_acpi_parse_madt_its
, 0);
5621 acpi_its_srat_maps_free();
5624 static void __init
its_acpi_probe(void) { }
5627 int __init
its_lpi_memreserve_init(void)
5631 if (!efi_enabled(EFI_CONFIG_TABLES
))
5634 if (list_empty(&its_nodes
))
5637 gic_rdists
->cpuhp_memreserve_state
= CPUHP_INVALID
;
5638 state
= cpuhp_setup_state(CPUHP_AP_ONLINE_DYN
,
5639 "irqchip/arm/gicv3/memreserve:online",
5640 its_cpu_memreserve_lpi
,
5645 gic_rdists
->cpuhp_memreserve_state
= state
;
5650 int __init
its_init(struct fwnode_handle
*handle
, struct rdists
*rdists
,
5651 struct irq_domain
*parent_domain
)
5653 struct device_node
*of_node
;
5654 struct its_node
*its
;
5655 bool has_v4
= false;
5656 bool has_v4_1
= false;
5659 gic_rdists
= rdists
;
5661 its_parent
= parent_domain
;
5662 of_node
= to_of_node(handle
);
5664 its_of_probe(of_node
);
5668 if (list_empty(&its_nodes
)) {
5669 pr_warn("ITS: No ITS available, not enabling LPIs\n");
5673 err
= allocate_lpi_tables();
5677 list_for_each_entry(its
, &its_nodes
, entry
) {
5678 has_v4
|= is_v4(its
);
5679 has_v4_1
|= is_v4_1(its
);
5682 /* Don't bother with inconsistent systems */
5683 if (WARN_ON(!has_v4_1
&& rdists
->has_rvpeid
))
5684 rdists
->has_rvpeid
= false;
5686 if (has_v4
& rdists
->has_vlpis
) {
5687 const struct irq_domain_ops
*sgi_ops
;
5690 sgi_ops
= &its_sgi_domain_ops
;
5694 if (its_init_vpe_domain() ||
5695 its_init_v4(parent_domain
, &its_vpe_domain_ops
, sgi_ops
)) {
5696 rdists
->has_vlpis
= false;
5697 pr_err("ITS: Disabling GICv4 support\n");
5701 register_syscore_ops(&its_syscore_ops
);