--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2024-2026 Loongson Technologies, Inc.
+ */
+#define pr_fmt(fmt) "redirect: " fmt
+
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/irq-msi-lib.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/msi.h>
+#include <linux/spinlock.h>
+
+#include <asm/irq.h>
+#include <asm/loongarch.h>
+#include <asm/loongson.h>
+#include <asm/numa.h>
+#include <asm/setup.h>
+
+#include "irq-loongson.h"
+
+#define LOONGARCH_IOCSR_REDIRECT_CFG 0x15e0
+#define LOONGARCH_IOCSR_REDIRECT_TBR 0x15e8 /* IRT BASE REG */
+#define LOONGARCH_IOCSR_REDIRECT_CQB 0x15f0 /* IRT CACHE QUEUE BASE */
+#define LOONGARCH_IOCSR_REDIRECT_CQH 0x15f8 /* IRT CACHE QUEUE HEAD, 32bit */
+#define LOONGARCH_IOCSR_REDIRECT_CQT 0x15fc /* IRT CACHE QUEUE TAIL, 32bit */
+
+#define CQB_ADDR_MASK GENMASK_U64(47, 12)
+#define CQB_SIZE_MASK 0xf
+
+#define GPID_ADDR_MASK GENMASK_U64(47, 6)
+#define GPID_ADDR_SHIFT 6
+
+#define INVALID_INDEX 0
+#define CFG_DISABLE_IDLE 2
+
+#define MAX_IR_ENGINES 16
+
+struct redirect_entry {
+ struct {
+ u64 valid : 1,
+ res1 : 5,
+ gpid : 42,
+ res2 : 8,
+ vector : 8;
+ } lo;
+ u64 hi;
+};
+
+#define IRD_ENTRY_SIZE sizeof(struct redirect_entry)
+#define IRD_ENTRIES SZ_64K
+#define IRD_TABLE_PAGE_ORDER get_order(IRD_ENTRIES * IRD_ENTRY_SIZE)
+
+struct redirect_cmd {
+ union {
+ u64 cmd_info;
+ struct {
+ u64 res1 : 4,
+ type : 1,
+ need_notice : 1,
+ pad1 : 2,
+ index : 16,
+ pad2 : 40;
+ } index;
+ };
+ u64 notice_addr;
+};
+
+#define IRD_CMD_SIZE sizeof(struct redirect_cmd)
+#define INV_QUEUE_SIZE SZ_4K
+#define INV_QUEUE_PAGE_ORDER get_order(INV_QUEUE_SIZE * IRD_CMD_SIZE)
+
+struct redirect_gpid {
+ u64 pir[4]; /* Pending interrupt requested */
+ u8 en : 1, /* Doorbell */
+ res1 : 7;
+ u8 irqnum;
+ u16 res2;
+ u32 dstcpu;
+ u32 rsvd[6];
+};
+
+struct redirect_table {
+ struct redirect_entry *table;
+ unsigned long *bitmap;
+ raw_spinlock_t lock;
+};
+
+struct redirect_queue {
+ struct redirect_cmd *cmd_base;
+ int head;
+ int tail;
+ raw_spinlock_t lock;
+};
+
+struct redirect_desc {
+ struct redirect_table ird_table;
+ struct redirect_queue inv_queue;
+ int node;
+};
+
+struct redirect_item {
+ int index;
+ struct redirect_desc *irde;
+ struct redirect_gpid *gpid;
+};
+
+static struct irq_domain *redirect_domain;
+static struct redirect_desc redirect_descs[MAX_IR_ENGINES];
+
+static phys_addr_t msi_base_addr;
+static phys_addr_t redirect_reg_base = LOONGSON_REG_BASE;
+
+#ifdef CONFIG_32BIT
+
+#define REDIRECT_REG(reg, node) \
+ ((void __iomem *)(IO_BASE | redirect_reg_base | (reg)))
+
+#else
+
+#define REDIRECT_REG(reg, node) \
+ ((void __iomem *)(IO_BASE | redirect_reg_base | (u64)(node) << NODE_ADDRSPACE_SHIFT | (reg)))
+
+#endif
+
+static inline u32 redirect_read_reg32(u32 node, u32 reg)
+{
+ return readl(REDIRECT_REG(reg, node));
+}
+
+static inline void redirect_write_reg32(u32 node, u32 val, u32 reg)
+{
+ writel(val, REDIRECT_REG(reg, node));
+}
+
+static inline void redirect_write_reg64(u32 node, u64 val, u32 reg)
+{
+ writeq(val, REDIRECT_REG(reg, node));
+}
+
+static inline struct redirect_entry *item_get_entry(struct redirect_item *item)
+{
+ return item->irde->ird_table.table + item->index;
+}
+
+static inline bool invalid_queue_is_full(int node, u32 *tail)
+{
+ u32 head = redirect_read_reg32(node, LOONGARCH_IOCSR_REDIRECT_CQH);
+
+ *tail = redirect_read_reg32(node, LOONGARCH_IOCSR_REDIRECT_CQT);
+
+ return head == ((*tail + 1) % INV_QUEUE_SIZE);
+}
+
+static void invalid_enqueue(struct redirect_item *item, struct redirect_cmd *cmd)
+{
+ struct redirect_queue *inv_queue = &item->irde->inv_queue;
+ u32 tail;
+
+ guard(raw_spinlock_irqsave)(&inv_queue->lock);
+
+ while (invalid_queue_is_full(item->irde->node, &tail))
+ cpu_relax();
+
+ memcpy(&inv_queue->cmd_base[tail], cmd, sizeof(*cmd));
+
+ redirect_write_reg32(item->irde->node, (tail + 1) % INV_QUEUE_SIZE, LOONGARCH_IOCSR_REDIRECT_CQT);
+}
+
+static void irde_invalidate_entry(struct redirect_item *item)
+{
+ struct redirect_cmd cmd;
+ u64 raddr = 0;
+
+ cmd.cmd_info = 0;
+ cmd.index.type = INVALID_INDEX;
+ cmd.index.need_notice = 1;
+ cmd.index.index = item->index;
+ cmd.notice_addr = (u64)(__pa(&raddr));
+
+ invalid_enqueue(item, &cmd);
+
+ /*
+ * The CPU needs to wait here for cmd to complete, and it determines this
+ * by checking whether the invalidation queue has already written a valid value
+ * to cmd.notice_addr.
+ */
+ while (!raddr)
+ cpu_relax();
+}
+
+static inline struct avecintc_data *irq_data_get_avec_data(struct irq_data *data)
+{
+ return data->parent_data->chip_data;
+}
+
+static int redirect_table_alloc(int node, u32 nr_irqs)
+{
+ struct redirect_table *ird_table = &redirect_descs[node].ird_table;
+ int index, order = 0;
+
+ if (nr_irqs > 1) {
+ nr_irqs = __roundup_pow_of_two(nr_irqs);
+ order = ilog2(nr_irqs);
+ }
+
+ guard(raw_spinlock_irqsave)(&ird_table->lock);
+
+ index = bitmap_find_free_region(ird_table->bitmap, IRD_ENTRIES, order);
+ if (index < 0) {
+ pr_err("No redirect entry to use\n");
+ return -EINVAL;
+ }
+
+ return index;
+}
+
+static void redirect_table_free(struct redirect_item *item)
+{
+ struct redirect_table *ird_table = &item->irde->ird_table;
+ struct redirect_entry *entry = item_get_entry(item);
+
+ memset(entry, 0, sizeof(*entry));
+
+ scoped_guard(raw_spinlock_irq, &ird_table->lock)
+ clear_bit(item->index, ird_table->bitmap);
+
+ kfree(item->gpid);
+
+ irde_invalidate_entry(item);
+}
+
+static inline void redirect_domain_prepare_entry(struct redirect_item *item,
+ struct avecintc_data *adata)
+{
+ struct redirect_entry *entry = item_get_entry(item);
+
+ item->gpid->en = 1;
+ item->gpid->dstcpu = adata->cpu;
+ item->gpid->irqnum = adata->vec;
+
+ entry->lo.valid = 1;
+ entry->lo.vector = 0xff;
+ entry->lo.gpid = ((unsigned long)item->gpid & GPID_ADDR_MASK) >> GPID_ADDR_SHIFT;
+}
+
+static void redirect_free_resources(struct irq_domain *domain,
+ unsigned int virq, unsigned int nr_irqs)
+{
+ for (int i = 0; i < nr_irqs; i++) {
+ struct irq_data *irq_data = irq_domain_get_irq_data(domain, virq + i);
+
+ if (irq_data && irq_data->chip_data) {
+ struct redirect_item *item = irq_data->chip_data;
+
+ redirect_table_free(item);
+ kfree(item);
+ }
+ }
+}
+
+#ifdef CONFIG_SMP
+static int redirect_set_affinity(struct irq_data *data, const struct cpumask *dest, bool force)
+{
+ struct avecintc_data *adata = irq_data_get_avec_data(data);
+ struct redirect_item *item = data->chip_data;
+ int ret;
+
+ ret = irq_chip_set_affinity_parent(data, dest, force);
+ switch (ret) {
+ case IRQ_SET_MASK_OK:
+ break;
+ case IRQ_SET_MASK_OK_DONE:
+ return ret;
+ default:
+ pr_err("IRDE: set_affinity error %d\n", ret);
+ return ret;
+ }
+
+ redirect_domain_prepare_entry(item, adata);
+ irde_invalidate_entry(item);
+ avecintc_sync(adata);
+
+ return IRQ_SET_MASK_OK;
+}
+#endif
+
+static void redirect_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
+{
+ struct redirect_item *item = irq_data_get_irq_chip_data(d);
+
+ msg->address_hi = 0x0;
+ msg->address_lo = (msi_base_addr | 1 << 2);
+ msg->data = item->index;
+}
+
+static struct irq_chip loongarch_redirect_chip = {
+ .name = "REDIRECT",
+ .irq_ack = irq_chip_ack_parent,
+ .irq_mask = irq_chip_mask_parent,
+ .irq_unmask = irq_chip_unmask_parent,
+#ifdef CONFIG_SMP
+ .irq_set_affinity = redirect_set_affinity,
+#endif
+ .irq_compose_msi_msg = redirect_compose_msi_msg,
+};
+
+static int redirect_domain_alloc(struct irq_domain *domain, unsigned int virq,
+ unsigned int nr_irqs, void *arg)
+{
+ msi_alloc_info_t *info = arg;
+ int ret, i, node, index;
+
+ node = dev_to_node(info->desc->dev);
+
+ ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
+ if (ret < 0)
+ return ret;
+
+ index = redirect_table_alloc(node, nr_irqs);
+ if (index < 0) {
+ pr_err("Alloc redirect table entry failed\n");
+ return -EINVAL;
+ }
+
+ for (i = 0; i < nr_irqs; i++) {
+ struct irq_data *irq_data = irq_domain_get_irq_data(domain, virq + i);
+ struct redirect_item *item;
+
+ item = kzalloc(sizeof(*item), GFP_KERNEL);
+ if (!item) {
+ pr_err("Alloc redirect descriptor failed\n");
+ goto out_free_resources;
+ }
+ item->irde = &redirect_descs[node];
+
+ /*
+ * Only bits 47:6 of the GPID are passed to the controller,
+ * 64-byte alignment must be guarantee and make kzalloc can
+ * align to the respective size.
+ */
+ static_assert(sizeof(*item->gpid) == 64);
+ item->gpid = kzalloc_node(sizeof(*item->gpid), GFP_KERNEL, node);
+ if (!item->gpid) {
+ pr_err("Alloc redirect GPID failed\n");
+ goto out_free_resources;
+ }
+ item->index = index + i;
+
+ irq_data->chip_data = item;
+ irq_data->chip = &loongarch_redirect_chip;
+
+ redirect_domain_prepare_entry(item, irq_data_get_avec_data(irq_data));
+ }
+
+ return 0;
+
+out_free_resources:
+ redirect_free_resources(domain, virq, nr_irqs);
+ irq_domain_free_irqs_common(domain, virq, nr_irqs);
+
+ return -ENOMEM;
+}
+
+static void redirect_domain_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs)
+{
+ redirect_free_resources(domain, virq, nr_irqs);
+ return irq_domain_free_irqs_common(domain, virq, nr_irqs);
+}
+
+static const struct irq_domain_ops redirect_domain_ops = {
+ .alloc = redirect_domain_alloc,
+ .free = redirect_domain_free,
+ .select = msi_lib_irq_domain_select,
+};
+
+static int redirect_table_init(struct redirect_desc *irde)
+{
+ struct redirect_table *ird_table = &irde->ird_table;
+ unsigned long *bitmap;
+ struct folio *folio;
+
+ folio = __folio_alloc_node(GFP_KERNEL | __GFP_ZERO, IRD_TABLE_PAGE_ORDER, irde->node);
+ if (!folio) {
+ pr_err("Node [%d] redirect table alloc pages failed!\n", irde->node);
+ return -ENOMEM;
+ }
+ ird_table->table = folio_address(folio);
+
+ bitmap = bitmap_zalloc(IRD_ENTRIES, GFP_KERNEL);
+ if (!bitmap) {
+ pr_err("Node [%d] redirect table bitmap alloc pages failed!\n", irde->node);
+ folio_put(folio);
+ ird_table->table = NULL;
+ return -ENOMEM;
+ }
+ ird_table->bitmap = bitmap;
+
+ raw_spin_lock_init(&ird_table->lock);
+
+ return 0;
+}
+
+static int redirect_queue_init(struct redirect_desc *irde)
+{
+ struct redirect_queue *inv_queue = &irde->inv_queue;
+ struct folio *folio;
+
+ folio = __folio_alloc_node(GFP_KERNEL | __GFP_ZERO, INV_QUEUE_PAGE_ORDER, irde->node);
+ if (!folio) {
+ pr_err("Node [%d] invalid queue alloc pages failed!\n", irde->node);
+ return -ENOMEM;
+ }
+
+ inv_queue->cmd_base = folio_address(folio);
+ inv_queue->head = 0;
+ inv_queue->tail = 0;
+ raw_spin_lock_init(&inv_queue->lock);
+
+ return 0;
+}
+
+static void redirect_irde_cfg(struct redirect_desc *irde)
+{
+ redirect_write_reg64(irde->node, CFG_DISABLE_IDLE, LOONGARCH_IOCSR_REDIRECT_CFG);
+ redirect_write_reg64(irde->node, __pa(irde->ird_table.table), LOONGARCH_IOCSR_REDIRECT_TBR);
+ redirect_write_reg32(irde->node, 0, LOONGARCH_IOCSR_REDIRECT_CQH);
+ redirect_write_reg32(irde->node, 0, LOONGARCH_IOCSR_REDIRECT_CQT);
+ redirect_write_reg64(irde->node, ((unsigned long)irde->inv_queue.cmd_base & CQB_ADDR_MASK) |
+ CQB_SIZE_MASK, LOONGARCH_IOCSR_REDIRECT_CQB);
+}
+
+static void __init redirect_irde_free(struct redirect_desc *irde)
+{
+ struct redirect_table *ird_table = &redirect_descs->ird_table;
+ struct redirect_queue *inv_queue = &redirect_descs->inv_queue;
+
+ if (ird_table->table) {
+ folio_put(virt_to_folio(ird_table->table));
+ ird_table->table = NULL;
+ }
+
+ if (ird_table->bitmap) {
+ bitmap_free(ird_table->bitmap);
+ ird_table->bitmap = NULL;
+ }
+
+ if (inv_queue->cmd_base) {
+ folio_put(virt_to_folio(inv_queue->cmd_base));
+ inv_queue->cmd_base = NULL;
+ }
+}
+
+static int __init redirect_irde_init(int node)
+{
+ struct redirect_desc *irde = &redirect_descs[node];
+ int ret;
+
+ irde->node = node;
+
+ ret = redirect_table_init(irde);
+ if (ret)
+ return ret;
+
+ ret = redirect_queue_init(irde);
+ if (ret) {
+ redirect_irde_free(irde);
+ return ret;
+ }
+
+ redirect_irde_cfg(irde);
+
+ return 0;
+}
+
+static int __init pch_msi_parse_madt(union acpi_subtable_headers *header, const unsigned long end)
+{
+ struct acpi_madt_msi_pic *pchmsi_entry = (struct acpi_madt_msi_pic *)header;
+
+ msi_base_addr = pchmsi_entry->msg_address - AVEC_MSG_OFFSET;
+
+ return pch_msi_acpi_init_avec(redirect_domain);
+}
+
+static int __init acpi_cascade_irqdomain_init(void)
+{
+ return acpi_table_parse_madt(ACPI_MADT_TYPE_MSI_PIC, pch_msi_parse_madt, 1);
+}
+
+int __init redirect_acpi_init(struct irq_domain *parent)
+{
+ struct fwnode_handle *fwnode;
+ int ret = -EINVAL, node;
+
+ fwnode = irq_domain_alloc_named_fwnode("redirect");
+ if (!fwnode) {
+ pr_err("Unable to alloc redirect domain handle\n");
+ goto fail;
+ }
+
+ redirect_domain = irq_domain_create_hierarchy(parent, 0, IRD_ENTRIES, fwnode,
+ &redirect_domain_ops, redirect_descs);
+ if (!redirect_domain) {
+ pr_err("Unable to alloc redirect domain\n");
+ goto out_free_fwnode;
+ }
+
+ for_each_node_mask(node, node_possible_map) {
+ ret = redirect_irde_init(node);
+ if (ret)
+ goto out_clear_irde;
+ }
+
+ ret = acpi_cascade_irqdomain_init();
+ if (ret < 0) {
+ pr_err("Failed to cascade IRQ domain, ret=%d\n", ret);
+ goto out_clear_irde;
+ }
+
+ pr_info("init succeeded\n");
+
+ return 0;
+
+out_clear_irde:
+ for_each_node_mask(node, node_possible_map) {
+ redirect_irde_free(&redirect_descs[node]);
+ }
+ irq_domain_remove(redirect_domain);
+out_free_fwnode:
+ irq_domain_free_fwnode(fwnode);
+fail:
+ return ret;
+}
projects / thirdparty / linux.git / commitdiff
