+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2020 Loongson Technologies, Inc.
- */
-
-#include <linux/cpuhotplug.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/irqchip.h>
-#include <linux/irqchip/chained_irq.h>
-#include <linux/irqdomain.h>
-#include <linux/kernel.h>
-#include <linux/msi.h>
-#include <linux/radix-tree.h>
-#include <linux/spinlock.h>
-
-#include <asm/loongarch.h>
-#include <asm/setup.h>
-
-#define VECTORS_PER_REG 64
-#define ILR_INVALID_MASK 0x80000000UL
-#define ILR_VECTOR_MASK 0xffUL
-#define AVEC_MSG_OFFSET 0x100000
-
-static phys_addr_t msi_base_v2;
-static DEFINE_PER_CPU(struct irq_desc * [NR_VECTORS], irq_map);
-
-struct pending_list {
- struct list_head head;
-};
-
-static DEFINE_PER_CPU(struct pending_list, pending_list);
-
-struct loongarch_avec_chip {
- struct fwnode_handle *fwnode;
- struct irq_domain *domain;
- struct irq_matrix *vector_matrix;
- raw_spinlock_t lock;
-};
-
-static struct loongarch_avec_chip loongarch_avec;
-
-struct loongarch_avec_data {
- struct list_head entry;
- unsigned int cpu;
- unsigned int vec;
- unsigned int prev_cpu;
- unsigned int prev_vec;
- unsigned int moving : 1,
- managed : 1;
-};
-
-static struct cpumask intersect_mask;
-
-static int assign_irq_vector(struct irq_data *irqd, const struct cpumask *dest,
- unsigned int *cpu)
-{
- return irq_matrix_alloc(loongarch_avec.vector_matrix, dest, false, cpu);
-}
-
-static inline void loongarch_avec_ack_irq(struct irq_data *d)
-{
-}
-
-static inline void loongarch_avec_unmask_irq(struct irq_data *d)
-{
-}
-
-static inline void loongarch_avec_mask_irq(struct irq_data *d)
-{
-}
-
-static void loongarch_avec_sync(struct loongarch_avec_data *adata)
-{
- struct pending_list *plist;
-
- if (cpu_online(adata->prev_cpu)) {
- plist = per_cpu_ptr(&pending_list, adata->prev_cpu);
- list_add_tail(&adata->entry, &plist->head);
- adata->moving = true;
- loongson_send_ipi_single(adata->prev_cpu, SMP_CLEAR_VECT);
- }
- adata->prev_cpu = adata->cpu;
- adata->prev_vec = adata->vec;
-}
-
-static int loongarch_avec_set_affinity(struct irq_data *data, const struct cpumask *dest,
- bool force)
-{
- struct loongarch_avec_data *adata;
- unsigned int cpu, vector;
- unsigned long flags;
- int ret;
-
- raw_spin_lock_irqsave(&loongarch_avec.lock, flags);
- adata = irq_data_get_irq_chip_data(data);
-
- if (adata->vec && cpu_online(adata->cpu) && cpumask_test_cpu(adata->cpu, dest)) {
- raw_spin_unlock_irqrestore(&loongarch_avec.lock, flags);
- return 0;
- }
- if (adata->moving)
- return -EBUSY;
-
- cpumask_and(&intersect_mask, dest, cpu_online_mask);
-
- ret = assign_irq_vector(data, &intersect_mask, &cpu);
- if (ret < 0) {
- raw_spin_unlock_irqrestore(&loongarch_avec.lock, flags);
- return ret;
- }
- vector = ret;
- adata->cpu = cpu;
- adata->vec = vector;
- per_cpu_ptr(irq_map, adata->cpu)[adata->vec] = irq_data_to_desc(data);
- loongarch_avec_sync(adata);
-
- raw_spin_unlock_irqrestore(&loongarch_avec.lock, flags);
- irq_data_update_effective_affinity(data, cpumask_of(cpu));
-
- return IRQ_SET_MASK_OK;
-}
-
-static void loongarch_avec_compose_msg(struct irq_data *d,
- struct msi_msg *msg)
-{
- struct loongarch_avec_data *avec_data;
-
- avec_data = irq_data_get_irq_chip_data(d);
-
- msg->address_hi = 0x0;
- msg->address_lo = msi_base_v2 | ((avec_data->vec & 0xff) << 4) |
- ((cpu_logical_map(avec_data->cpu & 0xffff)) << 12);
- msg->data = 0x0;
-
-}
-
-static struct irq_chip loongarch_avec_controller = {
- .name = "CORE_AVEC",
- .irq_ack = loongarch_avec_ack_irq,
- .irq_mask = loongarch_avec_mask_irq,
- .irq_unmask = loongarch_avec_unmask_irq,
- .irq_set_affinity = loongarch_avec_set_affinity,
- .irq_compose_msi_msg = loongarch_avec_compose_msg,
-};
-
-void complete_irq_moving(void)
-{
- struct pending_list *plist = this_cpu_ptr(&pending_list);
- struct loongarch_avec_data *adata, *tmp;
- int cpu, vector, bias;
- u64 irr;
-
- raw_spin_lock(&loongarch_avec.lock);
-
- list_for_each_entry_safe(adata, tmp, &plist->head, entry) {
- cpu = adata->prev_cpu;
- vector = adata->prev_vec;
- bias = vector / VECTORS_PER_REG;
- switch (bias) {
- case 0:
- irr = csr_read64(LOONGARCH_CSR_IRR0);
- case 1:
- irr = csr_read64(LOONGARCH_CSR_IRR1);
- case 2:
- irr = csr_read64(LOONGARCH_CSR_IRR2);
- case 3:
- irr = csr_read64(LOONGARCH_CSR_IRR3);
- }
-
- if (irr & (1UL << (vector % VECTORS_PER_REG))) {
- loongson_send_ipi_single(cpu, SMP_CLEAR_VECT);
- continue;
- }
- list_del(&adata->entry);
- irq_matrix_free(loongarch_avec.vector_matrix, cpu, vector, adata->managed);
- this_cpu_write(irq_map[vector], NULL);
- adata->moving = 0;
- }
- raw_spin_unlock(&loongarch_avec.lock);
-}
-
-static void loongarch_avec_dispatch(struct irq_desc *desc)
-{
- struct irq_chip *chip = irq_desc_get_chip(desc);
- unsigned long vector;
- struct irq_desc *d;
-
- chained_irq_enter(chip, desc);
- vector = csr_read64(LOONGARCH_CSR_ILR);
- if (vector & ILR_INVALID_MASK)
- return;
-
- vector &= ILR_VECTOR_MASK;
-
- d = this_cpu_read(irq_map[vector]);
- if (d) {
- generic_handle_irq_desc(d);
- } else {
- pr_warn("IRQ ERROR:Unexpected irq occur on cpu %d[vector %ld]\n",
- smp_processor_id(), vector);
- }
-
- chained_irq_exit(chip, desc);
-}
-
-static int loongarch_avec_alloc(struct irq_domain *domain, unsigned int virq,
- unsigned int nr_irqs, void *arg)
-{
- struct loongarch_avec_data *adata;
- struct irq_data *irqd;
- unsigned int cpu, vector, i, ret;
- unsigned long flags;
-
- raw_spin_lock_irqsave(&loongarch_avec.lock, flags);
- for (i = 0; i < nr_irqs; i++) {
- irqd = irq_domain_get_irq_data(domain, virq + i);
- adata = kzalloc(sizeof(*adata), GFP_KERNEL);
- if (!adata) {
- raw_spin_unlock_irqrestore(&loongarch_avec.lock, flags);
- return -ENOMEM;
- }
- ret = assign_irq_vector(irqd, cpu_online_mask, &cpu);
- if (ret < 0) {
- raw_spin_unlock_irqrestore(&loongarch_avec.lock, flags);
- return ret;
- }
- vector = ret;
- adata->prev_cpu = adata->cpu = cpu;
- adata->prev_vec = adata->vec = vector;
- adata->managed = irqd_affinity_is_managed(irqd);
- irq_domain_set_info(domain, virq + i, virq + i, &loongarch_avec_controller,
- adata, handle_edge_irq, NULL, NULL);
- adata->moving = 0;
- irqd_set_single_target(irqd);
- irqd_set_affinity_on_activate(irqd);
-
- per_cpu_ptr(irq_map, adata->cpu)[adata->vec] = irq_data_to_desc(irqd);
- }
- raw_spin_unlock_irqrestore(&loongarch_avec.lock, flags);
-
- return 0;
-}
-
-static void clear_free_vector(struct irq_data *irqd)
-{
- struct loongarch_avec_data *adata = irq_data_get_irq_chip_data(irqd);
- bool managed = irqd_affinity_is_managed(irqd);
-
- per_cpu(irq_map, adata->cpu)[adata->vec] = NULL;
- irq_matrix_free(loongarch_avec.vector_matrix, adata->cpu, adata->vec, managed);
- adata->cpu = 0;
- adata->vec = 0;
- if (!adata->moving)
- return;
-
- per_cpu(irq_map, adata->prev_cpu)[adata->prev_vec] = 0;
- irq_matrix_free(loongarch_avec.vector_matrix, adata->prev_cpu,
- adata->prev_vec, adata->managed);
- adata->prev_vec = 0;
- adata->prev_cpu = 0;
- adata->moving = 0;
- list_del_init(&adata->entry);
-}
-
-static void loongarch_avec_free(struct irq_domain *domain, unsigned int virq,
- unsigned int nr_irqs)
-{
- struct irq_data *d;
- unsigned long flags;
- unsigned int i;
-
- raw_spin_lock_irqsave(&loongarch_avec.lock, flags);
- for (i = 0; i < nr_irqs; i++) {
- d = irq_domain_get_irq_data(domain, virq + i);
- if (d) {
- clear_free_vector(d);
- irq_domain_reset_irq_data(d);
-
- }
- }
-
- raw_spin_unlock_irqrestore(&loongarch_avec.lock, flags);
-}
-
-static const struct irq_domain_ops loongarch_avec_domain_ops = {
- .alloc = loongarch_avec_alloc,
- .free = loongarch_avec_free,
-};
-
-static int __init irq_matrix_init(void)
-{
- int i;
-
- loongarch_avec.vector_matrix = irq_alloc_matrix(NR_VECTORS, 0, NR_VECTORS - 1);
- if (!loongarch_avec.vector_matrix)
- return -ENOMEM;
- for (i = 0; i < NR_LEGACY_VECTORS; i++)
- irq_matrix_assign_system(loongarch_avec.vector_matrix, i, false);
-
- irq_matrix_online(loongarch_avec.vector_matrix);
-
- return 0;
-}
-
-static int __init loongarch_avec_init(struct irq_domain *parent)
-{
- struct pending_list *plist = per_cpu_ptr(&pending_list, 0);
- int ret = 0, parent_irq;
- unsigned long tmp;
-
- raw_spin_lock_init(&loongarch_avec.lock);
-
- loongarch_avec.fwnode = irq_domain_alloc_named_fwnode("CORE_AVEC");
- if (!loongarch_avec.fwnode) {
- pr_err("Unable to allocate domain handle\n");
- ret = -ENOMEM;
- goto out;
- }
-
- loongarch_avec.domain = irq_domain_create_tree(loongarch_avec.fwnode,
- &loongarch_avec_domain_ops, NULL);
- if (!loongarch_avec.domain) {
- pr_err("core-vec: cannot create IRQ domain\n");
- ret = -ENOMEM;
- goto out_free_handle;
- }
-
- parent_irq = irq_create_mapping(parent, INT_AVEC);
- if (!parent_irq) {
- pr_err("Failed to mapping hwirq\n");
- ret = -EINVAL;
- goto out_remove_domain;
- }
- irq_set_chained_handler_and_data(parent_irq, loongarch_avec_dispatch, NULL);
-
- ret = irq_matrix_init();
- if (ret) {
- pr_err("Failed to init irq matrix\n");
- goto out_free_matrix;
- }
-
- INIT_LIST_HEAD(&plist->head);
- tmp = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
- tmp |= IOCSR_MISC_FUNC_AVEC_EN;
- iocsr_write64(tmp, LOONGARCH_IOCSR_MISC_FUNC);
-
- return ret;
-
-out_free_matrix:
- kfree(loongarch_avec.vector_matrix);
-out_remove_domain:
- irq_domain_remove(loongarch_avec.domain);
-out_free_handle:
- irq_domain_free_fwnode(loongarch_avec.fwnode);
-out:
- return ret;
-}
-
-void loongarch_avec_offline_cpu(unsigned int cpu)
-{
- struct pending_list *plist = per_cpu_ptr(&pending_list, cpu);
- unsigned long flags;
-
- raw_spin_lock_irqsave(&loongarch_avec.lock, flags);
- if (list_empty(&plist->head))
- irq_matrix_offline(loongarch_avec.vector_matrix);
- else
- pr_warn("cpu %d advanced extioi is busy\n", cpu);
- raw_spin_unlock_irqrestore(&loongarch_avec.lock, flags);
-}
-
-void loongarch_avec_online_cpu(unsigned int cpu)
-{
- struct pending_list *plist = per_cpu_ptr(&pending_list, cpu);
- unsigned long flags;
-
- raw_spin_lock_irqsave(&loongarch_avec.lock, flags);
-
- irq_matrix_online(loongarch_avec.vector_matrix);
-
- INIT_LIST_HEAD(&plist->head);
-
- raw_spin_unlock_irqrestore(&loongarch_avec.lock, flags);
-}
-
-static int __init pch_msi_parse_madt(union acpi_subtable_headers *header,
- const unsigned long end)
-{
- struct acpi_madt_msi_pic *pchmsi_entry = (struct acpi_madt_msi_pic *)header;
-
- msi_base_v2 = pchmsi_entry->msg_address - AVEC_MSG_OFFSET;
- return pch_msi_acpi_init_v2(loongarch_avec.domain, pchmsi_entry);
-}
-
-static inline int __init acpi_cascade_irqdomain_init(void)
-{
- return acpi_table_parse_madt(ACPI_MADT_TYPE_MSI_PIC, pch_msi_parse_madt, 1);
-}
-
-int __init loongarch_avec_acpi_init(struct irq_domain *parent)
-{
- int ret = 0;
-
- ret = loongarch_avec_init(parent);
- if (ret) {
- pr_err("Failed to init irq domain\n");
- return ret;
- }
-
- ret = acpi_cascade_irqdomain_init();
- if (ret) {
- pr_err("Failed to cascade IRQ domain\n");
- return ret;
- }
-
- return ret;
-}
projects / thirdparty / kernel / linux.git / commitdiff
