#include <linux/kernel_stat.h>
#include <linux/pci.h>
#include <linux/msi.h>
+#include <linux/irqchip/irq-msi-lib.h>
#include <linux/smp.h>
#include <asm/isc.h>
}
/* Register adapter interruptions */
-static int zpci_set_irq(struct zpci_dev *zdev)
+int zpci_set_irq(struct zpci_dev *zdev)
{
int rc;
static int zpci_set_irq_affinity(struct irq_data *data, const struct cpumask *dest,
bool force)
{
- struct msi_desc *entry = irq_data_get_msi_desc(data);
- struct msi_msg msg = entry->msg;
- int cpu_addr = smp_cpu_get_cpu_address(cpumask_first(dest));
+ irq_data_update_affinity(data, dest);
+ return IRQ_SET_MASK_OK;
+}
- msg.address_lo &= 0xff0000ff;
- msg.address_lo |= (cpu_addr << 8);
- pci_write_msi_msg(data->irq, &msg);
+/*
+ * Encode the hwirq number for the parent domain. The encoding must be unique
+ * for each IRQ of each device in the parent domain, so it uses the devfn to
+ * identify the device and the msi_index to identify the IRQ within that device.
+ */
+static inline u32 zpci_encode_hwirq(u8 devfn, u16 msi_index)
+{
+ return (devfn << 16) | msi_index;
+}
- return IRQ_SET_MASK_OK;
+static inline u16 zpci_decode_hwirq_msi_index(irq_hw_number_t hwirq)
+{
+ return hwirq & 0xffff;
+}
+
+static void zpci_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
+{
+ struct msi_desc *desc = irq_data_get_msi_desc(data);
+ struct zpci_dev *zdev = to_zpci_dev(desc->dev);
+
+ if (irq_delivery == DIRECTED) {
+ int cpu = cpumask_first(irq_data_get_affinity_mask(data));
+
+ msg->address_lo = zdev->msi_addr & 0xff0000ff;
+ msg->address_lo |= (smp_cpu_get_cpu_address(cpu) << 8);
+ } else {
+ msg->address_lo = zdev->msi_addr & 0xffffffff;
+ }
+ msg->address_hi = zdev->msi_addr >> 32;
+ msg->data = zpci_decode_hwirq_msi_index(data->hwirq);
}
static struct irq_chip zpci_irq_chip = {
.name = "PCI-MSI",
- .irq_unmask = pci_msi_unmask_irq,
- .irq_mask = pci_msi_mask_irq,
+ .irq_compose_msi_msg = zpci_compose_msi_msg,
};
static void zpci_handle_cpu_local_irq(bool rescan)
{
struct airq_iv *dibv = zpci_ibv[smp_processor_id()];
union zpci_sic_iib iib = {{0}};
+ struct irq_domain *msi_domain;
+ irq_hw_number_t hwirq;
unsigned long bit;
int irqs_on = 0;
continue;
}
inc_irq_stat(IRQIO_MSI);
- generic_handle_irq(airq_iv_get_data(dibv, bit));
+ hwirq = airq_iv_get_data(dibv, bit);
+ msi_domain = (struct irq_domain *)airq_iv_get_ptr(dibv, bit);
+ generic_handle_domain_irq(msi_domain, hwirq);
}
}
struct tpi_info *tpi_info)
{
union zpci_sic_iib iib = {{0}};
+ struct irq_domain *msi_domain;
+ irq_hw_number_t hwirq;
unsigned long si, ai;
struct airq_iv *aibv;
int irqs_on = 0;
break;
inc_irq_stat(IRQIO_MSI);
airq_iv_lock(aibv, ai);
- generic_handle_irq(airq_iv_get_data(aibv, ai));
+ hwirq = airq_iv_get_data(aibv, ai);
+ msi_domain = (struct irq_domain *)airq_iv_get_ptr(aibv, ai);
+ generic_handle_domain_irq(msi_domain, hwirq);
airq_iv_unlock(aibv, ai);
}
}
zdev->aisb = *bit;
/* Create adapter interrupt vector */
- zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK, NULL);
+ zdev->aibv = airq_iv_create(msi_vecs,
+ AIRQ_IV_PTR | AIRQ_IV_DATA | AIRQ_IV_BITLOCK,
+ NULL);
if (!zdev->aibv)
return -ENOMEM;
return 0;
}
-int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
+bool arch_restore_msi_irqs(struct pci_dev *pdev)
{
- unsigned int hwirq, msi_vecs, irqs_per_msi, i, cpu;
struct zpci_dev *zdev = to_zpci(pdev);
- struct msi_desc *msi;
- struct msi_msg msg;
- unsigned long bit;
- int cpu_addr;
- int rc, irq;
+ zpci_set_irq(zdev);
+ return true;
+}
+
+static struct airq_struct zpci_airq = {
+ .handler = zpci_floating_irq_handler,
+ .isc = PCI_ISC,
+};
+
+static void zpci_msi_teardown_directed(struct zpci_dev *zdev)
+{
+ airq_iv_free(zpci_ibv[0], zdev->msi_first_bit, zdev->max_msi);
+ zdev->msi_first_bit = -1U;
+ zdev->msi_nr_irqs = 0;
+}
+
+static void zpci_msi_teardown_floating(struct zpci_dev *zdev)
+{
+ airq_iv_release(zdev->aibv);
+ zdev->aibv = NULL;
+ airq_iv_free_bit(zpci_sbv, zdev->aisb);
zdev->aisb = -1UL;
zdev->msi_first_bit = -1U;
+ zdev->msi_nr_irqs = 0;
+}
+
+static void zpci_msi_teardown(struct irq_domain *domain, msi_alloc_info_t *arg)
+{
+ struct zpci_dev *zdev = to_zpci_dev(domain->dev);
+
+ zpci_clear_irq(zdev);
+ if (irq_delivery == DIRECTED)
+ zpci_msi_teardown_directed(zdev);
+ else
+ zpci_msi_teardown_floating(zdev);
+}
+
+static int zpci_msi_prepare(struct irq_domain *domain,
+ struct device *dev, int nvec,
+ msi_alloc_info_t *info)
+{
+ struct zpci_dev *zdev = to_zpci_dev(dev);
+ struct pci_dev *pdev = to_pci_dev(dev);
+ unsigned long bit;
+ int msi_vecs, rc;
msi_vecs = min_t(unsigned int, nvec, zdev->max_msi);
if (msi_vecs < nvec) {
- pr_info("%s requested %d irqs, allocate system limit of %d",
+ pr_info("%s requested %d IRQs, allocate system limit of %d\n",
pci_name(pdev), nvec, zdev->max_msi);
}
rc = __alloc_airq(zdev, msi_vecs, &bit);
- if (rc < 0)
+ if (rc) {
+ pr_err("Allocating adapter IRQs for %s failed\n", pci_name(pdev));
return rc;
+ }
- /*
- * Request MSI interrupts:
- * When using MSI, nvec_used interrupt sources and their irq
- * descriptors are controlled through one msi descriptor.
- * Thus the outer loop over msi descriptors shall run only once,
- * while two inner loops iterate over the interrupt vectors.
- * When using MSI-X, each interrupt vector/irq descriptor
- * is bound to exactly one msi descriptor (nvec_used is one).
- * So the inner loops are executed once, while the outer iterates
- * over the MSI-X descriptors.
- */
- hwirq = bit;
- msi_for_each_desc(msi, &pdev->dev, MSI_DESC_NOTASSOCIATED) {
- if (hwirq - bit >= msi_vecs)
- break;
- irqs_per_msi = min_t(unsigned int, msi_vecs, msi->nvec_used);
- irq = __irq_alloc_descs(-1, 0, irqs_per_msi, 0, THIS_MODULE,
- (irq_delivery == DIRECTED) ?
- msi->affinity : NULL);
- if (irq < 0)
- return -ENOMEM;
+ zdev->msi_first_bit = bit;
+ zdev->msi_nr_irqs = msi_vecs;
+ rc = zpci_set_irq(zdev);
+ if (rc) {
+ pr_err("Registering adapter IRQs for %s failed\n",
+ pci_name(pdev));
+
+ if (irq_delivery == DIRECTED)
+ zpci_msi_teardown_directed(zdev);
+ else
+ zpci_msi_teardown_floating(zdev);
+ return rc;
+ }
+ return 0;
+}
- for (i = 0; i < irqs_per_msi; i++) {
- rc = irq_set_msi_desc_off(irq, i, msi);
- if (rc)
- return rc;
- irq_set_chip_and_handler(irq + i, &zpci_irq_chip,
- handle_percpu_irq);
- }
+static int zpci_msi_domain_alloc(struct irq_domain *domain, unsigned int virq,
+ unsigned int nr_irqs, void *args)
+{
+ struct msi_desc *desc = ((msi_alloc_info_t *)args)->desc;
+ struct zpci_dev *zdev = to_zpci_dev(desc->dev);
+ struct zpci_bus *zbus = zdev->zbus;
+ unsigned int cpu, hwirq;
+ unsigned long bit;
+ int i;
- msg.data = hwirq - bit;
- if (irq_delivery == DIRECTED) {
- if (msi->affinity)
- cpu = cpumask_first(&msi->affinity->mask);
- else
- cpu = 0;
- cpu_addr = smp_cpu_get_cpu_address(cpu);
+ bit = zdev->msi_first_bit + desc->msi_index;
+ hwirq = zpci_encode_hwirq(zdev->devfn, desc->msi_index);
- msg.address_lo = zdev->msi_addr & 0xff0000ff;
- msg.address_lo |= (cpu_addr << 8);
+ if (desc->msi_index + nr_irqs > zdev->max_msi)
+ return -EINVAL;
+ for (i = 0; i < nr_irqs; i++) {
+ irq_domain_set_info(domain, virq + i, hwirq + i,
+ &zpci_irq_chip, zdev,
+ handle_percpu_irq, NULL, NULL);
+
+ if (irq_delivery == DIRECTED) {
for_each_possible_cpu(cpu) {
- for (i = 0; i < irqs_per_msi; i++)
- airq_iv_set_data(zpci_ibv[cpu],
- hwirq + i, irq + i);
+ airq_iv_set_ptr(zpci_ibv[cpu], bit + i,
+ (unsigned long)zbus->msi_parent_domain);
+ airq_iv_set_data(zpci_ibv[cpu], bit + i, hwirq + i);
}
} else {
- msg.address_lo = zdev->msi_addr & 0xffffffff;
- for (i = 0; i < irqs_per_msi; i++)
- airq_iv_set_data(zdev->aibv, hwirq + i, irq + i);
+ airq_iv_set_ptr(zdev->aibv, bit + i,
+ (unsigned long)zbus->msi_parent_domain);
+ airq_iv_set_data(zdev->aibv, bit + i, hwirq + i);
}
- msg.address_hi = zdev->msi_addr >> 32;
- pci_write_msi_msg(irq, &msg);
- hwirq += irqs_per_msi;
}
- zdev->msi_first_bit = bit;
- zdev->msi_nr_irqs = hwirq - bit;
-
- rc = zpci_set_irq(zdev);
- if (rc)
- return rc;
-
- return (zdev->msi_nr_irqs == nvec) ? 0 : zdev->msi_nr_irqs;
+ return 0;
}
-void arch_teardown_msi_irqs(struct pci_dev *pdev)
+static void zpci_msi_clear_airq(struct irq_data *d, int i)
{
- struct zpci_dev *zdev = to_zpci(pdev);
- struct msi_desc *msi;
- unsigned int i;
- int rc;
+ struct msi_desc *desc = irq_data_get_msi_desc(d);
+ struct zpci_dev *zdev = to_zpci_dev(desc->dev);
+ unsigned long bit;
+ unsigned int cpu;
+ u16 msi_index;
- /* Disable interrupts */
- rc = zpci_clear_irq(zdev);
- if (rc)
- return;
+ msi_index = zpci_decode_hwirq_msi_index(d->hwirq);
+ bit = zdev->msi_first_bit + msi_index;
- /* Release MSI interrupts */
- msi_for_each_desc(msi, &pdev->dev, MSI_DESC_ASSOCIATED) {
- for (i = 0; i < msi->nvec_used; i++) {
- irq_set_msi_desc(msi->irq + i, NULL);
- irq_free_desc(msi->irq + i);
+ if (irq_delivery == DIRECTED) {
+ for_each_possible_cpu(cpu) {
+ airq_iv_set_ptr(zpci_ibv[cpu], bit + i, 0);
+ airq_iv_set_data(zpci_ibv[cpu], bit + i, 0);
}
- msi->msg.address_lo = 0;
- msi->msg.address_hi = 0;
- msi->msg.data = 0;
- msi->irq = 0;
+ } else {
+ airq_iv_set_ptr(zdev->aibv, bit + i, 0);
+ airq_iv_set_data(zdev->aibv, bit + i, 0);
}
+}
- if (zdev->aisb != -1UL) {
- zpci_ibv[zdev->aisb] = NULL;
- airq_iv_free_bit(zpci_sbv, zdev->aisb);
- zdev->aisb = -1UL;
- }
- if (zdev->aibv) {
- airq_iv_release(zdev->aibv);
- zdev->aibv = NULL;
- }
+static void zpci_msi_domain_free(struct irq_domain *domain, unsigned int virq,
+ unsigned int nr_irqs)
+{
+ struct irq_data *d;
+ int i;
- if ((irq_delivery == DIRECTED) && zdev->msi_first_bit != -1U)
- airq_iv_free(zpci_ibv[0], zdev->msi_first_bit, zdev->msi_nr_irqs);
+ for (i = 0; i < nr_irqs; i++) {
+ d = irq_domain_get_irq_data(domain, virq + i);
+ zpci_msi_clear_airq(d, i);
+ irq_domain_reset_irq_data(d);
+ }
}
-bool arch_restore_msi_irqs(struct pci_dev *pdev)
+static const struct irq_domain_ops zpci_msi_domain_ops = {
+ .alloc = zpci_msi_domain_alloc,
+ .free = zpci_msi_domain_free,
+};
+
+static bool zpci_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
+ struct irq_domain *real_parent,
+ struct msi_domain_info *info)
{
- struct zpci_dev *zdev = to_zpci(pdev);
+ if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info))
+ return false;
+
+ info->ops->msi_prepare = zpci_msi_prepare;
+ info->ops->msi_teardown = zpci_msi_teardown;
- zpci_set_irq(zdev);
return true;
}
-static struct airq_struct zpci_airq = {
- .handler = zpci_floating_irq_handler,
- .isc = PCI_ISC,
+static struct msi_parent_ops zpci_msi_parent_ops = {
+ .supported_flags = MSI_GENERIC_FLAGS_MASK |
+ MSI_FLAG_PCI_MSIX |
+ MSI_FLAG_MULTI_PCI_MSI,
+ .required_flags = MSI_FLAG_USE_DEF_DOM_OPS |
+ MSI_FLAG_USE_DEF_CHIP_OPS,
+ .init_dev_msi_info = zpci_init_dev_msi_info,
};
+int zpci_create_parent_msi_domain(struct zpci_bus *zbus)
+{
+ char fwnode_name[18];
+
+ snprintf(fwnode_name, sizeof(fwnode_name), "ZPCI_MSI_DOM_%04x", zbus->domain_nr);
+ struct irq_domain_info info = {
+ .fwnode = irq_domain_alloc_named_fwnode(fwnode_name),
+ .ops = &zpci_msi_domain_ops,
+ };
+
+ if (!info.fwnode) {
+ pr_err("Failed to allocate fwnode for MSI IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ if (irq_delivery == FLOATING)
+ zpci_msi_parent_ops.required_flags |= MSI_FLAG_NO_AFFINITY;
+
+ zbus->msi_parent_domain = msi_create_parent_irq_domain(&info, &zpci_msi_parent_ops);
+ if (!zbus->msi_parent_domain) {
+ irq_domain_free_fwnode(info.fwnode);
+ pr_err("Failed to create MSI IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+void zpci_remove_parent_msi_domain(struct zpci_bus *zbus)
+{
+ struct fwnode_handle *fn;
+
+ fn = zbus->msi_parent_domain->fwnode;
+ irq_domain_remove(zbus->msi_parent_domain);
+ irq_domain_free_fwnode(fn);
+}
+
static void __init cpu_enable_directed_irq(void *unused)
{
union zpci_sic_iib iib = {{0}};
* is only done on the first vector.
*/
zpci_ibv[cpu] = airq_iv_create(cache_line_size() * BITS_PER_BYTE,
+ AIRQ_IV_PTR |
AIRQ_IV_DATA |
AIRQ_IV_CACHELINE |
(!cpu ? AIRQ_IV_ALLOC : 0), NULL);
projects / thirdparty / kernel / linux.git / commitdiff
