hisi_acc_vfio_pci: adapt to new migration configuration

On new platforms greater than QM_HW_V3, the migration region has been
relocated from the VF to the PF. The VF's own configuration space is
restored to the complete 64KB, and there is no need to divide the
size of the BAR configuration space equally. The driver should be
modified accordingly to adapt to the new hardware device.

On the older hardware platform QM_HW_V3, the live migration configuration
region is placed in the latter 32K portion of the VF's BAR2 configuration
space. On the new hardware platform QM_HW_V4, the live migration
configuration region also exists in the same 32K area immediately following
the VF's BAR2, just like on QM_HW_V3.

However, access to this region is now controlled by hardware. Additionally,
a copy of the live migration configuration region is present in the PF's
BAR2 configuration space. On the new hardware platform QM_HW_V4, when an
older version of the driver is loaded, it behaves like QM_HW_V3 and uses
the configuration region in the VF, ensuring that the live migration
function continues to work normally. When the new version of the driver is
loaded, it directly uses the configuration region in the PF. Meanwhile,
hardware configuration disables the live migration configuration region
in the VF's BAR2: reads return all 0xF values, and writes are silently
ignored.

Signed-off-by: Longfang Liu <liulongfang@huawei.com>
Reviewed-by: Shameer Kolothum <shameerkolothum@gmail.com>
Link: https://lore.kernel.org/r/20251030015744.131771-3-liulongfang@huawei.com
Signed-off-by: Alex Williamson <alex@shazbot.org>

diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
index fde33f54e99ec56f59b80373635be502a129c9d9..498cb7d1c9e508576797f16d03b54d17257359dc 100644 (file)
--- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
+++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
@@ -125,9 +125,25 @@ static int qm_get_cqc(struct hisi_qm *qm, u64 *addr)
        return 0;
 }
 
+static void qm_xqc_reg_offsets(struct hisi_qm *qm,
+                              u32 *eqc_addr, u32 *aeqc_addr)
+{
+       struct hisi_acc_vf_core_device *hisi_acc_vdev =
+               container_of(qm, struct hisi_acc_vf_core_device, vf_qm);
+
+       if (hisi_acc_vdev->drv_mode == HW_ACC_MIG_VF_CTRL) {
+               *eqc_addr = QM_EQC_VF_DW0;
+               *aeqc_addr = QM_AEQC_VF_DW0;
+       } else {
+               *eqc_addr = QM_EQC_PF_DW0;
+               *aeqc_addr = QM_AEQC_PF_DW0;
+       }
+}
+
 static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
 {
        struct device *dev = &qm->pdev->dev;
+       u32 eqc_addr, aeqc_addr;
        int ret;
 
        ret = qm_read_regs(qm, QM_VF_AEQ_INT_MASK, &vf_data->aeq_int_mask, 1);
@@ -167,15 +183,16 @@ static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
                return ret;
        }
 
+       qm_xqc_reg_offsets(qm, &eqc_addr, &aeqc_addr);
        /* QM_EQC_DW has 7 regs */
-       ret = qm_read_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7);
+       ret = qm_read_regs(qm, eqc_addr, vf_data->qm_eqc_dw, 7);
        if (ret) {
                dev_err(dev, "failed to read QM_EQC_DW\n");
                return ret;
        }
 
        /* QM_AEQC_DW has 7 regs */
-       ret = qm_read_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw, 7);
+       ret = qm_read_regs(qm, aeqc_addr, vf_data->qm_aeqc_dw, 7);
        if (ret) {
                dev_err(dev, "failed to read QM_AEQC_DW\n");
                return ret;
@@ -187,6 +204,7 @@ static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
 static int qm_set_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
 {
        struct device *dev = &qm->pdev->dev;
+       u32 eqc_addr, aeqc_addr;
        int ret;
 
        /* Check VF state */
@@ -239,15 +257,16 @@ static int qm_set_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
                return ret;
        }
 
+       qm_xqc_reg_offsets(qm, &eqc_addr, &aeqc_addr);
        /* QM_EQC_DW has 7 regs */
-       ret = qm_write_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7);
+       ret = qm_write_regs(qm, eqc_addr, vf_data->qm_eqc_dw, 7);
        if (ret) {
                dev_err(dev, "failed to write QM_EQC_DW\n");
                return ret;
        }
 
        /* QM_AEQC_DW has 7 regs */
-       ret = qm_write_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw, 7);
+       ret = qm_write_regs(qm, aeqc_addr, vf_data->qm_aeqc_dw, 7);
        if (ret) {
                dev_err(dev, "failed to write QM_AEQC_DW\n");
                return ret;
@@ -1186,34 +1205,52 @@ static int hisi_acc_vf_qm_init(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 {
        struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;
        struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
+       struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;
        struct pci_dev *vf_dev = vdev->pdev;
+       u32 val;
 
-       /*
-        * ACC VF dev BAR2 region consists of both functional register space
-        * and migration control register space. For migration to work, we
-        * need access to both. Hence, we map the entire BAR2 region here.
-        * But unnecessarily exposing the migration BAR region to the Guest
-        * has the potential to prevent/corrupt the Guest migration. Hence,
-        * we restrict access to the migration control space from
-        * Guest(Please see mmap/ioctl/read/write override functions).
-        *
-        * Please note that it is OK to expose the entire VF BAR if migration
-        * is not supported or required as this cannot affect the ACC PF
-        * configurations.
-        *
-        * Also the HiSilicon ACC VF devices supported by this driver on
-        * HiSilicon hardware platforms are integrated end point devices
-        * and the platform lacks the capability to perform any PCIe P2P
-        * between these devices.
-        */
+       val = readl(pf_qm->io_base + QM_MIG_REGION_SEL);
+       if (pf_qm->ver > QM_HW_V3 && (val & QM_MIG_REGION_EN))
+               hisi_acc_vdev->drv_mode = HW_ACC_MIG_PF_CTRL;
+       else
+               hisi_acc_vdev->drv_mode = HW_ACC_MIG_VF_CTRL;
 
-       vf_qm->io_base =
-               ioremap(pci_resource_start(vf_dev, VFIO_PCI_BAR2_REGION_INDEX),
-                       pci_resource_len(vf_dev, VFIO_PCI_BAR2_REGION_INDEX));
-       if (!vf_qm->io_base)
-               return -EIO;
+       if (hisi_acc_vdev->drv_mode == HW_ACC_MIG_PF_CTRL) {
+               /*
+                * On hardware platforms greater than QM_HW_V3, the migration function
+                * register is placed in the BAR2 configuration region of the PF,
+                * and each VF device occupies 8KB of configuration space.
+                */
+               vf_qm->io_base = pf_qm->io_base + QM_MIG_REGION_OFFSET +
+                                hisi_acc_vdev->vf_id * QM_MIG_REGION_SIZE;
+       } else {
+               /*
+                * ACC VF dev BAR2 region consists of both functional register space
+                * and migration control register space. For migration to work, we
+                * need access to both. Hence, we map the entire BAR2 region here.
+                * But unnecessarily exposing the migration BAR region to the Guest
+                * has the potential to prevent/corrupt the Guest migration. Hence,
+                * we restrict access to the migration control space from
+                * Guest(Please see mmap/ioctl/read/write override functions).
+                *
+                * Please note that it is OK to expose the entire VF BAR if migration
+                * is not supported or required as this cannot affect the ACC PF
+                * configurations.
+                *
+                * Also the HiSilicon ACC VF devices supported by this driver on
+                * HiSilicon hardware platforms are integrated end point devices
+                * and the platform lacks the capability to perform any PCIe P2P
+                * between these devices.
+                */
 
+               vf_qm->io_base =
+                       ioremap(pci_resource_start(vf_dev, VFIO_PCI_BAR2_REGION_INDEX),
+                               pci_resource_len(vf_dev, VFIO_PCI_BAR2_REGION_INDEX));
+               if (!vf_qm->io_base)
+                       return -EIO;
+       }
        vf_qm->fun_type = QM_HW_VF;
+       vf_qm->ver = pf_qm->ver;
        vf_qm->pdev = vf_dev;
        mutex_init(&vf_qm->mailbox_lock);
 
@@ -1250,6 +1287,28 @@ static struct hisi_qm *hisi_acc_get_pf_qm(struct pci_dev *pdev)
        return !IS_ERR(pf_qm) ? pf_qm : NULL;
 }
 
+static size_t hisi_acc_get_resource_len(struct vfio_pci_core_device *vdev,
+                                       unsigned int index)
+{
+       struct hisi_acc_vf_core_device *hisi_acc_vdev =
+                       hisi_acc_drvdata(vdev->pdev);
+
+       /*
+        * On the old HW_ACC_MIG_VF_CTRL mode device, the ACC VF device
+        * BAR2 region encompasses both functional register space
+        * and migration control register space.
+        * only the functional region should be report to Guest.
+        */
+       if (hisi_acc_vdev->drv_mode == HW_ACC_MIG_VF_CTRL)
+               return (pci_resource_len(vdev->pdev, index) >> 1);
+       /*
+        * On the new HW device, the migration control register
+        * has been moved to the PF device BAR2 region.
+        * The VF device BAR2 is entirely functional register space.
+        */
+       return pci_resource_len(vdev->pdev, index);
+}
+
 static int hisi_acc_pci_rw_access_check(struct vfio_device *core_vdev,
                                        size_t count, loff_t *ppos,
                                        size_t *new_count)
@@ -1260,8 +1319,9 @@ static int hisi_acc_pci_rw_access_check(struct vfio_device *core_vdev,
 
        if (index == VFIO_PCI_BAR2_REGION_INDEX) {
                loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
-               resource_size_t end = pci_resource_len(vdev->pdev, index) / 2;
+               resource_size_t end;
 
+               end = hisi_acc_get_resource_len(vdev, index);
                /* Check if access is for migration control region */
                if (pos >= end)
                        return -EINVAL;
@@ -1282,8 +1342,9 @@ static int hisi_acc_vfio_pci_mmap(struct vfio_device *core_vdev,
        index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
        if (index == VFIO_PCI_BAR2_REGION_INDEX) {
                u64 req_len, pgoff, req_start;
-               resource_size_t end = pci_resource_len(vdev->pdev, index) / 2;
+               resource_size_t end;
 
+               end = hisi_acc_get_resource_len(vdev, index);
                req_len = vma->vm_end - vma->vm_start;
                pgoff = vma->vm_pgoff &
                        ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
@@ -1330,7 +1391,6 @@ static long hisi_acc_vfio_pci_ioctl(struct vfio_device *core_vdev, unsigned int
        if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
                struct vfio_pci_core_device *vdev =
                        container_of(core_vdev, struct vfio_pci_core_device, vdev);
-               struct pci_dev *pdev = vdev->pdev;
                struct vfio_region_info info;
                unsigned long minsz;
 
@@ -1345,12 +1405,7 @@ static long hisi_acc_vfio_pci_ioctl(struct vfio_device *core_vdev, unsigned int
                if (info.index == VFIO_PCI_BAR2_REGION_INDEX) {
                        info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
 
-                       /*
-                        * ACC VF dev BAR2 region consists of both functional
-                        * register space and migration control register space.
-                        * Report only the functional region to Guest.
-                        */
-                       info.size = pci_resource_len(pdev, info.index) / 2;
+                       info.size = hisi_acc_get_resource_len(vdev, info.index);
 
                        info.flags = VFIO_REGION_INFO_FLAG_READ |
                                        VFIO_REGION_INFO_FLAG_WRITE |
@@ -1521,7 +1576,8 @@ static void hisi_acc_vfio_pci_close_device(struct vfio_device *core_vdev)
        hisi_acc_vf_disable_fds(hisi_acc_vdev);
        mutex_lock(&hisi_acc_vdev->open_mutex);
        hisi_acc_vdev->dev_opened = false;
-       iounmap(vf_qm->io_base);
+       if (hisi_acc_vdev->drv_mode == HW_ACC_MIG_VF_CTRL)
+               iounmap(vf_qm->io_base);
        mutex_unlock(&hisi_acc_vdev->open_mutex);
        vfio_pci_core_close_device(core_vdev);
 }

diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
index 91002ceeebc18a35a80b5ddb314c0ec8a657b624..cd55eba64dfb238f209e5e58ae0990bcb7d6444f 100644 (file)
--- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
+++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h
@@ -50,8 +50,10 @@
 #define QM_QUE_ISO_CFG_V       0x0030
 #define QM_PAGE_SIZE           0x0034
 
-#define QM_EQC_DW0             0X8000
-#define QM_AEQC_DW0            0X8020
+#define QM_EQC_VF_DW0          0X8000
+#define QM_AEQC_VF_DW0         0X8020
+#define QM_EQC_PF_DW0          0x1c00
+#define QM_AEQC_PF_DW0         0x1c20
 
 #define ACC_DRV_MAJOR_VER 1
 #define ACC_DRV_MINOR_VER 0
@@ -59,6 +61,22 @@
 #define ACC_DEV_MAGIC_V1       0XCDCDCDCDFEEDAACC
 #define ACC_DEV_MAGIC_V2       0xAACCFEEDDECADEDE
 
+#define QM_MIG_REGION_OFFSET           0x180000
+#define QM_MIG_REGION_SIZE             0x2000
+
+/**
+ * On HW_ACC_MIG_VF_CTRL mode, the configuration domain supporting live
+ * migration functionality is located in the latter 32KB of the VF's BAR2.
+ * The Guest is only provided with the first 32KB of the VF's BAR2.
+ * On HW_ACC_MIG_PF_CTRL mode, the configuration domain supporting live
+ * migration functionality is located in the PF's BAR2, and the entire 64KB
+ * of the VF's BAR2 is allocated to the Guest.
+ */
+enum hw_drv_mode {
+       HW_ACC_MIG_VF_CTRL = 0,
+       HW_ACC_MIG_PF_CTRL,
+};
+
 struct acc_vf_data {
 #define QM_MATCH_SIZE offsetofend(struct acc_vf_data, qm_rsv_state)
        /* QM match information */
@@ -125,6 +143,7 @@ struct hisi_acc_vf_core_device {
        struct pci_dev *vf_dev;
        struct hisi_qm *pf_qm;
        struct hisi_qm vf_qm;
+       enum hw_drv_mode drv_mode;
        /*
         * vf_qm_state represents the QM_VF_STATE register value.
         * It is set by Guest driver for the ACC VF dev indicating