src/patches/suse-2.6.27.31/patches.arch/x2APIC_PATCH_08_of_41_fe962e90cb17a8426e144dee970e77ed789d98ee

   1 From: Suresh Siddha <suresh.b.siddha@intel.com>
   2 Subject: x64, x2apic/intr-remap: Queued invalidation infrastructure (part of VT-d)
   3 References: fate #303948 and fate #303984
   4 Patch-Mainline: queued for .28
   5 Commit-ID: fe962e90cb17a8426e144dee970e77ed789d98ee
   6
   7 Signed-off-by: Thomas Renninger <trenn@suse.de>
   8
   9 Queued invalidation (part of Intel Virtualization Technology for
  10 Directed I/O architecture) infrastructure.
  11
  12 This will be used for invalidating the interrupt entry cache in the
  13 case of Interrupt-remapping and IOTLB invalidation in the case
  14 of DMA-remapping.
  15
  16 Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
  17 Cc: akpm@linux-foundation.org
  18 Cc: arjan@linux.intel.com
  19 Cc: andi@firstfloor.org
  20 Cc: ebiederm@xmission.com
  21 Cc: jbarnes@virtuousgeek.org
  22 Cc: steiner@sgi.com
  23 Signed-off-by: Ingo Molnar <mingo@elte.hu>
  24
  25 ---
  26  drivers/pci/dmar.c        |  150 ++++++++++++++++++++++++++++++++++++++++++++++
  27  drivers/pci/intel-iommu.c |    7 --
  28  drivers/pci/intel-iommu.h |   61 ++++++++++++++++++
  29  3 files changed, 211 insertions(+), 7 deletions(-)
  30
  31 --- a/drivers/pci/dmar.c
  32 +++ b/drivers/pci/dmar.c
  33 @@ -28,6 +28,7 @@
  34
  35  #include <linux/pci.h>
  36  #include <linux/dmar.h>
  37 +#include <linux/timer.h>
  38  #include "iova.h"
  39  #include "intel-iommu.h"
  40
  41 @@ -511,3 +512,152 @@ void free_iommu(struct intel_iommu *iomm
  42                 iounmap(iommu->reg);
  43         kfree(iommu);
  44  }
  45 +
  46 +/*
  47 + * Reclaim all the submitted descriptors which have completed its work.
  48 + */
  49 +static inline void reclaim_free_desc(struct q_inval *qi)
  50 +{
  51 +       while (qi->desc_status[qi->free_tail] == QI_DONE) {
  52 +               qi->desc_status[qi->free_tail] = QI_FREE;
  53 +               qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
  54 +               qi->free_cnt++;
  55 +       }
  56 +}
  57 +
  58 +/*
  59 + * Submit the queued invalidation descriptor to the remapping
  60 + * hardware unit and wait for its completion.
  61 + */
  62 +void qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
  63 +{
  64 +       struct q_inval *qi = iommu->qi;
  65 +       struct qi_desc *hw, wait_desc;
  66 +       int wait_index, index;
  67 +       unsigned long flags;
  68 +
  69 +       if (!qi)
  70 +               return;
  71 +
  72 +       hw = qi->desc;
  73 +
  74 +       spin_lock(&qi->q_lock);
  75 +       while (qi->free_cnt < 3) {
  76 +               spin_unlock(&qi->q_lock);
  77 +               cpu_relax();
  78 +               spin_lock(&qi->q_lock);
  79 +       }
  80 +
  81 +       index = qi->free_head;
  82 +       wait_index = (index + 1) % QI_LENGTH;
  83 +
  84 +       qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
  85 +
  86 +       hw[index] = *desc;
  87 +
  88 +       wait_desc.low = QI_IWD_STATUS_DATA(2) | QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
  89 +       wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
  90 +
  91 +       hw[wait_index] = wait_desc;
  92 +
  93 +       __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
  94 +       __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
  95 +
  96 +       qi->free_head = (qi->free_head + 2) % QI_LENGTH;
  97 +       qi->free_cnt -= 2;
  98 +
  99 +       spin_lock_irqsave(&iommu->register_lock, flags);
 100 +       /*
 101 +        * update the HW tail register indicating the presence of
 102 +        * new descriptors.
 103 +        */
 104 +       writel(qi->free_head << 4, iommu->reg + DMAR_IQT_REG);
 105 +       spin_unlock_irqrestore(&iommu->register_lock, flags);
 106 +
 107 +       while (qi->desc_status[wait_index] != QI_DONE) {
 108 +               spin_unlock(&qi->q_lock);
 109 +               cpu_relax();
 110 +               spin_lock(&qi->q_lock);
 111 +       }
 112 +
 113 +       qi->desc_status[index] = QI_DONE;
 114 +
 115 +       reclaim_free_desc(qi);
 116 +       spin_unlock(&qi->q_lock);
 117 +}
 118 +
 119 +/*
 120 + * Flush the global interrupt entry cache.
 121 + */
 122 +void qi_global_iec(struct intel_iommu *iommu)
 123 +{
 124 +       struct qi_desc desc;
 125 +
 126 +       desc.low = QI_IEC_TYPE;
 127 +       desc.high = 0;
 128 +
 129 +       qi_submit_sync(&desc, iommu);
 130 +}
 131 +
 132 +/*
 133 + * Enable Queued Invalidation interface. This is a must to support
 134 + * interrupt-remapping. Also used by DMA-remapping, which replaces
 135 + * register based IOTLB invalidation.
 136 + */
 137 +int dmar_enable_qi(struct intel_iommu *iommu)
 138 +{
 139 +       u32 cmd, sts;
 140 +       unsigned long flags;
 141 +       struct q_inval *qi;
 142 +
 143 +       if (!ecap_qis(iommu->ecap))
 144 +               return -ENOENT;
 145 +
 146 +       /*
 147 +        * queued invalidation is already setup and enabled.
 148 +        */
 149 +       if (iommu->qi)
 150 +               return 0;
 151 +
 152 +       iommu->qi = kmalloc(sizeof(*qi), GFP_KERNEL);
 153 +       if (!iommu->qi)
 154 +               return -ENOMEM;
 155 +
 156 +       qi = iommu->qi;
 157 +
 158 +       qi->desc = (void *)(get_zeroed_page(GFP_KERNEL));
 159 +       if (!qi->desc) {
 160 +               kfree(qi);
 161 +               iommu->qi = 0;
 162 +               return -ENOMEM;
 163 +       }
 164 +
 165 +       qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_KERNEL);
 166 +       if (!qi->desc_status) {
 167 +               free_page((unsigned long) qi->desc);
 168 +               kfree(qi);
 169 +               iommu->qi = 0;
 170 +               return -ENOMEM;
 171 +       }
 172 +
 173 +       qi->free_head = qi->free_tail = 0;
 174 +       qi->free_cnt = QI_LENGTH;
 175 +
 176 +       spin_lock_init(&qi->q_lock);
 177 +
 178 +       spin_lock_irqsave(&iommu->register_lock, flags);
 179 +       /* write zero to the tail reg */
 180 +       writel(0, iommu->reg + DMAR_IQT_REG);
 181 +
 182 +       dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
 183 +
 184 +       cmd = iommu->gcmd | DMA_GCMD_QIE;
 185 +       iommu->gcmd |= DMA_GCMD_QIE;
 186 +       writel(cmd, iommu->reg + DMAR_GCMD_REG);
 187 +
 188 +       /* Make sure hardware complete it */
 189 +       IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
 190 +       spin_unlock_irqrestore(&iommu->register_lock, flags);
 191 +
 192 +       return 0;
 193 +}
 194 --- a/drivers/pci/intel-iommu.c
 195 +++ b/drivers/pci/intel-iommu.c
 196 @@ -183,13 +183,6 @@ void free_iova_mem(struct iova *iova)
 197         kmem_cache_free(iommu_iova_cache, iova);
 198  }
 199
 200 -static inline void __iommu_flush_cache(
 201 -       struct intel_iommu *iommu, void *addr, int size)
 202 -{
 203 -       if (!ecap_coherent(iommu->ecap))
 204 -               clflush_cache_range(addr, size);
 205 -}
 206 -
 207  /* Gets context entry for a given bus and devfn */
 208  static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 209                 u8 bus, u8 devfn)
 210 --- a/drivers/pci/intel-iommu.h
 211 +++ b/drivers/pci/intel-iommu.h
 212 @@ -27,6 +27,7 @@
 213  #include <linux/sysdev.h>
 214  #include "iova.h"
 215  #include <linux/io.h>
 216 +#include <asm/cacheflush.h>
 217  #include "dma_remapping.h"
 218
 219  /*
 220 @@ -51,6 +52,10 @@
 221  #define        DMAR_PLMLIMIT_REG 0x6c  /* PMRR low limit */
 222  #define        DMAR_PHMBASE_REG 0x70   /* pmrr high base addr */
 223  #define        DMAR_PHMLIMIT_REG 0x78  /* pmrr high limit */
 224 +#define DMAR_IQH_REG   0x80    /* Invalidation queue head register */
 225 +#define DMAR_IQT_REG   0x88    /* Invalidation queue tail register */
 226 +#define DMAR_IQA_REG   0x90    /* Invalidation queue addr register */
 227 +#define DMAR_ICS_REG   0x98    /* Invalidation complete status register */
 228
 229  #define OFFSET_STRIDE          (9)
 230  /*
 231 @@ -114,6 +119,7 @@ static inline void dmar_writeq(void __io
 232  #define ecap_max_iotlb_offset(e) \
 233         (ecap_iotlb_offset(e) + ecap_niotlb_iunits(e) * 16)
 234  #define ecap_coherent(e)       ((e) & 0x1)
 235 +#define ecap_qis(e)            ((e) & 0x2)
 236  #define ecap_eim_support(e)    ((e >> 4) & 0x1)
 237  #define ecap_ir_support(e)     ((e >> 3) & 0x1)
 238
 239 @@ -131,6 +137,17 @@ static inline void dmar_writeq(void __io
 240  #define DMA_TLB_IH_NONLEAF (((u64)1) << 6)
 241  #define DMA_TLB_MAX_SIZE (0x3f)
 242
 243 +/* INVALID_DESC */
 244 +#define DMA_ID_TLB_GLOBAL_FLUSH        (((u64)1) << 3)
 245 +#define DMA_ID_TLB_DSI_FLUSH   (((u64)2) << 3)
 246 +#define DMA_ID_TLB_PSI_FLUSH   (((u64)3) << 3)
 247 +#define DMA_ID_TLB_READ_DRAIN  (((u64)1) << 7)
 248 +#define DMA_ID_TLB_WRITE_DRAIN (((u64)1) << 6)
 249 +#define DMA_ID_TLB_DID(id)     (((u64)((id & 0xffff) << 16)))
 250 +#define DMA_ID_TLB_IH_NONLEAF  (((u64)1) << 6)
 251 +#define DMA_ID_TLB_ADDR(addr)  (addr)
 252 +#define DMA_ID_TLB_ADDR_MASK(mask)     (mask)
 253 +
 254  /* PMEN_REG */
 255  #define DMA_PMEN_EPM (((u32)1)<<31)
 256  #define DMA_PMEN_PRS (((u32)1)<<0)
 257 @@ -140,6 +157,7 @@ static inline void dmar_writeq(void __io
 258  #define DMA_GCMD_SRTP (((u32)1) << 30)
 259  #define DMA_GCMD_SFL (((u32)1) << 29)
 260  #define DMA_GCMD_EAFL (((u32)1) << 28)
 261 +#define DMA_GCMD_QIE (((u32)1) << 26)
 262  #define DMA_GCMD_WBF (((u32)1) << 27)
 263
 264  /* GSTS_REG */
 265 @@ -147,6 +165,7 @@ static inline void dmar_writeq(void __io
 266  #define DMA_GSTS_RTPS (((u32)1) << 30)
 267  #define DMA_GSTS_FLS (((u32)1) << 29)
 268  #define DMA_GSTS_AFLS (((u32)1) << 28)
 269 +#define DMA_GSTS_QIES (((u32)1) << 26)
 270  #define DMA_GSTS_WBFS (((u32)1) << 27)
 271
 272  /* CCMD_REG */
 273 @@ -192,6 +211,40 @@ static inline void dmar_writeq(void __io
 274         }\
 275  }
 276
 277 +#define QI_LENGTH      256     /* queue length */
 278 +
 279 +enum {
 280 +       QI_FREE,
 281 +       QI_IN_USE,
 282 +       QI_DONE
 283 +};
 284 +
 285 +#define QI_CC_TYPE             0x1
 286 +#define QI_IOTLB_TYPE          0x2
 287 +#define QI_DIOTLB_TYPE         0x3
 288 +#define QI_IEC_TYPE            0x4
 289 +#define QI_IWD_TYPE            0x5
 290 +
 291 +#define QI_IEC_SELECTIVE       (((u64)1) << 4)
 292 +#define QI_IEC_IIDEX(idx)      (((u64)(idx & 0xffff) << 32))
 293 +#define QI_IEC_IM(m)           (((u64)(m & 0x1f) << 27))
 294 +
 295 +#define QI_IWD_STATUS_DATA(d)  (((u64)d) << 32)
 296 +#define QI_IWD_STATUS_WRITE    (((u64)1) << 5)
 297 +
 298 +struct qi_desc {
 299 +       u64 low, high;
 300 +};
 301 +
 302 +struct q_inval {
 303 +       spinlock_t      q_lock;
 304 +       struct qi_desc  *desc;          /* invalidation queue */
 305 +       int             *desc_status;   /* desc status */
 306 +       int             free_head;      /* first free entry */
 307 +       int             free_tail;      /* last free entry */
 308 +       int             free_cnt;
 309 +};
 310 +
 311  struct intel_iommu {
 312         void __iomem    *reg; /* Pointer to hardware regs, virtual addr */
 313         u64             cap;
 314 @@ -212,8 +265,16 @@ struct intel_iommu {
 315         struct msi_msg saved_msg;
 316         struct sys_device sysdev;
 317  #endif
 318 +       struct q_inval  *qi;            /* Queued invalidation info */
 319  };
 320
 321 +static inline void __iommu_flush_cache(
 322 +       struct intel_iommu *iommu, void *addr, int size)
 323 +{
 324 +       if (!ecap_coherent(iommu->ecap))
 325 +               clflush_cache_range(addr, size);
 326 +}
 327 +
 328  extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev);
 329
 330  extern int alloc_iommu(struct dmar_drhd_unit *drhd);