src/patches/suse-2.6.27.25/patches.arch/x2APIC_PATCH_09_of_41_2ae21010694e56461a63bfc80e960090ce0a5ed9

   1 From: Suresh Siddha <suresh.b.siddha@intel.com>
   2 Subject: x64, x2apic/intr-remap: Interrupt remapping infrastructure
   3 References: fate #303948 and fate #303984
   4 Patch-Mainline: queued for .28
   5 Commit-ID: 2ae21010694e56461a63bfc80e960090ce0a5ed9
   6
   7 Signed-off-by: Thomas Renninger <trenn@suse.de>
   8
   9 Interrupt remapping (part of Intel Virtualization Tech for directed I/O)
  10 infrastructure.
  11
  12 Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
  13 Cc: akpm@linux-foundation.org
  14 Cc: arjan@linux.intel.com
  15 Cc: andi@firstfloor.org
  16 Cc: ebiederm@xmission.com
  17 Cc: jbarnes@virtuousgeek.org
  18 Cc: steiner@sgi.com
  19 Signed-off-by: Ingo Molnar <mingo@elte.hu>
  20
  21 ---
  22  drivers/pci/dma_remapping.h  |    2
  23  drivers/pci/dmar.c           |   16 +++++
  24  drivers/pci/intel-iommu.c    |   25 ++-----
  25  drivers/pci/intel-iommu.h    |   24 ++++++-
  26  drivers/pci/intr_remapping.c |  137 +++++++++++++++++++++++++++++++++++++++++++
  27  drivers/pci/intr_remapping.h |    2
  28  include/linux/dmar.h         |  120 ++++++++++++++++++++++++++-----------
  29  7 files changed, 272 insertions(+), 54 deletions(-)
  30
  31 --- a/drivers/pci/dmar.c
  32 +++ b/drivers/pci/dmar.c
  33 @@ -451,6 +451,22 @@ int __init early_dmar_detect(void)
  34         return (ACPI_SUCCESS(status) ? 1 : 0);
  35  }
  36
  37 +void __init detect_intel_iommu(void)
  38 +{
  39 +       int ret;
  40 +
  41 +       ret = early_dmar_detect();
  42 +
  43 +#ifdef CONFIG_DMAR
  44 +       {
  45 +               if (ret && !no_iommu && !iommu_detected && !swiotlb &&
  46 +                   !dmar_disabled)
  47 +                       iommu_detected = 1;
  48 +       }
  49 +#endif
  50 +}
  51 +
  52 +
  53  int alloc_iommu(struct dmar_drhd_unit *drhd)
  54  {
  55         struct intel_iommu *iommu;
  56 --- a/drivers/pci/dma_remapping.h
  57 +++ b/drivers/pci/dma_remapping.h
  58 @@ -145,6 +145,8 @@ struct device_domain_info {
  59  extern int init_dmars(void);
  60  extern void free_dmar_iommu(struct intel_iommu *iommu);
  61
  62 +extern int dmar_disabled;
  63 +
  64  #ifndef CONFIG_DMAR_GFX_WA
  65  static inline void iommu_prepare_gfx_mapping(void)
  66  {
  67 --- a/drivers/pci/intel-iommu.c
  68 +++ b/drivers/pci/intel-iommu.c
  69 @@ -78,7 +78,7 @@ static long list_size;
  70
  71  static void domain_remove_dev_info(struct dmar_domain *domain);
  72
  73 -static int dmar_disabled;
  74 +int dmar_disabled;
  75  static int __initdata dmar_map_gfx = 1;
  76  static int dmar_forcedac;
  77  static int intel_iommu_strict;
  78 @@ -2259,19 +2259,6 @@ static struct dmi_system_id __initdata i
  79         { }
  80  };
  81
  82 -
  83 -void __init detect_intel_iommu(void)
  84 -{
  85 -       if (swiotlb || no_iommu || iommu_detected || dmar_disabled)
  86 -               return;
  87 -       if (early_dmar_detect()) {
  88 -               dmi_check_system(intel_iommu_dmi_table);
  89 -               if (dmar_disabled)
  90 -                       return;
  91 -               iommu_detected = 1;
  92 -       }
  93 -}
  94 -
  95  static void __init init_no_remapping_devices(void)
  96  {
  97         struct dmar_drhd_unit *drhd;
  98 @@ -2318,15 +2305,19 @@ int __init intel_iommu_init(void)
  99  {
 100         int ret = 0;
 101
 102 -       if (no_iommu || swiotlb || dmar_disabled)
 103 -               return -ENODEV;
 104 -
 105         if (dmar_table_init())
 106                 return  -ENODEV;
 107
 108         if (dmar_dev_scope_init())
 109                 return  -ENODEV;
 110
 111 +       /*
 112 +        * Check the need for DMA-remapping initialization now.
 113 +        * Above initialization will also be used by Interrupt-remapping.
 114 +        */
 115 +       if (no_iommu || swiotlb || dmar_disabled)
 116 +               return -ENODEV;
 117 +
 118         iommu_init_mempool();
 119         dmar_init_reserved_ranges();
 120
 121 --- a/drivers/pci/intel-iommu.h
 122 +++ b/drivers/pci/intel-iommu.h
 123 @@ -56,6 +56,7 @@
 124  #define DMAR_IQT_REG   0x88    /* Invalidation queue tail register */
 125  #define DMAR_IQA_REG   0x90    /* Invalidation queue addr register */
 126  #define DMAR_ICS_REG   0x98    /* Invalidation complete status register */
 127 +#define DMAR_IRTA_REG  0xb8    /* Interrupt remapping table addr register */
 128
 129  #define OFFSET_STRIDE          (9)
 130  /*
 131 @@ -157,16 +158,20 @@ static inline void dmar_writeq(void __io
 132  #define DMA_GCMD_SRTP (((u32)1) << 30)
 133  #define DMA_GCMD_SFL (((u32)1) << 29)
 134  #define DMA_GCMD_EAFL (((u32)1) << 28)
 135 -#define DMA_GCMD_QIE (((u32)1) << 26)
 136  #define DMA_GCMD_WBF (((u32)1) << 27)
 137 +#define DMA_GCMD_QIE (((u32)1) << 26)
 138 +#define DMA_GCMD_SIRTP (((u32)1) << 24)
 139 +#define DMA_GCMD_IRE (((u32) 1) << 25)
 140
 141  /* GSTS_REG */
 142  #define DMA_GSTS_TES (((u32)1) << 31)
 143  #define DMA_GSTS_RTPS (((u32)1) << 30)
 144  #define DMA_GSTS_FLS (((u32)1) << 29)
 145  #define DMA_GSTS_AFLS (((u32)1) << 28)
 146 -#define DMA_GSTS_QIES (((u32)1) << 26)
 147  #define DMA_GSTS_WBFS (((u32)1) << 27)
 148 +#define DMA_GSTS_QIES (((u32)1) << 26)
 149 +#define DMA_GSTS_IRTPS (((u32)1) << 24)
 150 +#define DMA_GSTS_IRES (((u32)1) << 25)
 151
 152  /* CCMD_REG */
 153  #define DMA_CCMD_ICC (((u64)1) << 63)
 154 @@ -245,6 +250,16 @@ struct q_inval {
 155         int             free_cnt;
 156  };
 157
 158 +#ifdef CONFIG_INTR_REMAP
 159 +/* 1MB - maximum possible interrupt remapping table size */
 160 +#define INTR_REMAP_PAGE_ORDER  8
 161 +#define INTR_REMAP_TABLE_REG_SIZE      0xf
 162 +
 163 +struct ir_table {
 164 +       struct irte *base;
 165 +};
 166 +#endif
 167 +
 168  struct intel_iommu {
 169         void __iomem    *reg; /* Pointer to hardware regs, virtual addr */
 170         u64             cap;
 171 @@ -266,6 +281,9 @@ struct intel_iommu {
 172         struct sys_device sysdev;
 173  #endif
 174         struct q_inval  *qi;            /* Queued invalidation info */
 175 +#ifdef CONFIG_INTR_REMAP
 176 +       struct ir_table *ir_table;      /* Interrupt remapping info */
 177 +#endif
 178  };
 179
 180  static inline void __iommu_flush_cache(
 181 @@ -279,5 +297,7 @@ extern struct dmar_drhd_unit * dmar_find
 182
 183  extern int alloc_iommu(struct dmar_drhd_unit *drhd);
 184  extern void free_iommu(struct intel_iommu *iommu);
 185 +extern int dmar_enable_qi(struct intel_iommu *iommu);
 186 +extern void qi_global_iec(struct intel_iommu *iommu);
 187
 188  #endif
 189 --- a/drivers/pci/intr_remapping.c
 190 +++ b/drivers/pci/intr_remapping.c
 191 @@ -1,10 +1,147 @@
 192  #include <linux/dmar.h>
 193 +#include <linux/spinlock.h>
 194 +#include <linux/jiffies.h>
 195 +#include <linux/pci.h>
 196  #include <asm/io_apic.h>
 197  #include "intel-iommu.h"
 198  #include "intr_remapping.h"
 199
 200  static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
 201  static int ir_ioapic_num;
 202 +int intr_remapping_enabled;
 203 +
 204 +static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode)
 205 +{
 206 +       u64 addr;
 207 +       u32 cmd, sts;
 208 +       unsigned long flags;
 209 +
 210 +       addr = virt_to_phys((void *)iommu->ir_table->base);
 211 +
 212 +       spin_lock_irqsave(&iommu->register_lock, flags);
 213 +
 214 +       dmar_writeq(iommu->reg + DMAR_IRTA_REG,
 215 +                   (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE);
 216 +
 217 +       /* Set interrupt-remapping table pointer */
 218 +       cmd = iommu->gcmd | DMA_GCMD_SIRTP;
 219 +       writel(cmd, iommu->reg + DMAR_GCMD_REG);
 220 +
 221 +       IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 222 +                     readl, (sts & DMA_GSTS_IRTPS), sts);
 223 +       spin_unlock_irqrestore(&iommu->register_lock, flags);
 224 +
 225 +       /*
 226 +        * global invalidation of interrupt entry cache before enabling
 227 +        * interrupt-remapping.
 228 +        */
 229 +       qi_global_iec(iommu);
 230 +
 231 +       spin_lock_irqsave(&iommu->register_lock, flags);
 232 +
 233 +       /* Enable interrupt-remapping */
 234 +       cmd = iommu->gcmd | DMA_GCMD_IRE;
 235 +       iommu->gcmd |= DMA_GCMD_IRE;
 236 +       writel(cmd, iommu->reg + DMAR_GCMD_REG);
 237 +
 238 +       IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 239 +                     readl, (sts & DMA_GSTS_IRES), sts);
 240 +
 241 +       spin_unlock_irqrestore(&iommu->register_lock, flags);
 242 +}
 243 +
 244 +
 245 +static int setup_intr_remapping(struct intel_iommu *iommu, int mode)
 246 +{
 247 +       struct ir_table *ir_table;
 248 +       struct page *pages;
 249 +
 250 +       ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table),
 251 +                                            GFP_KERNEL);
 252 +
 253 +       if (!iommu->ir_table)
 254 +               return -ENOMEM;
 255 +
 256 +       pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, INTR_REMAP_PAGE_ORDER);
 257 +
 258 +       if (!pages) {
 259 +               printk(KERN_ERR "failed to allocate pages of order %d\n",
 260 +                      INTR_REMAP_PAGE_ORDER);
 261 +               kfree(iommu->ir_table);
 262 +               return -ENOMEM;
 263 +       }
 264 +
 265 +       ir_table->base = page_address(pages);
 266 +
 267 +       iommu_set_intr_remapping(iommu, mode);
 268 +       return 0;
 269 +}
 270 +
 271 +int __init enable_intr_remapping(int eim)
 272 +{
 273 +       struct dmar_drhd_unit *drhd;
 274 +       int setup = 0;
 275 +
 276 +       /*
 277 +        * check for the Interrupt-remapping support
 278 +        */
 279 +       for_each_drhd_unit(drhd) {
 280 +               struct intel_iommu *iommu = drhd->iommu;
 281 +
 282 +               if (!ecap_ir_support(iommu->ecap))
 283 +                       continue;
 284 +
 285 +               if (eim && !ecap_eim_support(iommu->ecap)) {
 286 +                       printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
 287 +                              " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
 288 +                       return -1;
 289 +               }
 290 +       }
 291 +
 292 +       /*
 293 +        * Enable queued invalidation for all the DRHD's.
 294 +        */
 295 +       for_each_drhd_unit(drhd) {
 296 +               int ret;
 297 +               struct intel_iommu *iommu = drhd->iommu;
 298 +               ret = dmar_enable_qi(iommu);
 299 +
 300 +               if (ret) {
 301 +                       printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
 302 +                              " invalidation, ecap %Lx, ret %d\n",
 303 +                              drhd->reg_base_addr, iommu->ecap, ret);
 304 +                       return -1;
 305 +               }
 306 +       }
 307 +
 308 +       /*
 309 +        * Setup Interrupt-remapping for all the DRHD's now.
 310 +        */
 311 +       for_each_drhd_unit(drhd) {
 312 +               struct intel_iommu *iommu = drhd->iommu;
 313 +
 314 +               if (!ecap_ir_support(iommu->ecap))
 315 +                       continue;
 316 +
 317 +               if (setup_intr_remapping(iommu, eim))
 318 +                       goto error;
 319 +
 320 +               setup = 1;
 321 +       }
 322 +
 323 +       if (!setup)
 324 +               goto error;
 325 +
 326 +       intr_remapping_enabled = 1;
 327 +
 328 +       return 0;
 329 +
 330 +error:
 331 +       /*
 332 +        * handle error condition gracefully here!
 333 +        */
 334 +       return -1;
 335 +}
 336
 337  static int ir_parse_ioapic_scope(struct acpi_dmar_header *header,
 338                                  struct intel_iommu *iommu)
 339 --- a/drivers/pci/intr_remapping.h
 340 +++ b/drivers/pci/intr_remapping.h
 341 @@ -4,3 +4,5 @@ struct ioapic_scope {
 342         struct intel_iommu *iommu;
 343         unsigned int id;
 344  };
 345 +
 346 +#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0)
 347 --- a/include/linux/dmar.h
 348 +++ b/include/linux/dmar.h
 349 @@ -25,9 +25,85 @@
 350  #include <linux/types.h>
 351  #include <linux/msi.h>
 352
 353 -#ifdef CONFIG_DMAR
 354 +#if defined(CONFIG_DMAR) || defined(CONFIG_INTR_REMAP)
 355  struct intel_iommu;
 356
 357 +struct dmar_drhd_unit {
 358 +       struct list_head list;          /* list of drhd units   */
 359 +       struct  acpi_dmar_header *hdr;  /* ACPI header          */
 360 +       u64     reg_base_addr;          /* register base address*/
 361 +       struct  pci_dev **devices;      /* target device array  */
 362 +       int     devices_cnt;            /* target device count  */
 363 +       u8      ignored:1;              /* ignore drhd          */
 364 +       u8      include_all:1;
 365 +       struct intel_iommu *iommu;
 366 +};
 367 +
 368 +extern struct list_head dmar_drhd_units;
 369 +
 370 +#define for_each_drhd_unit(drhd) \
 371 +       list_for_each_entry(drhd, &dmar_drhd_units, list)
 372 +
 373 +extern int dmar_table_init(void);
 374 +extern int early_dmar_detect(void);
 375 +extern int dmar_dev_scope_init(void);
 376 +
 377 +/* Intel IOMMU detection */
 378 +extern void detect_intel_iommu(void);
 379 +
 380 +
 381 +extern int parse_ioapics_under_ir(void);
 382 +extern int alloc_iommu(struct dmar_drhd_unit *);
 383 +#else
 384 +static inline void detect_intel_iommu(void)
 385 +{
 386 +       return;
 387 +}
 388 +
 389 +static inline int dmar_table_init(void)
 390 +{
 391 +       return -ENODEV;
 392 +}
 393 +#endif /* !CONFIG_DMAR && !CONFIG_INTR_REMAP */
 394 +
 395 +#ifdef CONFIG_INTR_REMAP
 396 +extern int intr_remapping_enabled;
 397 +extern int enable_intr_remapping(int);
 398 +
 399 +struct irte {
 400 +       union {
 401 +               struct {
 402 +                       __u64   present         : 1,
 403 +                               fpd             : 1,
 404 +                               dst_mode        : 1,
 405 +                               redir_hint      : 1,
 406 +                               trigger_mode    : 1,
 407 +                               dlvry_mode      : 3,
 408 +                               avail           : 4,
 409 +                               __reserved_1    : 4,
 410 +                               vector          : 8,
 411 +                               __reserved_2    : 8,
 412 +                               dest_id         : 32;
 413 +               };
 414 +               __u64 low;
 415 +       };
 416 +
 417 +       union {
 418 +               struct {
 419 +                       __u64   sid             : 16,
 420 +                               sq              : 2,
 421 +                               svt             : 2,
 422 +                               __reserved_3    : 44;
 423 +               };
 424 +               __u64 high;
 425 +       };
 426 +};
 427 +#else
 428 +#define enable_intr_remapping(mode)    (-1)
 429 +#define intr_remapping_enabled         (0)
 430 +#endif
 431 +
 432 +#ifdef CONFIG_DMAR
 433  extern const char *dmar_get_fault_reason(u8 fault_reason);
 434
 435  /* Can't use the common MSI interrupt functions
 436 @@ -40,29 +116,8 @@ extern void dmar_msi_write(int irq, stru
 437  extern int dmar_set_interrupt(struct intel_iommu *iommu);
 438  extern int arch_setup_dmar_msi(unsigned int irq);
 439
 440 -/* Intel IOMMU detection and initialization functions */
 441 -extern void detect_intel_iommu(void);
 442 -extern int intel_iommu_init(void);
 443 -
 444 -extern int dmar_table_init(void);
 445 -extern int early_dmar_detect(void);
 446 -extern int dmar_dev_scope_init(void);
 447 -extern int parse_ioapics_under_ir(void);
 448 -
 449 -extern struct list_head dmar_drhd_units;
 450 +extern int iommu_detected, no_iommu;
 451  extern struct list_head dmar_rmrr_units;
 452 -
 453 -struct dmar_drhd_unit {
 454 -       struct list_head list;          /* list of drhd units   */
 455 -       struct  acpi_dmar_header *hdr;  /* ACPI header          */
 456 -       u64     reg_base_addr;          /* register base address*/
 457 -       struct  pci_dev **devices;      /* target device array  */
 458 -       int     devices_cnt;            /* target device count  */
 459 -       u8      ignored:1;              /* ignore drhd          */
 460 -       u8      include_all:1;
 461 -       struct intel_iommu *iommu;
 462 -};
 463 -
 464  struct dmar_rmrr_unit {
 465         struct list_head list;          /* list of rmrr units   */
 466         struct acpi_dmar_header *hdr;   /* ACPI header          */
 467 @@ -72,24 +127,19 @@ struct dmar_rmrr_unit {
 468         int     devices_cnt;            /* target device count */
 469  };
 470
 471 -#define for_each_drhd_unit(drhd) \
 472 -       list_for_each_entry(drhd, &dmar_drhd_units, list)
 473  #define for_each_rmrr_units(rmrr) \
 474         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 475 -
 476 -extern int alloc_iommu(struct dmar_drhd_unit *);
 477 +/* Intel DMAR  initialization functions */
 478 +extern int intel_iommu_init(void);
 479 +extern int dmar_disabled;
 480  #else
 481 -static inline void detect_intel_iommu(void)
 482 -{
 483 -       return;
 484 -}
 485  static inline int intel_iommu_init(void)
 486  {
 487 +#ifdef CONFIG_INTR_REMAP
 488 +       return dmar_dev_scope_init();
 489 +#else
 490         return -ENODEV;
 491 -}
 492 -static inline int dmar_table_init(void)
 493 -{
 494 -       return -ENODEV;
 495 +#endif
 496  }
 497  #endif /* !CONFIG_DMAR */
 498  #endif /* __DMAR_H__ */