src/patches/60064_xen-x86-machphys-prediction.patch1

   1 From: jbeulich@novell.com
   2 Subject: properly predict phys<->mach translations
   3 Patch-mainline: obsolete
   4
   5 Index: head-2008-11-04/include/asm-x86/mach-xen/asm/maddr_32.h
   6 ===================================================================
   7 --- head-2008-11-04.orig/include/asm-x86/mach-xen/asm/maddr_32.h        2008-11-04 11:54:55.000000000 +0100
   8 +++ head-2008-11-04/include/asm-x86/mach-xen/asm/maddr_32.h     2008-10-17 12:45:23.000000000 +0200
   9 @@ -30,17 +30,19 @@ extern unsigned int   machine_to_phys_or
  10
  11  static inline unsigned long pfn_to_mfn(unsigned long pfn)
  12  {
  13 -       if (xen_feature(XENFEAT_auto_translated_physmap))
  14 +       if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
  15                 return pfn;
  16 -       BUG_ON(max_mapnr && pfn >= max_mapnr);
  17 +       if (likely(max_mapnr))
  18 +               BUG_ON(pfn >= max_mapnr);
  19         return phys_to_machine_mapping[pfn] & ~FOREIGN_FRAME_BIT;
  20  }
  21
  22  static inline int phys_to_machine_mapping_valid(unsigned long pfn)
  23  {
  24 -       if (xen_feature(XENFEAT_auto_translated_physmap))
  25 +       if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
  26                 return 1;
  27 -       BUG_ON(max_mapnr && pfn >= max_mapnr);
  28 +       if (likely(max_mapnr))
  29 +               BUG_ON(pfn >= max_mapnr);
  30         return (phys_to_machine_mapping[pfn] != INVALID_P2M_ENTRY);
  31  }
  32
  33 @@ -48,7 +50,7 @@ static inline unsigned long mfn_to_pfn(u
  34  {
  35         unsigned long pfn;
  36
  37 -       if (xen_feature(XENFEAT_auto_translated_physmap))
  38 +       if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
  39                 return mfn;
  40
  41         if (unlikely((mfn >> machine_to_phys_order) != 0))
  42 @@ -95,17 +97,18 @@ static inline unsigned long mfn_to_pfn(u
  43  static inline unsigned long mfn_to_local_pfn(unsigned long mfn)
  44  {
  45         unsigned long pfn = mfn_to_pfn(mfn);
  46 -       if ((pfn < max_mapnr)
  47 -           && !xen_feature(XENFEAT_auto_translated_physmap)
  48 -           && (phys_to_machine_mapping[pfn] != mfn))
  49 +       if (likely(pfn < max_mapnr)
  50 +           && likely(!xen_feature(XENFEAT_auto_translated_physmap))
  51 +           && unlikely(phys_to_machine_mapping[pfn] != mfn))
  52                 return max_mapnr; /* force !pfn_valid() */
  53         return pfn;
  54  }
  55
  56  static inline void set_phys_to_machine(unsigned long pfn, unsigned long mfn)
  57  {
  58 -       BUG_ON(max_mapnr && pfn >= max_mapnr);
  59 -       if (xen_feature(XENFEAT_auto_translated_physmap)) {
  60 +       if (likely(max_mapnr))
  61 +               BUG_ON(pfn >= max_mapnr);
  62 +       if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
  63                 BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
  64                 return;
  65         }
  66 Index: head-2008-11-04/include/asm-x86/mach-xen/asm/maddr_64.h
  67 ===================================================================
  68 --- head-2008-11-04.orig/include/asm-x86/mach-xen/asm/maddr_64.h        2008-11-04 11:54:55.000000000 +0100
  69 +++ head-2008-11-04/include/asm-x86/mach-xen/asm/maddr_64.h     2008-10-17 12:46:35.000000000 +0200
  70 @@ -25,17 +25,19 @@ extern unsigned int   machine_to_phys_or
  71
  72  static inline unsigned long pfn_to_mfn(unsigned long pfn)
  73  {
  74 -       if (xen_feature(XENFEAT_auto_translated_physmap))
  75 +       if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
  76                 return pfn;
  77 -       BUG_ON(max_mapnr && pfn >= max_mapnr);
  78 +       if (likely(max_mapnr))
  79 +               BUG_ON(pfn >= max_mapnr);
  80         return phys_to_machine_mapping[pfn] & ~FOREIGN_FRAME_BIT;
  81  }
  82
  83  static inline int phys_to_machine_mapping_valid(unsigned long pfn)
  84  {
  85 -       if (xen_feature(XENFEAT_auto_translated_physmap))
  86 +       if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
  87                 return 1;
  88 -       BUG_ON(max_mapnr && pfn >= max_mapnr);
  89 +       if (likely(max_mapnr))
  90 +               BUG_ON(pfn >= max_mapnr);
  91         return (phys_to_machine_mapping[pfn] != INVALID_P2M_ENTRY);
  92  }
  93
  94 @@ -43,7 +45,7 @@ static inline unsigned long mfn_to_pfn(u
  95  {
  96         unsigned long pfn;
  97
  98 -       if (xen_feature(XENFEAT_auto_translated_physmap))
  99 +       if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
 100                 return mfn;
 101
 102         if (unlikely((mfn >> machine_to_phys_order) != 0))
 103 @@ -90,17 +92,18 @@ static inline unsigned long mfn_to_pfn(u
 104  static inline unsigned long mfn_to_local_pfn(unsigned long mfn)
 105  {
 106         unsigned long pfn = mfn_to_pfn(mfn);
 107 -       if ((pfn < max_mapnr)
 108 -           && !xen_feature(XENFEAT_auto_translated_physmap)
 109 -           && (phys_to_machine_mapping[pfn] != mfn))
 110 +       if (likely(pfn < max_mapnr)
 111 +           && likely(!xen_feature(XENFEAT_auto_translated_physmap))
 112 +           && unlikely(phys_to_machine_mapping[pfn] != mfn))
 113                 return max_mapnr; /* force !pfn_valid() */
 114         return pfn;
 115  }
 116
 117  static inline void set_phys_to_machine(unsigned long pfn, unsigned long mfn)
 118  {
 119 -       BUG_ON(max_mapnr && pfn >= max_mapnr);
 120 -       if (xen_feature(XENFEAT_auto_translated_physmap)) {
 121 +       if (likely(max_mapnr))
 122 +               BUG_ON(pfn >= max_mapnr);
 123 +       if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
 124                 BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
 125                 return;
 126         }
 127 Index: head-2008-11-04/include/asm-x86/mach-xen/asm/page.h
 128 ===================================================================
 129 --- head-2008-11-04.orig/include/asm-x86/mach-xen/asm/page.h    2008-11-04 11:55:22.000000000 +0100
 130 +++ head-2008-11-04/include/asm-x86/mach-xen/asm/page.h 2008-10-17 12:53:40.000000000 +0200
 131 @@ -102,7 +102,7 @@ static inline void copy_user_page(void *
 132  #define __pgd_ma(x) ((pgd_t) { (x) } )
 133  static inline pgd_t xen_make_pgd(pgdval_t val)
 134  {
 135 -       if (val & _PAGE_PRESENT)
 136 +       if (likely(val & _PAGE_PRESENT))
 137                 val = pte_phys_to_machine(val);
 138         return (pgd_t) { val };
 139  }
 140 @@ -112,10 +112,10 @@ static inline pgdval_t xen_pgd_val(pgd_t
 141  {
 142         pgdval_t ret = __pgd_val(pgd);
 143  #if PAGETABLE_LEVELS == 2 && CONFIG_XEN_COMPAT <= 0x030002
 144 -       if (ret)
 145 +       if (likely(ret))
 146                 ret = machine_to_phys(ret) | _PAGE_PRESENT;
 147  #else
 148 -       if (ret & _PAGE_PRESENT)
 149 +       if (likely(ret & _PAGE_PRESENT))
 150                 ret = pte_machine_to_phys(ret);
 151  #endif
 152         return ret;
 153 @@ -128,7 +128,7 @@ typedef struct { pudval_t pud; } pud_t;
 154  #define __pud_ma(x) ((pud_t) { (x) } )
 155  static inline pud_t xen_make_pud(pudval_t val)
 156  {
 157 -       if (val & _PAGE_PRESENT)
 158 +       if (likely(val & _PAGE_PRESENT))
 159                 val = pte_phys_to_machine(val);
 160         return (pud_t) { val };
 161  }
 162 @@ -137,7 +137,7 @@ static inline pud_t xen_make_pud(pudval_
 163  static inline pudval_t xen_pud_val(pud_t pud)
 164  {
 165         pudval_t ret = __pud_val(pud);
 166 -       if (ret & _PAGE_PRESENT)
 167 +       if (likely(ret & _PAGE_PRESENT))
 168                 ret = pte_machine_to_phys(ret);
 169         return ret;
 170  }
 171 @@ -156,7 +156,7 @@ typedef struct { pmdval_t pmd; } pmd_t;
 172  #define __pmd_ma(x)    ((pmd_t) { (x) } )
 173  static inline pmd_t xen_make_pmd(pmdval_t val)
 174  {
 175 -       if (val & _PAGE_PRESENT)
 176 +       if (likely(val & _PAGE_PRESENT))
 177                 val = pte_phys_to_machine(val);
 178         return (pmd_t) { val };
 179  }
 180 @@ -166,10 +166,10 @@ static inline pmdval_t xen_pmd_val(pmd_t
 181  {
 182         pmdval_t ret = __pmd_val(pmd);
 183  #if CONFIG_XEN_COMPAT <= 0x030002
 184 -       if (ret)
 185 +       if (likely(ret))
 186                 ret = pte_machine_to_phys(ret) | _PAGE_PRESENT;
 187  #else
 188 -       if (ret & _PAGE_PRESENT)
 189 +       if (likely(ret & _PAGE_PRESENT))
 190                 ret = pte_machine_to_phys(ret);
 191  #endif
 192         return ret;
 193 @@ -188,7 +188,7 @@ static inline pmdval_t xen_pmd_val(pmd_t
 194  #define __pte_ma(x) ((pte_t) { .pte = (x) } )
 195  static inline pte_t xen_make_pte(pteval_t val)
 196  {
 197 -       if ((val & (_PAGE_PRESENT|_PAGE_IO)) == _PAGE_PRESENT)
 198 +       if (likely((val & (_PAGE_PRESENT|_PAGE_IO)) == _PAGE_PRESENT))
 199                 val = pte_phys_to_machine(val);
 200         return (pte_t) { .pte = val };
 201  }
 202 @@ -197,7 +197,7 @@ static inline pte_t xen_make_pte(pteval_
 203  static inline pteval_t xen_pte_val(pte_t pte)
 204  {
 205         pteval_t ret = __pte_val(pte);
 206 -       if ((pte.pte_low & (_PAGE_PRESENT|_PAGE_IO)) == _PAGE_PRESENT)
 207 +       if (likely((pte.pte_low & (_PAGE_PRESENT|_PAGE_IO)) == _PAGE_PRESENT))
 208                 ret = pte_machine_to_phys(ret);
 209         return ret;
 210  }