src/patches/60062_xen-x86-pmd-handling.patch1

   1 From: jbeulich@novell.com
   2 Subject: consolidate pmd/pud/pgd entry handling
   3 Patch-mainline: obsolete
   4
   5 Index: head-2008-12-01/arch/x86/mm/hypervisor.c
   6 ===================================================================
   7 --- head-2008-12-01.orig/arch/x86/mm/hypervisor.c       2008-12-01 12:10:32.000000000 +0100
   8 +++ head-2008-12-01/arch/x86/mm/hypervisor.c    2008-12-01 12:13:06.000000000 +0100
   9 @@ -356,31 +356,91 @@ void xen_l1_entry_update(pte_t *ptr, pte
  10  }
  11  EXPORT_SYMBOL_GPL(xen_l1_entry_update);
  12
  13 +static void do_lN_entry_update(mmu_update_t *mmu, unsigned int mmu_count,
  14 +                               struct page *page)
  15 +{
  16 +       if (likely(page)) {
  17 +               multicall_entry_t mcl[2];
  18 +               unsigned long pfn = page_to_pfn(page);
  19 +
  20 +               MULTI_update_va_mapping(mcl,
  21 +                                       (unsigned long)__va(pfn << PAGE_SHIFT),
  22 +                                       pfn_pte(pfn, PAGE_KERNEL_RO), 0);
  23 +               SetPagePinned(page);
  24 +               MULTI_mmu_update(mcl + 1, mmu, mmu_count, NULL, DOMID_SELF);
  25 +               if (unlikely(HYPERVISOR_multicall_check(mcl, 2, NULL)))
  26 +                       BUG();
  27 +       } else if (unlikely(HYPERVISOR_mmu_update(mmu, mmu_count,
  28 +                                                 NULL, DOMID_SELF) < 0))
  29 +               BUG();
  30 +}
  31 +
  32  void xen_l2_entry_update(pmd_t *ptr, pmd_t val)
  33  {
  34         mmu_update_t u;
  35 +       struct page *page = NULL;
  36 +
  37 +       if (likely(pmd_present(val)) && likely(!pmd_large(val))
  38 +           && likely(mem_map)
  39 +           && likely(PagePinned(virt_to_page(ptr)))) {
  40 +               page = pmd_page(val);
  41 +               if (unlikely(PagePinned(page)))
  42 +                       page = NULL;
  43 +               else if (PageHighMem(page)) {
  44 +#ifdef CONFIG_HIGHPTE
  45 +                       BUG();
  46 +#endif
  47 +                       kmap_flush_unused();
  48 +                       page = NULL;
  49 +               }
  50 +       }
  51         u.ptr = virt_to_machine(ptr);
  52         u.val = __pmd_val(val);
  53 -       BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
  54 +       do_lN_entry_update(&u, 1, page);
  55  }
  56
  57  #if defined(CONFIG_X86_PAE) || defined(CONFIG_X86_64)
  58  void xen_l3_entry_update(pud_t *ptr, pud_t val)
  59  {
  60         mmu_update_t u;
  61 +       struct page *page = NULL;
  62 +
  63 +       if (likely(pud_present(val))
  64 +#ifdef CONFIG_X86_64
  65 +           && likely(!pud_large(val))
  66 +#endif
  67 +           && likely(mem_map)
  68 +           && likely(PagePinned(virt_to_page(ptr)))) {
  69 +               page = pud_page(val);
  70 +               if (unlikely(PagePinned(page)))
  71 +                       page = NULL;
  72 +       }
  73         u.ptr = virt_to_machine(ptr);
  74         u.val = __pud_val(val);
  75 -       BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
  76 +       do_lN_entry_update(&u, 1, page);
  77  }
  78  #endif
  79
  80  #ifdef CONFIG_X86_64
  81 -void xen_l4_entry_update(pgd_t *ptr, pgd_t val)
  82 +void xen_l4_entry_update(pgd_t *ptr, int user, pgd_t val)
  83  {
  84 -       mmu_update_t u;
  85 -       u.ptr = virt_to_machine(ptr);
  86 -       u.val = __pgd_val(val);
  87 -       BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
  88 +       mmu_update_t u[2];
  89 +       struct page *page = NULL;
  90 +
  91 +       if (likely(pgd_present(val)) && likely(mem_map)
  92 +           && likely(PagePinned(virt_to_page(ptr)))) {
  93 +               page = pgd_page(val);
  94 +               if (unlikely(PagePinned(page)))
  95 +                       page = NULL;
  96 +       }
  97 +       u[0].ptr = virt_to_machine(ptr);
  98 +       u[0].val = __pgd_val(val);
  99 +       if (user) {
 100 +               u[1].ptr = virt_to_machine(__user_pgd(ptr));
 101 +               u[1].val = __pgd_val(val);
 102 +               do_lN_entry_update(u, 2, page);
 103 +       } else
 104 +               do_lN_entry_update(u, 1, page);
 105  }
 106  #endif /* CONFIG_X86_64 */
 107
 108 Index: head-2008-12-01/arch/x86/mm/init_32-xen.c
 109 ===================================================================
 110 --- head-2008-12-01.orig/arch/x86/mm/init_32-xen.c      2008-12-01 11:49:07.000000000 +0100
 111 +++ head-2008-12-01/arch/x86/mm/init_32-xen.c   2008-12-01 12:13:06.000000000 +0100
 112 @@ -681,6 +681,8 @@ static void __init zone_sizes_init(void)
 113  #endif
 114
 115         free_area_init_nodes(max_zone_pfns);
 116 +
 117 +       xen_init_pgd_pin();
 118  }
 119
 120  void __init setup_bootmem_allocator(void)
 121 @@ -1040,8 +1042,6 @@ void __init mem_init(void)
 122         cpa_init();
 123         save_pg_dir();
 124         zap_low_mappings();
 125 -
 126 -       SetPagePinned(virt_to_page(init_mm.pgd));
 127  }
 128
 129  #ifdef CONFIG_MEMORY_HOTPLUG
 130 Index: head-2008-12-01/arch/x86/mm/init_64-xen.c
 131 ===================================================================
 132 --- head-2008-12-01.orig/arch/x86/mm/init_64-xen.c      2008-12-01 11:49:07.000000000 +0100
 133 +++ head-2008-12-01/arch/x86/mm/init_64-xen.c   2008-12-01 12:13:06.000000000 +0100
 134 @@ -195,7 +195,10 @@ set_pte_vaddr_pud(pud_t *pud_page, unsig
 135         if (pud_none(*pud)) {
 136                 pmd = (pmd_t *) spp_getpage();
 137                 make_page_readonly(pmd, XENFEAT_writable_page_tables);
 138 -               pud_populate(&init_mm, pud, pmd);
 139 +               if (!after_bootmem)
 140 +                       xen_l3_entry_update(pud, __pud(__pa(pmd) | _PAGE_TABLE));
 141 +               else
 142 +                       pud_populate(&init_mm, pud, pmd);
 143                 if (pmd != pmd_offset(pud, 0)) {
 144                         printk(KERN_ERR "PAGETABLE BUG #01! %p <-> %p\n",
 145                                 pmd, pmd_offset(pud, 0));
 146 @@ -462,7 +465,6 @@ phys_pmd_init(pmd_t *pmd_page, unsigned
 147                         early_make_page_readonly(pte, XENFEAT_writable_page_tables);
 148                         *pmd = __pmd(pte_phys | _PAGE_TABLE);
 149                 } else {
 150 -                       make_page_readonly(pte, XENFEAT_writable_page_tables);
 151                         spin_lock(&init_mm.page_table_lock);
 152                         pmd_populate_kernel(&init_mm, pmd, __va(pte_phys));
 153                         spin_unlock(&init_mm.page_table_lock);
 154 @@ -529,7 +531,6 @@ phys_pud_init(pud_t *pud_page, unsigned
 155                         else
 156                                 *pud = __pud(pmd_phys | _PAGE_TABLE);
 157                 } else {
 158 -                       make_page_readonly(pmd, XENFEAT_writable_page_tables);
 159                         spin_lock(&init_mm.page_table_lock);
 160                         pud_populate(&init_mm, pud, __va(pmd_phys));
 161                         spin_unlock(&init_mm.page_table_lock);
 162 @@ -824,7 +825,6 @@ static unsigned long __init kernel_physi
 163                         early_make_page_readonly(pud, XENFEAT_writable_page_tables);
 164                         xen_l4_entry_update(pgd, __pgd(pud_phys | _PAGE_TABLE));
 165                 } else {
 166 -                       make_page_readonly(pud, XENFEAT_writable_page_tables);
 167                         spin_lock(&init_mm.page_table_lock);
 168                         pgd_populate(&init_mm, pgd, __va(pud_phys));
 169                         spin_unlock(&init_mm.page_table_lock);
 170 @@ -1013,7 +1013,7 @@ void __init paging_init(void)
 171         sparse_init();
 172         free_area_init_nodes(max_zone_pfns);
 173
 174 -       SetPagePinned(virt_to_page(init_mm.pgd));
 175 +       xen_init_pgd_pin();
 176  }
 177  #endif
 178
 179 Index: head-2008-12-01/arch/x86/mm/ioremap-xen.c
 180 ===================================================================
 181 --- head-2008-12-01.orig/arch/x86/mm/ioremap-xen.c      2008-12-01 12:07:15.000000000 +0100
 182 +++ head-2008-12-01/arch/x86/mm/ioremap-xen.c   2008-12-01 12:13:06.000000000 +0100
 183 @@ -678,7 +678,7 @@ void __init early_ioremap_clear(void)
 184                 printk(KERN_INFO "early_ioremap_clear()\n");
 185
 186         pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
 187 -       pmd_clear(pmd);
 188 +       xen_l2_entry_update(pmd, __pmd(0));
 189         make_lowmem_page_writable(bm_pte, XENFEAT_writable_page_tables);
 190         /* paravirt_release_pte(__pa(bm_pte) >> PAGE_SHIFT); */
 191         __flush_tlb_all();
 192 Index: head-2008-12-01/arch/x86/mm/pgtable-xen.c
 193 ===================================================================
 194 --- head-2008-12-01.orig/arch/x86/mm/pgtable-xen.c      2008-12-01 11:49:07.000000000 +0100
 195 +++ head-2008-12-01/arch/x86/mm/pgtable-xen.c   2008-12-01 12:13:06.000000000 +0100
 196 @@ -42,16 +42,16 @@ pgtable_t pte_alloc_one(struct mm_struct
 197  void __pte_free(pgtable_t pte)
 198  {
 199         if (!PageHighMem(pte)) {
 200 -               unsigned long va = (unsigned long)page_address(pte);
 201 -               unsigned int level;
 202 -               pte_t *ptep = lookup_address(va, &level);
 203 -
 204 -               BUG_ON(!ptep || level != PG_LEVEL_4K || !pte_present(*ptep));
 205 -               if (!pte_write(*ptep)
 206 -                   && HYPERVISOR_update_va_mapping(va,
 207 -                                                   mk_pte(pte, PAGE_KERNEL),
 208 -                                                   0))
 209 -                       BUG();
 210 +               if (PagePinned(pte)) {
 211 +                       unsigned long pfn = page_to_pfn(pte);
 212 +
 213 +                       if (HYPERVISOR_update_va_mapping((unsigned long)__va(pfn << PAGE_SHIFT),
 214 +                                                        pfn_pte(pfn,
 215 +                                                                PAGE_KERNEL),
 216 +                                                        0))
 217 +                               BUG();
 218 +                       ClearPagePinned(pte);
 219 +               }
 220         } else
 221  #ifdef CONFIG_HIGHPTE
 222                 ClearPagePinned(pte);
 223 @@ -93,14 +93,15 @@ pmd_t *pmd_alloc_one(struct mm_struct *m
 224
 225  void __pmd_free(pgtable_t pmd)
 226  {
 227 -       unsigned long va = (unsigned long)page_address(pmd);
 228 -       unsigned int level;
 229 -       pte_t *ptep = lookup_address(va, &level);
 230 -
 231 -       BUG_ON(!ptep || level != PG_LEVEL_4K || !pte_present(*ptep));
 232 -       if (!pte_write(*ptep)
 233 -           && HYPERVISOR_update_va_mapping(va, mk_pte(pmd, PAGE_KERNEL), 0))
 234 -               BUG();
 235 +       if (PagePinned(pmd)) {
 236 +               unsigned long pfn = page_to_pfn(pmd);
 237 +
 238 +               if (HYPERVISOR_update_va_mapping((unsigned long)__va(pfn << PAGE_SHIFT),
 239 +                                                pfn_pte(pfn, PAGE_KERNEL),
 240 +                                                0))
 241 +                       BUG();
 242 +               ClearPagePinned(pmd);
 243 +       }
 244
 245         ClearPageForeign(pmd);
 246         init_page_count(pmd);
 247 @@ -192,21 +193,20 @@ static inline unsigned int pgd_walk_set_
 248  {
 249         unsigned long pfn = page_to_pfn(page);
 250
 251 -       if (PageHighMem(page)) {
 252 -               if (pgprot_val(flags) & _PAGE_RW)
 253 -                       ClearPagePinned(page);
 254 -               else
 255 -                       SetPagePinned(page);
 256 -       } else {
 257 -               MULTI_update_va_mapping(per_cpu(pb_mcl, cpu) + seq,
 258 -                                       (unsigned long)__va(pfn << PAGE_SHIFT),
 259 -                                       pfn_pte(pfn, flags), 0);
 260 -               if (unlikely(++seq == PIN_BATCH)) {
 261 -                       if (unlikely(HYPERVISOR_multicall_check(per_cpu(pb_mcl, cpu),
 262 -                                                               PIN_BATCH, NULL)))
 263 -                               BUG();
 264 -                       seq = 0;
 265 -               }
 266 +       if (pgprot_val(flags) & _PAGE_RW)
 267 +               ClearPagePinned(page);
 268 +       else
 269 +               SetPagePinned(page);
 270 +       if (PageHighMem(page))
 271 +               return seq;
 272 +       MULTI_update_va_mapping(per_cpu(pb_mcl, cpu) + seq,
 273 +                               (unsigned long)__va(pfn << PAGE_SHIFT),
 274 +                               pfn_pte(pfn, flags), 0);
 275 +       if (unlikely(++seq == PIN_BATCH)) {
 276 +               if (unlikely(HYPERVISOR_multicall_check(per_cpu(pb_mcl, cpu),
 277 +                                                       PIN_BATCH, NULL)))
 278 +                       BUG();
 279 +               seq = 0;
 280         }
 281
 282         return seq;
 283 @@ -253,6 +253,16 @@ static void pgd_walk(pgd_t *pgd_base, pg
 284                 }
 285         }
 286
 287 +#ifdef CONFIG_X86_PAE
 288 +       for (; g < PTRS_PER_PGD; g++, pgd++) {
 289 +               BUG_ON(pgd_none(*pgd));
 290 +               pud = pud_offset(pgd, 0);
 291 +               BUG_ON(pud_none(*pud));
 292 +               pmd = pmd_offset(pud, 0);
 293 +               seq = pgd_walk_set_prot(virt_to_page(pmd),flags,cpu,seq);
 294 +       }
 295 +#endif
 296 +
 297         mcl = per_cpu(pb_mcl, cpu);
 298  #ifdef CONFIG_X86_64
 299         if (unlikely(seq > PIN_BATCH - 2)) {
 300 @@ -288,6 +298,51 @@ static void pgd_walk(pgd_t *pgd_base, pg
 301         put_cpu();
 302  }
 303
 304 +void __init xen_init_pgd_pin(void)
 305 +{
 306 +       pgd_t       *pgd = init_mm.pgd;
 307 +       pud_t       *pud;
 308 +       pmd_t       *pmd;
 309 +       unsigned int g, u, m;
 310 +
 311 +       if (xen_feature(XENFEAT_auto_translated_physmap))
 312 +               return;
 313 +
 314 +       SetPagePinned(virt_to_page(pgd));
 315 +       for (g = 0; g < PTRS_PER_PGD; g++, pgd++) {
 316 +#ifndef CONFIG_X86_PAE
 317 +               if (g >= pgd_index(HYPERVISOR_VIRT_START)
 318 +                   && g <= pgd_index(HYPERVISOR_VIRT_END - 1))
 319 +                       continue;
 320 +#endif
 321 +               if (!pgd_present(*pgd))
 322 +                       continue;
 323 +               pud = pud_offset(pgd, 0);
 324 +               if (PTRS_PER_PUD > 1) /* not folded */
 325 +                       SetPagePinned(virt_to_page(pud));
 326 +               for (u = 0; u < PTRS_PER_PUD; u++, pud++) {
 327 +                       if (!pud_present(*pud))
 328 +                               continue;
 329 +                       pmd = pmd_offset(pud, 0);
 330 +                       if (PTRS_PER_PMD > 1) /* not folded */
 331 +                               SetPagePinned(virt_to_page(pmd));
 332 +                       for (m = 0; m < PTRS_PER_PMD; m++, pmd++) {
 333 +#ifdef CONFIG_X86_PAE
 334 +                               if (g == pgd_index(HYPERVISOR_VIRT_START)
 335 +                                   && m >= pmd_index(HYPERVISOR_VIRT_START))
 336 +                                       continue;
 337 +#endif
 338 +                               if (!pmd_present(*pmd))
 339 +                                       continue;
 340 +                               SetPagePinned(pmd_page(*pmd));
 341 +                       }
 342 +               }
 343 +       }
 344 +#ifdef CONFIG_X86_64
 345 +       SetPagePinned(virt_to_page(level3_user_pgt));
 346 +#endif
 347 +}
 348 +
 349  static void __pgd_pin(pgd_t *pgd)
 350  {
 351         pgd_walk(pgd, PAGE_KERNEL_RO);
 352 @@ -480,21 +535,18 @@ static void pgd_dtor(void *pgd)
 353
 354  void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
 355  {
 356 -       struct page *page = virt_to_page(pmd);
 357 -       unsigned long pfn = page_to_pfn(page);
 358 -
 359 -       paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
 360 -
 361         /* Note: almost everything apart from _PAGE_PRESENT is
 362            reserved at the pmd (PDPT) level. */
 363 -       if (PagePinned(virt_to_page(mm->pgd))) {
 364 -               BUG_ON(PageHighMem(page));
 365 -               BUG_ON(HYPERVISOR_update_va_mapping(
 366 -                         (unsigned long)__va(pfn << PAGE_SHIFT),
 367 -                         pfn_pte(pfn, PAGE_KERNEL_RO), 0));
 368 -               set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT));
 369 -       } else
 370 -               *pudp = __pud(__pa(pmd) | _PAGE_PRESENT);
 371 +       pud_t pud = __pud(__pa(pmd) | _PAGE_PRESENT);
 372 +
 373 +       paravirt_alloc_pmd(mm, page_to_pfn(virt_to_page(pmd)));
 374 +
 375 +       if (likely(!PagePinned(virt_to_page(pudp)))) {
 376 +               *pudp = pud;
 377 +               return;
 378 +       }
 379 +
 380 +       set_pud(pudp, pud);
 381
 382         /*
 383          * According to Intel App note "TLBs, Paging-Structure Caches,
 384 @@ -585,13 +637,10 @@ static void pgd_prepopulate_pmd(struct m
 385              i++, pud++, addr += PUD_SIZE) {
 386                 pmd_t *pmd = pmds[i];
 387
 388 -               if (i >= KERNEL_PGD_BOUNDARY) {
 389 +               if (i >= KERNEL_PGD_BOUNDARY)
 390                         memcpy(pmd,
 391                                (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
 392                                sizeof(pmd_t) * PTRS_PER_PMD);
 393 -                       make_lowmem_page_readonly(
 394 -                               pmd, XENFEAT_writable_page_tables);
 395 -               }
 396
 397                 /* It is safe to poke machine addresses of pmds under the pgd_lock. */
 398                 pud_populate(mm, pud, pmd);
 399 Index: head-2008-12-01/include/asm-x86/mach-xen/asm/hypervisor.h
 400 ===================================================================
 401 --- head-2008-12-01.orig/include/asm-x86/mach-xen/asm/hypervisor.h      2008-12-01 12:07:29.000000000 +0100
 402 +++ head-2008-12-01/include/asm-x86/mach-xen/asm/hypervisor.h   2008-12-01 12:13:06.000000000 +0100
 403 @@ -94,10 +94,12 @@ void xen_invlpg(unsigned long ptr);
 404  void xen_l1_entry_update(pte_t *ptr, pte_t val);
 405  void xen_l2_entry_update(pmd_t *ptr, pmd_t val);
 406  void xen_l3_entry_update(pud_t *ptr, pud_t val); /* x86_64/PAE */
 407 -void xen_l4_entry_update(pgd_t *ptr, pgd_t val); /* x86_64 only */
 408 +void xen_l4_entry_update(pgd_t *ptr, int user, pgd_t val); /* x86_64 only */
 409  void xen_pgd_pin(unsigned long ptr);
 410  void xen_pgd_unpin(unsigned long ptr);
 411
 412 +void xen_init_pgd_pin(void);
 413 +
 414  void xen_set_ldt(const void *ptr, unsigned int ents);
 415
 416  #ifdef CONFIG_SMP
 417 @@ -323,6 +325,18 @@ MULTI_update_va_mapping(
 418  }
 419
 420  static inline void
 421 +MULTI_mmu_update(multicall_entry_t *mcl, mmu_update_t *req,
 422 +                unsigned int count, unsigned int *success_count,
 423 +                domid_t domid)
 424 +{
 425 +    mcl->op = __HYPERVISOR_mmu_update;
 426 +    mcl->args[0] = (unsigned long)req;
 427 +    mcl->args[1] = count;
 428 +    mcl->args[2] = (unsigned long)success_count;
 429 +    mcl->args[3] = domid;
 430 +}
 431 +
 432 +static inline void
 433  MULTI_grant_table_op(multicall_entry_t *mcl, unsigned int cmd,
 434                      void *uop, unsigned int count)
 435  {
 436 Index: head-2008-12-01/include/asm-x86/mach-xen/asm/pgalloc.h
 437 ===================================================================
 438 --- head-2008-12-01.orig/include/asm-x86/mach-xen/asm/pgalloc.h 2008-12-01 11:49:07.000000000 +0100
 439 +++ head-2008-12-01/include/asm-x86/mach-xen/asm/pgalloc.h      2008-12-01 12:13:06.000000000 +0100
 440 @@ -64,20 +64,16 @@ static inline void pmd_populate(struct m
 441                                 struct page *pte)
 442  {
 443         unsigned long pfn = page_to_pfn(pte);
 444 +       pmd_t ent = __pmd(((pmdval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE);
 445
 446         paravirt_alloc_pte(mm, pfn);
 447 -       if (PagePinned(virt_to_page(mm->pgd))) {
 448 -               if (!PageHighMem(pte))
 449 -                       BUG_ON(HYPERVISOR_update_va_mapping(
 450 -                         (unsigned long)__va(pfn << PAGE_SHIFT),
 451 -                         pfn_pte(pfn, PAGE_KERNEL_RO), 0));
 452 -#ifndef CONFIG_X86_64
 453 -               else if (!TestSetPagePinned(pte))
 454 -                       kmap_flush_unused();
 455 +       if (PagePinned(virt_to_page(pmd))) {
 456 +#ifndef CONFIG_HIGHPTE
 457 +               BUG_ON(PageHighMem(pte));
 458  #endif
 459 -               set_pmd(pmd, __pmd(((pmdval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE));
 460 +               set_pmd(pmd, ent);
 461         } else
 462 -               *pmd = __pmd(((pmdval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE);
 463 +               *pmd = ent;
 464  }
 465
 466  #define pmd_pgtable(pmd) pmd_page(pmd)
 467 @@ -99,39 +95,28 @@ extern void pud_populate(struct mm_struc
 468  #else  /* !CONFIG_X86_PAE */
 469  static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
 470  {
 471 +       pud_t ent = __pud(_PAGE_TABLE | __pa(pmd));
 472 +
 473         paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
 474 -       if (unlikely(PagePinned(virt_to_page((mm)->pgd)))) {
 475 -               BUG_ON(HYPERVISOR_update_va_mapping(
 476 -                              (unsigned long)pmd,
 477 -                              pfn_pte(virt_to_phys(pmd)>>PAGE_SHIFT,
 478 -                                      PAGE_KERNEL_RO), 0));
 479 -               set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd)));
 480 -       } else
 481 -               *pud =  __pud(_PAGE_TABLE | __pa(pmd));
 482 +       if (PagePinned(virt_to_page(pud)))
 483 +               set_pud(pud, ent);
 484 +       else
 485 +               *pud = ent;
 486  }
 487  #endif /* CONFIG_X86_PAE */
 488
 489  #if PAGETABLE_LEVELS > 3
 490  #define __user_pgd(pgd) ((pgd) + PTRS_PER_PGD)
 491
 492 -/*
 493 - * We need to use the batch mode here, but pgd_pupulate() won't be
 494 - * be called frequently.
 495 - */
 496  static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
 497  {
 498 +       pgd_t ent = __pgd(_PAGE_TABLE | __pa(pud));
 499 +
 500         paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT);
 501 -       if (unlikely(PagePinned(virt_to_page((mm)->pgd)))) {
 502 -               BUG_ON(HYPERVISOR_update_va_mapping(
 503 -                              (unsigned long)pud,
 504 -                              pfn_pte(virt_to_phys(pud)>>PAGE_SHIFT,
 505 -                                      PAGE_KERNEL_RO), 0));
 506 -               set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pud)));
 507 -               set_pgd(__user_pgd(pgd), __pgd(_PAGE_TABLE | __pa(pud)));
 508 -       } else {
 509 -               *(pgd) =  __pgd(_PAGE_TABLE | __pa(pud));
 510 -               *__user_pgd(pgd) = *(pgd);
 511 -       }
 512 +       if (unlikely(PagePinned(virt_to_page(pgd))))
 513 +               xen_l4_entry_update(pgd, 1, ent);
 514 +       else
 515 +               *__user_pgd(pgd) = *pgd = ent;
 516  }
 517
 518  static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
 519 Index: head-2008-12-01/include/asm-x86/mach-xen/asm/pgtable-3level.h
 520 ===================================================================
 521 --- head-2008-12-01.orig/include/asm-x86/mach-xen/asm/pgtable-3level.h  2008-12-01 11:49:07.000000000 +0100
 522 +++ head-2008-12-01/include/asm-x86/mach-xen/asm/pgtable-3level.h       2008-12-01 12:13:06.000000000 +0100
 523 @@ -76,12 +76,15 @@ static inline void __xen_pte_clear(pte_t
 524         ptep->pte_high = 0;
 525  }
 526
 527 -static inline void xen_pmd_clear(pmd_t *pmd)
 528 -{
 529 -       xen_l2_entry_update(pmd, __pmd(0));
 530 -}
 531 +#define xen_pmd_clear(pmd)                     \
 532 +({                                             \
 533 +       pmd_t *__pmdp = (pmd);                  \
 534 +       PagePinned(virt_to_page(__pmdp))        \
 535 +       ? set_pmd(__pmdp, __pmd(0))             \
 536 +       : (void)(*__pmdp = __pmd(0));           \
 537 +})
 538
 539 -static inline void pud_clear(pud_t *pudp)
 540 +static inline void __xen_pud_clear(pud_t *pudp)
 541  {
 542         pgdval_t pgd;
 543
 544 @@ -102,13 +105,21 @@ static inline void pud_clear(pud_t *pudp
 545                 xen_tlb_flush();
 546  }
 547
 548 -#define pud_page(pud) ((struct page *) __va(pud_val(pud) & PTE_PFN_MASK))
 549 +#define xen_pud_clear(pudp)                    \
 550 +({                                             \
 551 +       pud_t *__pudp = (pudp);                 \
 552 +       PagePinned(virt_to_page(__pudp))        \
 553 +       ? __xen_pud_clear(__pudp)               \
 554 +       : (void)(*__pudp = __pud(0));           \
 555 +})
 556 +
 557 +#define pud_page(pud) pfn_to_page(pud_val(pud) >> PAGE_SHIFT)
 558
 559  #define pud_page_vaddr(pud) ((unsigned long) __va(pud_val(pud) & PTE_PFN_MASK))
 560
 561
 562  /* Find an entry in the second-level page table.. */
 563 -#define pmd_offset(pud, address) ((pmd_t *)pud_page(*(pud)) +  \
 564 +#define pmd_offset(pud, address) ((pmd_t *)pud_page_vaddr(*(pud)) +    \
 565                                   pmd_index(address))
 566
 567  #ifdef CONFIG_SMP
 568 Index: head-2008-12-01/include/asm-x86/mach-xen/asm/pgtable_64.h
 569 ===================================================================
 570 --- head-2008-12-01.orig/include/asm-x86/mach-xen/asm/pgtable_64.h      2008-12-01 11:49:07.000000000 +0100
 571 +++ head-2008-12-01/include/asm-x86/mach-xen/asm/pgtable_64.h   2008-12-01 12:13:06.000000000 +0100
 572 @@ -110,33 +110,41 @@ static inline void xen_set_pmd(pmd_t *pm
 573         xen_l2_entry_update(pmdp, pmd);
 574  }
 575
 576 -static inline void xen_pmd_clear(pmd_t *pmd)
 577 -{
 578 -       xen_set_pmd(pmd, xen_make_pmd(0));
 579 -}
 580 +#define xen_pmd_clear(pmd)                     \
 581 +({                                             \
 582 +       pmd_t *__pmdp = (pmd);                  \
 583 +       PagePinned(virt_to_page(__pmdp))        \
 584 +       ? set_pmd(__pmdp, xen_make_pmd(0))      \
 585 +       : (void)(*__pmdp = xen_make_pmd(0));    \
 586 +})
 587
 588  static inline void xen_set_pud(pud_t *pudp, pud_t pud)
 589  {
 590         xen_l3_entry_update(pudp, pud);
 591  }
 592
 593 -static inline void xen_pud_clear(pud_t *pud)
 594 -{
 595 -       xen_set_pud(pud, xen_make_pud(0));
 596 -}
 597 +#define xen_pud_clear(pud)                     \
 598 +({                                             \
 599 +       pud_t *__pudp = (pud);                  \
 600 +       PagePinned(virt_to_page(__pudp))        \
 601 +       ? set_pud(__pudp, xen_make_pud(0))      \
 602 +       : (void)(*__pudp = xen_make_pud(0));    \
 603 +})
 604
 605  #define __user_pgd(pgd) ((pgd) + PTRS_PER_PGD)
 606
 607  static inline void xen_set_pgd(pgd_t *pgdp, pgd_t pgd)
 608  {
 609 -       xen_l4_entry_update(pgdp, pgd);
 610 +       xen_l4_entry_update(pgdp, 0, pgd);
 611  }
 612
 613 -static inline void xen_pgd_clear(pgd_t *pgd)
 614 -{
 615 -       xen_set_pgd(pgd, xen_make_pgd(0));
 616 -       xen_set_pgd(__user_pgd(pgd), xen_make_pgd(0));
 617 -}
 618 +#define xen_pgd_clear(pgd)                     \
 619 +({                                             \
 620 +       pgd_t *__pgdp = (pgd);                  \
 621 +       PagePinned(virt_to_page(__pgdp))        \
 622 +       ? xen_l4_entry_update(__pgdp, 1, xen_make_pgd(0)) \
 623 +       : (void)(*__user_pgd(__pgdp) = *__pgdp = xen_make_pgd(0)); \
 624 +})
 625
 626  #define pte_same(a, b)         ((a).pte == (b).pte)
 627