1 From: jbeulich@novell.com
2 Subject: consolidate pmd/pud/pgd entry handling
3 Patch-mainline: obsolete
5 Index: head-2008-12-01/arch/x86/mm/hypervisor.c
6 ===================================================================
7 --- head-2008-12-01.orig/arch/x86/mm/hypervisor.c 2008-12-01 12:10:32.000000000 +0100
8 +++ head-2008-12-01/arch/x86/mm/hypervisor.c 2008-12-01 12:13:06.000000000 +0100
9 @@ -356,31 +356,91 @@ void xen_l1_entry_update(pte_t *ptr, pte
11 EXPORT_SYMBOL_GPL(xen_l1_entry_update);
13 +static void do_lN_entry_update(mmu_update_t *mmu, unsigned int mmu_count,
17 + multicall_entry_t mcl[2];
18 + unsigned long pfn = page_to_pfn(page);
20 + MULTI_update_va_mapping(mcl,
21 + (unsigned long)__va(pfn << PAGE_SHIFT),
22 + pfn_pte(pfn, PAGE_KERNEL_RO), 0);
23 + SetPagePinned(page);
24 + MULTI_mmu_update(mcl + 1, mmu, mmu_count, NULL, DOMID_SELF);
25 + if (unlikely(HYPERVISOR_multicall_check(mcl, 2, NULL)))
27 + } else if (unlikely(HYPERVISOR_mmu_update(mmu, mmu_count,
28 + NULL, DOMID_SELF) < 0))
32 void xen_l2_entry_update(pmd_t *ptr, pmd_t val)
35 + struct page *page = NULL;
37 + if (likely(pmd_present(val)) && likely(!pmd_large(val))
39 + && likely(PagePinned(virt_to_page(ptr)))) {
40 + page = pmd_page(val);
41 + if (unlikely(PagePinned(page)))
43 + else if (PageHighMem(page)) {
44 +#ifdef CONFIG_HIGHPTE
47 + kmap_flush_unused();
51 u.ptr = virt_to_machine(ptr);
52 u.val = __pmd_val(val);
53 - BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
54 + do_lN_entry_update(&u, 1, page);
57 #if defined(CONFIG_X86_PAE) || defined(CONFIG_X86_64)
58 void xen_l3_entry_update(pud_t *ptr, pud_t val)
61 + struct page *page = NULL;
63 + if (likely(pud_present(val))
65 + && likely(!pud_large(val))
68 + && likely(PagePinned(virt_to_page(ptr)))) {
69 + page = pud_page(val);
70 + if (unlikely(PagePinned(page)))
73 u.ptr = virt_to_machine(ptr);
74 u.val = __pud_val(val);
75 - BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
76 + do_lN_entry_update(&u, 1, page);
81 -void xen_l4_entry_update(pgd_t *ptr, pgd_t val)
82 +void xen_l4_entry_update(pgd_t *ptr, int user, pgd_t val)
85 - u.ptr = virt_to_machine(ptr);
86 - u.val = __pgd_val(val);
87 - BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
89 + struct page *page = NULL;
91 + if (likely(pgd_present(val)) && likely(mem_map)
92 + && likely(PagePinned(virt_to_page(ptr)))) {
93 + page = pgd_page(val);
94 + if (unlikely(PagePinned(page)))
97 + u[0].ptr = virt_to_machine(ptr);
98 + u[0].val = __pgd_val(val);
100 + u[1].ptr = virt_to_machine(__user_pgd(ptr));
101 + u[1].val = __pgd_val(val);
102 + do_lN_entry_update(u, 2, page);
104 + do_lN_entry_update(u, 1, page);
106 #endif /* CONFIG_X86_64 */
108 Index: head-2008-12-01/arch/x86/mm/init_32-xen.c
109 ===================================================================
110 --- head-2008-12-01.orig/arch/x86/mm/init_32-xen.c 2008-12-01 11:49:07.000000000 +0100
111 +++ head-2008-12-01/arch/x86/mm/init_32-xen.c 2008-12-01 12:13:06.000000000 +0100
112 @@ -681,6 +681,8 @@ static void __init zone_sizes_init(void)
115 free_area_init_nodes(max_zone_pfns);
117 + xen_init_pgd_pin();
120 void __init setup_bootmem_allocator(void)
121 @@ -1040,8 +1042,6 @@ void __init mem_init(void)
126 - SetPagePinned(virt_to_page(init_mm.pgd));
129 #ifdef CONFIG_MEMORY_HOTPLUG
130 Index: head-2008-12-01/arch/x86/mm/init_64-xen.c
131 ===================================================================
132 --- head-2008-12-01.orig/arch/x86/mm/init_64-xen.c 2008-12-01 11:49:07.000000000 +0100
133 +++ head-2008-12-01/arch/x86/mm/init_64-xen.c 2008-12-01 12:13:06.000000000 +0100
134 @@ -195,7 +195,10 @@ set_pte_vaddr_pud(pud_t *pud_page, unsig
135 if (pud_none(*pud)) {
136 pmd = (pmd_t *) spp_getpage();
137 make_page_readonly(pmd, XENFEAT_writable_page_tables);
138 - pud_populate(&init_mm, pud, pmd);
139 + if (!after_bootmem)
140 + xen_l3_entry_update(pud, __pud(__pa(pmd) | _PAGE_TABLE));
142 + pud_populate(&init_mm, pud, pmd);
143 if (pmd != pmd_offset(pud, 0)) {
144 printk(KERN_ERR "PAGETABLE BUG #01! %p <-> %p\n",
145 pmd, pmd_offset(pud, 0));
146 @@ -462,7 +465,6 @@ phys_pmd_init(pmd_t *pmd_page, unsigned
147 early_make_page_readonly(pte, XENFEAT_writable_page_tables);
148 *pmd = __pmd(pte_phys | _PAGE_TABLE);
150 - make_page_readonly(pte, XENFEAT_writable_page_tables);
151 spin_lock(&init_mm.page_table_lock);
152 pmd_populate_kernel(&init_mm, pmd, __va(pte_phys));
153 spin_unlock(&init_mm.page_table_lock);
154 @@ -529,7 +531,6 @@ phys_pud_init(pud_t *pud_page, unsigned
156 *pud = __pud(pmd_phys | _PAGE_TABLE);
158 - make_page_readonly(pmd, XENFEAT_writable_page_tables);
159 spin_lock(&init_mm.page_table_lock);
160 pud_populate(&init_mm, pud, __va(pmd_phys));
161 spin_unlock(&init_mm.page_table_lock);
162 @@ -824,7 +825,6 @@ static unsigned long __init kernel_physi
163 early_make_page_readonly(pud, XENFEAT_writable_page_tables);
164 xen_l4_entry_update(pgd, __pgd(pud_phys | _PAGE_TABLE));
166 - make_page_readonly(pud, XENFEAT_writable_page_tables);
167 spin_lock(&init_mm.page_table_lock);
168 pgd_populate(&init_mm, pgd, __va(pud_phys));
169 spin_unlock(&init_mm.page_table_lock);
170 @@ -1013,7 +1013,7 @@ void __init paging_init(void)
172 free_area_init_nodes(max_zone_pfns);
174 - SetPagePinned(virt_to_page(init_mm.pgd));
175 + xen_init_pgd_pin();
179 Index: head-2008-12-01/arch/x86/mm/ioremap-xen.c
180 ===================================================================
181 --- head-2008-12-01.orig/arch/x86/mm/ioremap-xen.c 2008-12-01 12:07:15.000000000 +0100
182 +++ head-2008-12-01/arch/x86/mm/ioremap-xen.c 2008-12-01 12:13:06.000000000 +0100
183 @@ -678,7 +678,7 @@ void __init early_ioremap_clear(void)
184 printk(KERN_INFO "early_ioremap_clear()\n");
186 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
188 + xen_l2_entry_update(pmd, __pmd(0));
189 make_lowmem_page_writable(bm_pte, XENFEAT_writable_page_tables);
190 /* paravirt_release_pte(__pa(bm_pte) >> PAGE_SHIFT); */
192 Index: head-2008-12-01/arch/x86/mm/pgtable-xen.c
193 ===================================================================
194 --- head-2008-12-01.orig/arch/x86/mm/pgtable-xen.c 2008-12-01 11:49:07.000000000 +0100
195 +++ head-2008-12-01/arch/x86/mm/pgtable-xen.c 2008-12-01 12:13:06.000000000 +0100
196 @@ -42,16 +42,16 @@ pgtable_t pte_alloc_one(struct mm_struct
197 void __pte_free(pgtable_t pte)
199 if (!PageHighMem(pte)) {
200 - unsigned long va = (unsigned long)page_address(pte);
201 - unsigned int level;
202 - pte_t *ptep = lookup_address(va, &level);
204 - BUG_ON(!ptep || level != PG_LEVEL_4K || !pte_present(*ptep));
205 - if (!pte_write(*ptep)
206 - && HYPERVISOR_update_va_mapping(va,
207 - mk_pte(pte, PAGE_KERNEL),
210 + if (PagePinned(pte)) {
211 + unsigned long pfn = page_to_pfn(pte);
213 + if (HYPERVISOR_update_va_mapping((unsigned long)__va(pfn << PAGE_SHIFT),
218 + ClearPagePinned(pte);
221 #ifdef CONFIG_HIGHPTE
222 ClearPagePinned(pte);
223 @@ -93,14 +93,15 @@ pmd_t *pmd_alloc_one(struct mm_struct *m
225 void __pmd_free(pgtable_t pmd)
227 - unsigned long va = (unsigned long)page_address(pmd);
228 - unsigned int level;
229 - pte_t *ptep = lookup_address(va, &level);
231 - BUG_ON(!ptep || level != PG_LEVEL_4K || !pte_present(*ptep));
232 - if (!pte_write(*ptep)
233 - && HYPERVISOR_update_va_mapping(va, mk_pte(pmd, PAGE_KERNEL), 0))
235 + if (PagePinned(pmd)) {
236 + unsigned long pfn = page_to_pfn(pmd);
238 + if (HYPERVISOR_update_va_mapping((unsigned long)__va(pfn << PAGE_SHIFT),
239 + pfn_pte(pfn, PAGE_KERNEL),
242 + ClearPagePinned(pmd);
245 ClearPageForeign(pmd);
246 init_page_count(pmd);
247 @@ -192,21 +193,20 @@ static inline unsigned int pgd_walk_set_
249 unsigned long pfn = page_to_pfn(page);
251 - if (PageHighMem(page)) {
252 - if (pgprot_val(flags) & _PAGE_RW)
253 - ClearPagePinned(page);
255 - SetPagePinned(page);
257 - MULTI_update_va_mapping(per_cpu(pb_mcl, cpu) + seq,
258 - (unsigned long)__va(pfn << PAGE_SHIFT),
259 - pfn_pte(pfn, flags), 0);
260 - if (unlikely(++seq == PIN_BATCH)) {
261 - if (unlikely(HYPERVISOR_multicall_check(per_cpu(pb_mcl, cpu),
266 + if (pgprot_val(flags) & _PAGE_RW)
267 + ClearPagePinned(page);
269 + SetPagePinned(page);
270 + if (PageHighMem(page))
272 + MULTI_update_va_mapping(per_cpu(pb_mcl, cpu) + seq,
273 + (unsigned long)__va(pfn << PAGE_SHIFT),
274 + pfn_pte(pfn, flags), 0);
275 + if (unlikely(++seq == PIN_BATCH)) {
276 + if (unlikely(HYPERVISOR_multicall_check(per_cpu(pb_mcl, cpu),
283 @@ -253,6 +253,16 @@ static void pgd_walk(pgd_t *pgd_base, pg
287 +#ifdef CONFIG_X86_PAE
288 + for (; g < PTRS_PER_PGD; g++, pgd++) {
289 + BUG_ON(pgd_none(*pgd));
290 + pud = pud_offset(pgd, 0);
291 + BUG_ON(pud_none(*pud));
292 + pmd = pmd_offset(pud, 0);
293 + seq = pgd_walk_set_prot(virt_to_page(pmd),flags,cpu,seq);
297 mcl = per_cpu(pb_mcl, cpu);
299 if (unlikely(seq > PIN_BATCH - 2)) {
300 @@ -288,6 +298,51 @@ static void pgd_walk(pgd_t *pgd_base, pg
304 +void __init xen_init_pgd_pin(void)
306 + pgd_t *pgd = init_mm.pgd;
309 + unsigned int g, u, m;
311 + if (xen_feature(XENFEAT_auto_translated_physmap))
314 + SetPagePinned(virt_to_page(pgd));
315 + for (g = 0; g < PTRS_PER_PGD; g++, pgd++) {
316 +#ifndef CONFIG_X86_PAE
317 + if (g >= pgd_index(HYPERVISOR_VIRT_START)
318 + && g <= pgd_index(HYPERVISOR_VIRT_END - 1))
321 + if (!pgd_present(*pgd))
323 + pud = pud_offset(pgd, 0);
324 + if (PTRS_PER_PUD > 1) /* not folded */
325 + SetPagePinned(virt_to_page(pud));
326 + for (u = 0; u < PTRS_PER_PUD; u++, pud++) {
327 + if (!pud_present(*pud))
329 + pmd = pmd_offset(pud, 0);
330 + if (PTRS_PER_PMD > 1) /* not folded */
331 + SetPagePinned(virt_to_page(pmd));
332 + for (m = 0; m < PTRS_PER_PMD; m++, pmd++) {
333 +#ifdef CONFIG_X86_PAE
334 + if (g == pgd_index(HYPERVISOR_VIRT_START)
335 + && m >= pmd_index(HYPERVISOR_VIRT_START))
338 + if (!pmd_present(*pmd))
340 + SetPagePinned(pmd_page(*pmd));
344 +#ifdef CONFIG_X86_64
345 + SetPagePinned(virt_to_page(level3_user_pgt));
349 static void __pgd_pin(pgd_t *pgd)
351 pgd_walk(pgd, PAGE_KERNEL_RO);
352 @@ -480,21 +535,18 @@ static void pgd_dtor(void *pgd)
354 void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
356 - struct page *page = virt_to_page(pmd);
357 - unsigned long pfn = page_to_pfn(page);
359 - paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
361 /* Note: almost everything apart from _PAGE_PRESENT is
362 reserved at the pmd (PDPT) level. */
363 - if (PagePinned(virt_to_page(mm->pgd))) {
364 - BUG_ON(PageHighMem(page));
365 - BUG_ON(HYPERVISOR_update_va_mapping(
366 - (unsigned long)__va(pfn << PAGE_SHIFT),
367 - pfn_pte(pfn, PAGE_KERNEL_RO), 0));
368 - set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT));
370 - *pudp = __pud(__pa(pmd) | _PAGE_PRESENT);
371 + pud_t pud = __pud(__pa(pmd) | _PAGE_PRESENT);
373 + paravirt_alloc_pmd(mm, page_to_pfn(virt_to_page(pmd)));
375 + if (likely(!PagePinned(virt_to_page(pudp)))) {
380 + set_pud(pudp, pud);
383 * According to Intel App note "TLBs, Paging-Structure Caches,
384 @@ -585,13 +637,10 @@ static void pgd_prepopulate_pmd(struct m
385 i++, pud++, addr += PUD_SIZE) {
386 pmd_t *pmd = pmds[i];
388 - if (i >= KERNEL_PGD_BOUNDARY) {
389 + if (i >= KERNEL_PGD_BOUNDARY)
391 (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
392 sizeof(pmd_t) * PTRS_PER_PMD);
393 - make_lowmem_page_readonly(
394 - pmd, XENFEAT_writable_page_tables);
397 /* It is safe to poke machine addresses of pmds under the pgd_lock. */
398 pud_populate(mm, pud, pmd);
399 Index: head-2008-12-01/include/asm-x86/mach-xen/asm/hypervisor.h
400 ===================================================================
401 --- head-2008-12-01.orig/include/asm-x86/mach-xen/asm/hypervisor.h 2008-12-01 12:07:29.000000000 +0100
402 +++ head-2008-12-01/include/asm-x86/mach-xen/asm/hypervisor.h 2008-12-01 12:13:06.000000000 +0100
403 @@ -94,10 +94,12 @@ void xen_invlpg(unsigned long ptr);
404 void xen_l1_entry_update(pte_t *ptr, pte_t val);
405 void xen_l2_entry_update(pmd_t *ptr, pmd_t val);
406 void xen_l3_entry_update(pud_t *ptr, pud_t val); /* x86_64/PAE */
407 -void xen_l4_entry_update(pgd_t *ptr, pgd_t val); /* x86_64 only */
408 +void xen_l4_entry_update(pgd_t *ptr, int user, pgd_t val); /* x86_64 only */
409 void xen_pgd_pin(unsigned long ptr);
410 void xen_pgd_unpin(unsigned long ptr);
412 +void xen_init_pgd_pin(void);
414 void xen_set_ldt(const void *ptr, unsigned int ents);
417 @@ -323,6 +325,18 @@ MULTI_update_va_mapping(
421 +MULTI_mmu_update(multicall_entry_t *mcl, mmu_update_t *req,
422 + unsigned int count, unsigned int *success_count,
425 + mcl->op = __HYPERVISOR_mmu_update;
426 + mcl->args[0] = (unsigned long)req;
427 + mcl->args[1] = count;
428 + mcl->args[2] = (unsigned long)success_count;
429 + mcl->args[3] = domid;
433 MULTI_grant_table_op(multicall_entry_t *mcl, unsigned int cmd,
434 void *uop, unsigned int count)
436 Index: head-2008-12-01/include/asm-x86/mach-xen/asm/pgalloc.h
437 ===================================================================
438 --- head-2008-12-01.orig/include/asm-x86/mach-xen/asm/pgalloc.h 2008-12-01 11:49:07.000000000 +0100
439 +++ head-2008-12-01/include/asm-x86/mach-xen/asm/pgalloc.h 2008-12-01 12:13:06.000000000 +0100
440 @@ -64,20 +64,16 @@ static inline void pmd_populate(struct m
443 unsigned long pfn = page_to_pfn(pte);
444 + pmd_t ent = __pmd(((pmdval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE);
446 paravirt_alloc_pte(mm, pfn);
447 - if (PagePinned(virt_to_page(mm->pgd))) {
448 - if (!PageHighMem(pte))
449 - BUG_ON(HYPERVISOR_update_va_mapping(
450 - (unsigned long)__va(pfn << PAGE_SHIFT),
451 - pfn_pte(pfn, PAGE_KERNEL_RO), 0));
452 -#ifndef CONFIG_X86_64
453 - else if (!TestSetPagePinned(pte))
454 - kmap_flush_unused();
455 + if (PagePinned(virt_to_page(pmd))) {
456 +#ifndef CONFIG_HIGHPTE
457 + BUG_ON(PageHighMem(pte));
459 - set_pmd(pmd, __pmd(((pmdval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE));
462 - *pmd = __pmd(((pmdval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE);
466 #define pmd_pgtable(pmd) pmd_page(pmd)
467 @@ -99,39 +95,28 @@ extern void pud_populate(struct mm_struc
468 #else /* !CONFIG_X86_PAE */
469 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
471 + pud_t ent = __pud(_PAGE_TABLE | __pa(pmd));
473 paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
474 - if (unlikely(PagePinned(virt_to_page((mm)->pgd)))) {
475 - BUG_ON(HYPERVISOR_update_va_mapping(
476 - (unsigned long)pmd,
477 - pfn_pte(virt_to_phys(pmd)>>PAGE_SHIFT,
478 - PAGE_KERNEL_RO), 0));
479 - set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd)));
481 - *pud = __pud(_PAGE_TABLE | __pa(pmd));
482 + if (PagePinned(virt_to_page(pud)))
487 #endif /* CONFIG_X86_PAE */
489 #if PAGETABLE_LEVELS > 3
490 #define __user_pgd(pgd) ((pgd) + PTRS_PER_PGD)
493 - * We need to use the batch mode here, but pgd_pupulate() won't be
494 - * be called frequently.
496 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
498 + pgd_t ent = __pgd(_PAGE_TABLE | __pa(pud));
500 paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT);
501 - if (unlikely(PagePinned(virt_to_page((mm)->pgd)))) {
502 - BUG_ON(HYPERVISOR_update_va_mapping(
503 - (unsigned long)pud,
504 - pfn_pte(virt_to_phys(pud)>>PAGE_SHIFT,
505 - PAGE_KERNEL_RO), 0));
506 - set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pud)));
507 - set_pgd(__user_pgd(pgd), __pgd(_PAGE_TABLE | __pa(pud)));
509 - *(pgd) = __pgd(_PAGE_TABLE | __pa(pud));
510 - *__user_pgd(pgd) = *(pgd);
512 + if (unlikely(PagePinned(virt_to_page(pgd))))
513 + xen_l4_entry_update(pgd, 1, ent);
515 + *__user_pgd(pgd) = *pgd = ent;
518 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
519 Index: head-2008-12-01/include/asm-x86/mach-xen/asm/pgtable-3level.h
520 ===================================================================
521 --- head-2008-12-01.orig/include/asm-x86/mach-xen/asm/pgtable-3level.h 2008-12-01 11:49:07.000000000 +0100
522 +++ head-2008-12-01/include/asm-x86/mach-xen/asm/pgtable-3level.h 2008-12-01 12:13:06.000000000 +0100
523 @@ -76,12 +76,15 @@ static inline void __xen_pte_clear(pte_t
527 -static inline void xen_pmd_clear(pmd_t *pmd)
529 - xen_l2_entry_update(pmd, __pmd(0));
531 +#define xen_pmd_clear(pmd) \
533 + pmd_t *__pmdp = (pmd); \
534 + PagePinned(virt_to_page(__pmdp)) \
535 + ? set_pmd(__pmdp, __pmd(0)) \
536 + : (void)(*__pmdp = __pmd(0)); \
539 -static inline void pud_clear(pud_t *pudp)
540 +static inline void __xen_pud_clear(pud_t *pudp)
544 @@ -102,13 +105,21 @@ static inline void pud_clear(pud_t *pudp
548 -#define pud_page(pud) ((struct page *) __va(pud_val(pud) & PTE_PFN_MASK))
549 +#define xen_pud_clear(pudp) \
551 + pud_t *__pudp = (pudp); \
552 + PagePinned(virt_to_page(__pudp)) \
553 + ? __xen_pud_clear(__pudp) \
554 + : (void)(*__pudp = __pud(0)); \
557 +#define pud_page(pud) pfn_to_page(pud_val(pud) >> PAGE_SHIFT)
559 #define pud_page_vaddr(pud) ((unsigned long) __va(pud_val(pud) & PTE_PFN_MASK))
562 /* Find an entry in the second-level page table.. */
563 -#define pmd_offset(pud, address) ((pmd_t *)pud_page(*(pud)) + \
564 +#define pmd_offset(pud, address) ((pmd_t *)pud_page_vaddr(*(pud)) + \
568 Index: head-2008-12-01/include/asm-x86/mach-xen/asm/pgtable_64.h
569 ===================================================================
570 --- head-2008-12-01.orig/include/asm-x86/mach-xen/asm/pgtable_64.h 2008-12-01 11:49:07.000000000 +0100
571 +++ head-2008-12-01/include/asm-x86/mach-xen/asm/pgtable_64.h 2008-12-01 12:13:06.000000000 +0100
572 @@ -110,33 +110,41 @@ static inline void xen_set_pmd(pmd_t *pm
573 xen_l2_entry_update(pmdp, pmd);
576 -static inline void xen_pmd_clear(pmd_t *pmd)
578 - xen_set_pmd(pmd, xen_make_pmd(0));
580 +#define xen_pmd_clear(pmd) \
582 + pmd_t *__pmdp = (pmd); \
583 + PagePinned(virt_to_page(__pmdp)) \
584 + ? set_pmd(__pmdp, xen_make_pmd(0)) \
585 + : (void)(*__pmdp = xen_make_pmd(0)); \
588 static inline void xen_set_pud(pud_t *pudp, pud_t pud)
590 xen_l3_entry_update(pudp, pud);
593 -static inline void xen_pud_clear(pud_t *pud)
595 - xen_set_pud(pud, xen_make_pud(0));
597 +#define xen_pud_clear(pud) \
599 + pud_t *__pudp = (pud); \
600 + PagePinned(virt_to_page(__pudp)) \
601 + ? set_pud(__pudp, xen_make_pud(0)) \
602 + : (void)(*__pudp = xen_make_pud(0)); \
605 #define __user_pgd(pgd) ((pgd) + PTRS_PER_PGD)
607 static inline void xen_set_pgd(pgd_t *pgdp, pgd_t pgd)
609 - xen_l4_entry_update(pgdp, pgd);
610 + xen_l4_entry_update(pgdp, 0, pgd);
613 -static inline void xen_pgd_clear(pgd_t *pgd)
615 - xen_set_pgd(pgd, xen_make_pgd(0));
616 - xen_set_pgd(__user_pgd(pgd), xen_make_pgd(0));
618 +#define xen_pgd_clear(pgd) \
620 + pgd_t *__pgdp = (pgd); \
621 + PagePinned(virt_to_page(__pgdp)) \
622 + ? xen_l4_entry_update(__pgdp, 1, xen_make_pgd(0)) \
623 + : (void)(*__user_pgd(__pgdp) = *__pgdp = xen_make_pgd(0)); \
626 #define pte_same(a, b) ((a).pte == (b).pte)