]>
Commit | Line | Data |
---|---|---|
3610cce8 | 1 | /* |
a53c8fab | 2 | * Copyright IBM Corp. 2007, 2011 |
3610cce8 MS |
3 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> |
4 | */ | |
5 | ||
6 | #include <linux/sched.h> | |
7 | #include <linux/kernel.h> | |
8 | #include <linux/errno.h> | |
5a0e3ad6 | 9 | #include <linux/gfp.h> |
3610cce8 MS |
10 | #include <linux/mm.h> |
11 | #include <linux/swap.h> | |
12 | #include <linux/smp.h> | |
3610cce8 | 13 | #include <linux/spinlock.h> |
80217147 | 14 | #include <linux/rcupdate.h> |
e5992f2e | 15 | #include <linux/slab.h> |
b31288fa | 16 | #include <linux/swapops.h> |
0b46e0a3 | 17 | #include <linux/sysctl.h> |
3ac8e380 DD |
18 | #include <linux/ksm.h> |
19 | #include <linux/mman.h> | |
3610cce8 | 20 | |
3610cce8 MS |
21 | #include <asm/pgtable.h> |
22 | #include <asm/pgalloc.h> | |
23 | #include <asm/tlb.h> | |
24 | #include <asm/tlbflush.h> | |
6252d702 | 25 | #include <asm/mmu_context.h> |
3610cce8 | 26 | |
ebde765c MS |
27 | static inline pte_t ptep_flush_direct(struct mm_struct *mm, |
28 | unsigned long addr, pte_t *ptep) | |
29 | { | |
ebde765c MS |
30 | pte_t old; |
31 | ||
32 | old = *ptep; | |
33 | if (unlikely(pte_val(old) & _PAGE_INVALID)) | |
34 | return old; | |
64f31d58 MS |
35 | atomic_inc(&mm->context.flush_count); |
36 | if (MACHINE_HAS_TLB_LC && | |
ebde765c | 37 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) |
34eeaf37 | 38 | __ptep_ipte(addr, ptep, IPTE_LOCAL); |
ebde765c | 39 | else |
34eeaf37 | 40 | __ptep_ipte(addr, ptep, IPTE_GLOBAL); |
64f31d58 | 41 | atomic_dec(&mm->context.flush_count); |
ebde765c MS |
42 | return old; |
43 | } | |
44 | ||
45 | static inline pte_t ptep_flush_lazy(struct mm_struct *mm, | |
46 | unsigned long addr, pte_t *ptep) | |
47 | { | |
ebde765c MS |
48 | pte_t old; |
49 | ||
50 | old = *ptep; | |
51 | if (unlikely(pte_val(old) & _PAGE_INVALID)) | |
52 | return old; | |
64f31d58 MS |
53 | atomic_inc(&mm->context.flush_count); |
54 | if (cpumask_equal(&mm->context.cpu_attach_mask, | |
55 | cpumask_of(smp_processor_id()))) { | |
ebde765c MS |
56 | pte_val(*ptep) |= _PAGE_INVALID; |
57 | mm->context.flush_mm = 1; | |
58 | } else | |
34eeaf37 | 59 | __ptep_ipte(addr, ptep, IPTE_GLOBAL); |
64f31d58 | 60 | atomic_dec(&mm->context.flush_count); |
ebde765c MS |
61 | return old; |
62 | } | |
63 | ||
1e133ab2 MS |
64 | static inline pgste_t pgste_get_lock(pte_t *ptep) |
65 | { | |
66 | unsigned long new = 0; | |
67 | #ifdef CONFIG_PGSTE | |
68 | unsigned long old; | |
69 | ||
1e133ab2 MS |
70 | asm( |
71 | " lg %0,%2\n" | |
72 | "0: lgr %1,%0\n" | |
73 | " nihh %0,0xff7f\n" /* clear PCL bit in old */ | |
74 | " oihh %1,0x0080\n" /* set PCL bit in new */ | |
75 | " csg %0,%1,%2\n" | |
76 | " jl 0b\n" | |
77 | : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE]) | |
78 | : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory"); | |
79 | #endif | |
80 | return __pgste(new); | |
81 | } | |
82 | ||
83 | static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) | |
84 | { | |
85 | #ifdef CONFIG_PGSTE | |
86 | asm( | |
87 | " nihh %1,0xff7f\n" /* clear PCL bit */ | |
88 | " stg %1,%0\n" | |
89 | : "=Q" (ptep[PTRS_PER_PTE]) | |
90 | : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) | |
91 | : "cc", "memory"); | |
1e133ab2 MS |
92 | #endif |
93 | } | |
94 | ||
95 | static inline pgste_t pgste_get(pte_t *ptep) | |
96 | { | |
97 | unsigned long pgste = 0; | |
98 | #ifdef CONFIG_PGSTE | |
99 | pgste = *(unsigned long *)(ptep + PTRS_PER_PTE); | |
100 | #endif | |
101 | return __pgste(pgste); | |
102 | } | |
103 | ||
104 | static inline void pgste_set(pte_t *ptep, pgste_t pgste) | |
105 | { | |
106 | #ifdef CONFIG_PGSTE | |
107 | *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste; | |
108 | #endif | |
109 | } | |
110 | ||
ebde765c MS |
111 | static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste, |
112 | struct mm_struct *mm) | |
113 | { | |
114 | #ifdef CONFIG_PGSTE | |
115 | unsigned long address, bits, skey; | |
116 | ||
117 | if (!mm_use_skey(mm) || pte_val(pte) & _PAGE_INVALID) | |
118 | return pgste; | |
119 | address = pte_val(pte) & PAGE_MASK; | |
120 | skey = (unsigned long) page_get_storage_key(address); | |
121 | bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); | |
122 | /* Transfer page changed & referenced bit to guest bits in pgste */ | |
123 | pgste_val(pgste) |= bits << 48; /* GR bit & GC bit */ | |
124 | /* Copy page access key and fetch protection bit to pgste */ | |
125 | pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT); | |
126 | pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; | |
127 | #endif | |
128 | return pgste; | |
129 | ||
130 | } | |
131 | ||
132 | static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry, | |
133 | struct mm_struct *mm) | |
134 | { | |
135 | #ifdef CONFIG_PGSTE | |
136 | unsigned long address; | |
137 | unsigned long nkey; | |
138 | ||
139 | if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID) | |
140 | return; | |
141 | VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID)); | |
142 | address = pte_val(entry) & PAGE_MASK; | |
143 | /* | |
144 | * Set page access key and fetch protection bit from pgste. | |
145 | * The guest C/R information is still in the PGSTE, set real | |
146 | * key C/R to 0. | |
147 | */ | |
148 | nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; | |
149 | nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; | |
150 | page_set_storage_key(address, nkey, 0); | |
151 | #endif | |
152 | } | |
153 | ||
154 | static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry) | |
155 | { | |
156 | #ifdef CONFIG_PGSTE | |
157 | if ((pte_val(entry) & _PAGE_PRESENT) && | |
158 | (pte_val(entry) & _PAGE_WRITE) && | |
159 | !(pte_val(entry) & _PAGE_INVALID)) { | |
160 | if (!MACHINE_HAS_ESOP) { | |
161 | /* | |
162 | * Without enhanced suppression-on-protection force | |
163 | * the dirty bit on for all writable ptes. | |
164 | */ | |
165 | pte_val(entry) |= _PAGE_DIRTY; | |
166 | pte_val(entry) &= ~_PAGE_PROTECT; | |
167 | } | |
168 | if (!(pte_val(entry) & _PAGE_PROTECT)) | |
169 | /* This pte allows write access, set user-dirty */ | |
170 | pgste_val(pgste) |= PGSTE_UC_BIT; | |
171 | } | |
172 | #endif | |
173 | *ptep = entry; | |
174 | return pgste; | |
175 | } | |
176 | ||
b2d73b2a MS |
177 | static inline pgste_t pgste_pte_notify(struct mm_struct *mm, |
178 | unsigned long addr, | |
179 | pte_t *ptep, pgste_t pgste) | |
ebde765c MS |
180 | { |
181 | #ifdef CONFIG_PGSTE | |
4be130a0 MS |
182 | unsigned long bits; |
183 | ||
184 | bits = pgste_val(pgste) & (PGSTE_IN_BIT | PGSTE_VSIE_BIT); | |
185 | if (bits) { | |
186 | pgste_val(pgste) ^= bits; | |
187 | ptep_notify(mm, addr, ptep, bits); | |
ebde765c MS |
188 | } |
189 | #endif | |
190 | return pgste; | |
191 | } | |
192 | ||
ebde765c MS |
193 | static inline pgste_t ptep_xchg_start(struct mm_struct *mm, |
194 | unsigned long addr, pte_t *ptep) | |
195 | { | |
196 | pgste_t pgste = __pgste(0); | |
197 | ||
198 | if (mm_has_pgste(mm)) { | |
199 | pgste = pgste_get_lock(ptep); | |
b2d73b2a | 200 | pgste = pgste_pte_notify(mm, addr, ptep, pgste); |
ebde765c MS |
201 | } |
202 | return pgste; | |
203 | } | |
204 | ||
0d6da872 | 205 | static inline pte_t ptep_xchg_commit(struct mm_struct *mm, |
ebde765c MS |
206 | unsigned long addr, pte_t *ptep, |
207 | pgste_t pgste, pte_t old, pte_t new) | |
208 | { | |
209 | if (mm_has_pgste(mm)) { | |
210 | if (pte_val(old) & _PAGE_INVALID) | |
211 | pgste_set_key(ptep, pgste, new, mm); | |
212 | if (pte_val(new) & _PAGE_INVALID) { | |
213 | pgste = pgste_update_all(old, pgste, mm); | |
214 | if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) == | |
215 | _PGSTE_GPS_USAGE_UNUSED) | |
216 | pte_val(old) |= _PAGE_UNUSED; | |
217 | } | |
218 | pgste = pgste_set_pte(ptep, pgste, new); | |
219 | pgste_set_unlock(ptep, pgste); | |
220 | } else { | |
221 | *ptep = new; | |
222 | } | |
0d6da872 | 223 | return old; |
ebde765c MS |
224 | } |
225 | ||
226 | pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
227 | pte_t *ptep, pte_t new) | |
228 | { | |
229 | pgste_t pgste; | |
230 | pte_t old; | |
231 | ||
a9809407 | 232 | preempt_disable(); |
ebde765c MS |
233 | pgste = ptep_xchg_start(mm, addr, ptep); |
234 | old = ptep_flush_direct(mm, addr, ptep); | |
0d6da872 | 235 | old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); |
a9809407 | 236 | preempt_enable(); |
ebde765c MS |
237 | return old; |
238 | } | |
239 | EXPORT_SYMBOL(ptep_xchg_direct); | |
240 | ||
241 | pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, | |
242 | pte_t *ptep, pte_t new) | |
243 | { | |
244 | pgste_t pgste; | |
245 | pte_t old; | |
246 | ||
a9809407 | 247 | preempt_disable(); |
ebde765c MS |
248 | pgste = ptep_xchg_start(mm, addr, ptep); |
249 | old = ptep_flush_lazy(mm, addr, ptep); | |
0d6da872 | 250 | old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); |
a9809407 | 251 | preempt_enable(); |
ebde765c MS |
252 | return old; |
253 | } | |
254 | EXPORT_SYMBOL(ptep_xchg_lazy); | |
255 | ||
256 | pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, | |
257 | pte_t *ptep) | |
258 | { | |
259 | pgste_t pgste; | |
260 | pte_t old; | |
261 | ||
a9809407 | 262 | preempt_disable(); |
ebde765c MS |
263 | pgste = ptep_xchg_start(mm, addr, ptep); |
264 | old = ptep_flush_lazy(mm, addr, ptep); | |
265 | if (mm_has_pgste(mm)) { | |
266 | pgste = pgste_update_all(old, pgste, mm); | |
267 | pgste_set(ptep, pgste); | |
268 | } | |
269 | return old; | |
270 | } | |
271 | EXPORT_SYMBOL(ptep_modify_prot_start); | |
272 | ||
273 | void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, | |
274 | pte_t *ptep, pte_t pte) | |
275 | { | |
276 | pgste_t pgste; | |
277 | ||
57d7f939 MS |
278 | if (!MACHINE_HAS_NX) |
279 | pte_val(pte) &= ~_PAGE_NOEXEC; | |
ebde765c MS |
280 | if (mm_has_pgste(mm)) { |
281 | pgste = pgste_get(ptep); | |
282 | pgste_set_key(ptep, pgste, pte, mm); | |
283 | pgste = pgste_set_pte(ptep, pgste, pte); | |
284 | pgste_set_unlock(ptep, pgste); | |
285 | } else { | |
286 | *ptep = pte; | |
287 | } | |
a9809407 | 288 | preempt_enable(); |
ebde765c MS |
289 | } |
290 | EXPORT_SYMBOL(ptep_modify_prot_commit); | |
291 | ||
227be799 MS |
292 | static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, |
293 | unsigned long addr, pmd_t *pmdp) | |
294 | { | |
227be799 MS |
295 | pmd_t old; |
296 | ||
297 | old = *pmdp; | |
298 | if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) | |
299 | return old; | |
300 | if (!MACHINE_HAS_IDTE) { | |
301 | __pmdp_csp(pmdp); | |
302 | return old; | |
303 | } | |
64f31d58 MS |
304 | atomic_inc(&mm->context.flush_count); |
305 | if (MACHINE_HAS_TLB_LC && | |
227be799 | 306 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) |
47e4d851 | 307 | __pmdp_idte(addr, pmdp, IDTE_LOCAL); |
227be799 | 308 | else |
47e4d851 | 309 | __pmdp_idte(addr, pmdp, IDTE_GLOBAL); |
64f31d58 | 310 | atomic_dec(&mm->context.flush_count); |
227be799 MS |
311 | return old; |
312 | } | |
313 | ||
314 | static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, | |
315 | unsigned long addr, pmd_t *pmdp) | |
316 | { | |
227be799 MS |
317 | pmd_t old; |
318 | ||
319 | old = *pmdp; | |
320 | if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) | |
321 | return old; | |
64f31d58 MS |
322 | atomic_inc(&mm->context.flush_count); |
323 | if (cpumask_equal(&mm->context.cpu_attach_mask, | |
324 | cpumask_of(smp_processor_id()))) { | |
227be799 MS |
325 | pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID; |
326 | mm->context.flush_mm = 1; | |
327 | } else if (MACHINE_HAS_IDTE) | |
47e4d851 | 328 | __pmdp_idte(addr, pmdp, IDTE_GLOBAL); |
227be799 MS |
329 | else |
330 | __pmdp_csp(pmdp); | |
64f31d58 | 331 | atomic_dec(&mm->context.flush_count); |
227be799 MS |
332 | return old; |
333 | } | |
334 | ||
335 | pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
336 | pmd_t *pmdp, pmd_t new) | |
337 | { | |
338 | pmd_t old; | |
339 | ||
a9809407 | 340 | preempt_disable(); |
227be799 MS |
341 | old = pmdp_flush_direct(mm, addr, pmdp); |
342 | *pmdp = new; | |
a9809407 | 343 | preempt_enable(); |
227be799 MS |
344 | return old; |
345 | } | |
346 | EXPORT_SYMBOL(pmdp_xchg_direct); | |
347 | ||
348 | pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr, | |
349 | pmd_t *pmdp, pmd_t new) | |
350 | { | |
351 | pmd_t old; | |
352 | ||
a9809407 | 353 | preempt_disable(); |
227be799 MS |
354 | old = pmdp_flush_lazy(mm, addr, pmdp); |
355 | *pmdp = new; | |
a9809407 | 356 | preempt_enable(); |
227be799 MS |
357 | return old; |
358 | } | |
359 | EXPORT_SYMBOL(pmdp_xchg_lazy); | |
360 | ||
d08de8e2 GS |
361 | static inline pud_t pudp_flush_direct(struct mm_struct *mm, |
362 | unsigned long addr, pud_t *pudp) | |
363 | { | |
364 | pud_t old; | |
365 | ||
366 | old = *pudp; | |
367 | if (pud_val(old) & _REGION_ENTRY_INVALID) | |
368 | return old; | |
369 | if (!MACHINE_HAS_IDTE) { | |
370 | /* | |
371 | * Invalid bit position is the same for pmd and pud, so we can | |
372 | * re-use _pmd_csp() here | |
373 | */ | |
374 | __pmdp_csp((pmd_t *) pudp); | |
375 | return old; | |
376 | } | |
377 | atomic_inc(&mm->context.flush_count); | |
378 | if (MACHINE_HAS_TLB_LC && | |
379 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) | |
47e4d851 | 380 | __pudp_idte(addr, pudp, IDTE_LOCAL); |
d08de8e2 | 381 | else |
47e4d851 | 382 | __pudp_idte(addr, pudp, IDTE_GLOBAL); |
d08de8e2 GS |
383 | atomic_dec(&mm->context.flush_count); |
384 | return old; | |
385 | } | |
386 | ||
387 | pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
388 | pud_t *pudp, pud_t new) | |
389 | { | |
390 | pud_t old; | |
391 | ||
392 | preempt_disable(); | |
393 | old = pudp_flush_direct(mm, addr, pudp); | |
394 | *pudp = new; | |
395 | preempt_enable(); | |
396 | return old; | |
397 | } | |
398 | EXPORT_SYMBOL(pudp_xchg_direct); | |
399 | ||
75077afb | 400 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
6b0b50b0 AK |
401 | void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, |
402 | pgtable_t pgtable) | |
9501d09f GS |
403 | { |
404 | struct list_head *lh = (struct list_head *) pgtable; | |
405 | ||
ec66ad66 | 406 | assert_spin_locked(pmd_lockptr(mm, pmdp)); |
9501d09f GS |
407 | |
408 | /* FIFO */ | |
c389a250 | 409 | if (!pmd_huge_pte(mm, pmdp)) |
9501d09f GS |
410 | INIT_LIST_HEAD(lh); |
411 | else | |
c389a250 KS |
412 | list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); |
413 | pmd_huge_pte(mm, pmdp) = pgtable; | |
9501d09f GS |
414 | } |
415 | ||
6b0b50b0 | 416 | pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) |
9501d09f GS |
417 | { |
418 | struct list_head *lh; | |
419 | pgtable_t pgtable; | |
420 | pte_t *ptep; | |
421 | ||
ec66ad66 | 422 | assert_spin_locked(pmd_lockptr(mm, pmdp)); |
9501d09f GS |
423 | |
424 | /* FIFO */ | |
c389a250 | 425 | pgtable = pmd_huge_pte(mm, pmdp); |
9501d09f GS |
426 | lh = (struct list_head *) pgtable; |
427 | if (list_empty(lh)) | |
c389a250 | 428 | pmd_huge_pte(mm, pmdp) = NULL; |
9501d09f | 429 | else { |
c389a250 | 430 | pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; |
9501d09f GS |
431 | list_del(lh); |
432 | } | |
433 | ptep = (pte_t *) pgtable; | |
e5098611 | 434 | pte_val(*ptep) = _PAGE_INVALID; |
9501d09f | 435 | ptep++; |
e5098611 | 436 | pte_val(*ptep) = _PAGE_INVALID; |
9501d09f GS |
437 | return pgtable; |
438 | } | |
75077afb | 439 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
1e133ab2 MS |
440 | |
441 | #ifdef CONFIG_PGSTE | |
442 | void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, | |
443 | pte_t *ptep, pte_t entry) | |
444 | { | |
445 | pgste_t pgste; | |
446 | ||
447 | /* the mm_has_pgste() check is done in set_pte_at() */ | |
a9809407 | 448 | preempt_disable(); |
1e133ab2 MS |
449 | pgste = pgste_get_lock(ptep); |
450 | pgste_val(pgste) &= ~_PGSTE_GPS_ZERO; | |
451 | pgste_set_key(ptep, pgste, entry, mm); | |
452 | pgste = pgste_set_pte(ptep, pgste, entry); | |
453 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 454 | preempt_enable(); |
1e133ab2 MS |
455 | } |
456 | ||
457 | void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
458 | { | |
459 | pgste_t pgste; | |
460 | ||
a9809407 | 461 | preempt_disable(); |
1e133ab2 MS |
462 | pgste = pgste_get_lock(ptep); |
463 | pgste_val(pgste) |= PGSTE_IN_BIT; | |
464 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 465 | preempt_enable(); |
1e133ab2 MS |
466 | } |
467 | ||
b2d73b2a MS |
468 | /** |
469 | * ptep_force_prot - change access rights of a locked pte | |
470 | * @mm: pointer to the process mm_struct | |
471 | * @addr: virtual address in the guest address space | |
472 | * @ptep: pointer to the page table entry | |
473 | * @prot: indicates guest access rights: PROT_NONE, PROT_READ or PROT_WRITE | |
4be130a0 | 474 | * @bit: pgste bit to set (e.g. for notification) |
b2d73b2a MS |
475 | * |
476 | * Returns 0 if the access rights were changed and -EAGAIN if the current | |
477 | * and requested access rights are incompatible. | |
478 | */ | |
479 | int ptep_force_prot(struct mm_struct *mm, unsigned long addr, | |
4be130a0 | 480 | pte_t *ptep, int prot, unsigned long bit) |
b2d73b2a MS |
481 | { |
482 | pte_t entry; | |
483 | pgste_t pgste; | |
484 | int pte_i, pte_p; | |
485 | ||
486 | pgste = pgste_get_lock(ptep); | |
487 | entry = *ptep; | |
488 | /* Check pte entry after all locks have been acquired */ | |
489 | pte_i = pte_val(entry) & _PAGE_INVALID; | |
490 | pte_p = pte_val(entry) & _PAGE_PROTECT; | |
491 | if ((pte_i && (prot != PROT_NONE)) || | |
492 | (pte_p && (prot & PROT_WRITE))) { | |
493 | pgste_set_unlock(ptep, pgste); | |
494 | return -EAGAIN; | |
495 | } | |
4be130a0 | 496 | /* Change access rights and set pgste bit */ |
b2d73b2a MS |
497 | if (prot == PROT_NONE && !pte_i) { |
498 | ptep_flush_direct(mm, addr, ptep); | |
499 | pgste = pgste_update_all(entry, pgste, mm); | |
500 | pte_val(entry) |= _PAGE_INVALID; | |
501 | } | |
502 | if (prot == PROT_READ && !pte_p) { | |
503 | ptep_flush_direct(mm, addr, ptep); | |
504 | pte_val(entry) &= ~_PAGE_INVALID; | |
505 | pte_val(entry) |= _PAGE_PROTECT; | |
506 | } | |
4be130a0 | 507 | pgste_val(pgste) |= bit; |
b2d73b2a MS |
508 | pgste = pgste_set_pte(ptep, pgste, entry); |
509 | pgste_set_unlock(ptep, pgste); | |
510 | return 0; | |
511 | } | |
512 | ||
4be130a0 | 513 | int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, |
a9d23e71 | 514 | pte_t *sptep, pte_t *tptep, pte_t pte) |
4be130a0 MS |
515 | { |
516 | pgste_t spgste, tpgste; | |
517 | pte_t spte, tpte; | |
518 | int rc = -EAGAIN; | |
519 | ||
a9d23e71 DH |
520 | if (!(pte_val(*tptep) & _PAGE_INVALID)) |
521 | return 0; /* already shadowed */ | |
4be130a0 MS |
522 | spgste = pgste_get_lock(sptep); |
523 | spte = *sptep; | |
524 | if (!(pte_val(spte) & _PAGE_INVALID) && | |
a9d23e71 DH |
525 | !((pte_val(spte) & _PAGE_PROTECT) && |
526 | !(pte_val(pte) & _PAGE_PROTECT))) { | |
4be130a0 MS |
527 | pgste_val(spgste) |= PGSTE_VSIE_BIT; |
528 | tpgste = pgste_get_lock(tptep); | |
529 | pte_val(tpte) = (pte_val(spte) & PAGE_MASK) | | |
a9d23e71 | 530 | (pte_val(pte) & _PAGE_PROTECT); |
4be130a0 MS |
531 | /* don't touch the storage key - it belongs to parent pgste */ |
532 | tpgste = pgste_set_pte(tptep, tpgste, tpte); | |
533 | pgste_set_unlock(tptep, tpgste); | |
a9d23e71 | 534 | rc = 1; |
4be130a0 MS |
535 | } |
536 | pgste_set_unlock(sptep, spgste); | |
537 | return rc; | |
538 | } | |
539 | ||
540 | void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep) | |
541 | { | |
542 | pgste_t pgste; | |
543 | ||
544 | pgste = pgste_get_lock(ptep); | |
545 | /* notifier is called by the caller */ | |
546 | ptep_flush_direct(mm, saddr, ptep); | |
547 | /* don't touch the storage key - it belongs to parent pgste */ | |
548 | pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID)); | |
549 | pgste_set_unlock(ptep, pgste); | |
550 | } | |
551 | ||
1e133ab2 MS |
552 | static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) |
553 | { | |
554 | if (!non_swap_entry(entry)) | |
555 | dec_mm_counter(mm, MM_SWAPENTS); | |
556 | else if (is_migration_entry(entry)) { | |
557 | struct page *page = migration_entry_to_page(entry); | |
558 | ||
559 | dec_mm_counter(mm, mm_counter(page)); | |
560 | } | |
561 | free_swap_and_cache(entry); | |
562 | } | |
563 | ||
564 | void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, | |
565 | pte_t *ptep, int reset) | |
566 | { | |
567 | unsigned long pgstev; | |
568 | pgste_t pgste; | |
569 | pte_t pte; | |
570 | ||
571 | /* Zap unused and logically-zero pages */ | |
a9809407 | 572 | preempt_disable(); |
1e133ab2 MS |
573 | pgste = pgste_get_lock(ptep); |
574 | pgstev = pgste_val(pgste); | |
575 | pte = *ptep; | |
1c343f7b | 576 | if (!reset && pte_swap(pte) && |
1e133ab2 MS |
577 | ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED || |
578 | (pgstev & _PGSTE_GPS_ZERO))) { | |
579 | ptep_zap_swap_entry(mm, pte_to_swp_entry(pte)); | |
580 | pte_clear(mm, addr, ptep); | |
581 | } | |
582 | if (reset) | |
583 | pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK; | |
584 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 585 | preempt_enable(); |
1e133ab2 MS |
586 | } |
587 | ||
588 | void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
589 | { | |
590 | unsigned long ptev; | |
591 | pgste_t pgste; | |
592 | ||
593 | /* Clear storage key */ | |
a9809407 | 594 | preempt_disable(); |
1e133ab2 MS |
595 | pgste = pgste_get_lock(ptep); |
596 | pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT | | |
597 | PGSTE_GR_BIT | PGSTE_GC_BIT); | |
598 | ptev = pte_val(*ptep); | |
599 | if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE)) | |
600 | page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1); | |
601 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 602 | preempt_enable(); |
1e133ab2 MS |
603 | } |
604 | ||
605 | /* | |
606 | * Test and reset if a guest page is dirty | |
607 | */ | |
608 | bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr) | |
609 | { | |
610 | spinlock_t *ptl; | |
2e4d8800 JF |
611 | pgd_t *pgd; |
612 | pud_t *pud; | |
613 | pmd_t *pmd; | |
1e133ab2 MS |
614 | pgste_t pgste; |
615 | pte_t *ptep; | |
616 | pte_t pte; | |
617 | bool dirty; | |
618 | ||
2e4d8800 JF |
619 | pgd = pgd_offset(mm, addr); |
620 | pud = pud_alloc(mm, pgd, addr); | |
621 | if (!pud) | |
622 | return false; | |
623 | pmd = pmd_alloc(mm, pud, addr); | |
624 | if (!pmd) | |
625 | return false; | |
626 | /* We can't run guests backed by huge pages, but userspace can | |
627 | * still set them up and then try to migrate them without any | |
628 | * migration support. | |
629 | */ | |
630 | if (pmd_large(*pmd)) | |
631 | return true; | |
632 | ||
633 | ptep = pte_alloc_map_lock(mm, pmd, addr, &ptl); | |
1e133ab2 MS |
634 | if (unlikely(!ptep)) |
635 | return false; | |
636 | ||
637 | pgste = pgste_get_lock(ptep); | |
638 | dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT); | |
639 | pgste_val(pgste) &= ~PGSTE_UC_BIT; | |
640 | pte = *ptep; | |
641 | if (dirty && (pte_val(pte) & _PAGE_PRESENT)) { | |
b2d73b2a | 642 | pgste = pgste_pte_notify(mm, addr, ptep, pgste); |
34eeaf37 | 643 | __ptep_ipte(addr, ptep, IPTE_GLOBAL); |
1e133ab2 MS |
644 | if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE)) |
645 | pte_val(pte) |= _PAGE_PROTECT; | |
646 | else | |
647 | pte_val(pte) |= _PAGE_INVALID; | |
648 | *ptep = pte; | |
649 | } | |
650 | pgste_set_unlock(ptep, pgste); | |
651 | ||
652 | spin_unlock(ptl); | |
653 | return dirty; | |
654 | } | |
655 | EXPORT_SYMBOL_GPL(test_and_clear_guest_dirty); | |
656 | ||
657 | int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, | |
658 | unsigned char key, bool nq) | |
659 | { | |
660 | unsigned long keyul; | |
661 | spinlock_t *ptl; | |
662 | pgste_t old, new; | |
663 | pte_t *ptep; | |
664 | ||
1e133ab2 | 665 | ptep = get_locked_pte(mm, addr, &ptl); |
d3ed1cee | 666 | if (unlikely(!ptep)) |
1e133ab2 | 667 | return -EFAULT; |
1e133ab2 MS |
668 | |
669 | new = old = pgste_get_lock(ptep); | |
670 | pgste_val(new) &= ~(PGSTE_GR_BIT | PGSTE_GC_BIT | | |
671 | PGSTE_ACC_BITS | PGSTE_FP_BIT); | |
672 | keyul = (unsigned long) key; | |
673 | pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48; | |
674 | pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; | |
675 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { | |
676 | unsigned long address, bits, skey; | |
677 | ||
678 | address = pte_val(*ptep) & PAGE_MASK; | |
679 | skey = (unsigned long) page_get_storage_key(address); | |
680 | bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); | |
681 | skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT); | |
682 | /* Set storage key ACC and FP */ | |
683 | page_set_storage_key(address, skey, !nq); | |
684 | /* Merge host changed & referenced into pgste */ | |
685 | pgste_val(new) |= bits << 52; | |
686 | } | |
687 | /* changing the guest storage key is considered a change of the page */ | |
688 | if ((pgste_val(new) ^ pgste_val(old)) & | |
689 | (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT)) | |
690 | pgste_val(new) |= PGSTE_UC_BIT; | |
691 | ||
692 | pgste_set_unlock(ptep, new); | |
693 | pte_unmap_unlock(ptep, ptl); | |
1e133ab2 MS |
694 | return 0; |
695 | } | |
696 | EXPORT_SYMBOL(set_guest_storage_key); | |
697 | ||
1824c723 DH |
698 | /** |
699 | * Conditionally set a guest storage key (handling csske). | |
700 | * oldkey will be updated when either mr or mc is set and a pointer is given. | |
701 | * | |
702 | * Returns 0 if a guests storage key update wasn't necessary, 1 if the guest | |
703 | * storage key was updated and -EFAULT on access errors. | |
704 | */ | |
705 | int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr, | |
706 | unsigned char key, unsigned char *oldkey, | |
707 | bool nq, bool mr, bool mc) | |
708 | { | |
709 | unsigned char tmp, mask = _PAGE_ACC_BITS | _PAGE_FP_BIT; | |
710 | int rc; | |
711 | ||
712 | /* we can drop the pgste lock between getting and setting the key */ | |
713 | if (mr | mc) { | |
714 | rc = get_guest_storage_key(current->mm, addr, &tmp); | |
715 | if (rc) | |
716 | return rc; | |
717 | if (oldkey) | |
718 | *oldkey = tmp; | |
719 | if (!mr) | |
720 | mask |= _PAGE_REFERENCED; | |
721 | if (!mc) | |
722 | mask |= _PAGE_CHANGED; | |
723 | if (!((tmp ^ key) & mask)) | |
724 | return 0; | |
725 | } | |
726 | rc = set_guest_storage_key(current->mm, addr, key, nq); | |
727 | return rc < 0 ? rc : 1; | |
728 | } | |
729 | EXPORT_SYMBOL(cond_set_guest_storage_key); | |
730 | ||
a7e19ab5 DH |
731 | /** |
732 | * Reset a guest reference bit (rrbe), returning the reference and changed bit. | |
733 | * | |
734 | * Returns < 0 in case of error, otherwise the cc to be reported to the guest. | |
735 | */ | |
736 | int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) | |
1e133ab2 | 737 | { |
1e133ab2 | 738 | spinlock_t *ptl; |
a7e19ab5 | 739 | pgste_t old, new; |
1e133ab2 | 740 | pte_t *ptep; |
a7e19ab5 | 741 | int cc = 0; |
1e133ab2 | 742 | |
1e133ab2 | 743 | ptep = get_locked_pte(mm, addr, &ptl); |
a7e19ab5 | 744 | if (unlikely(!ptep)) |
1e133ab2 | 745 | return -EFAULT; |
1e133ab2 | 746 | |
a7e19ab5 DH |
747 | new = old = pgste_get_lock(ptep); |
748 | /* Reset guest reference bit only */ | |
749 | pgste_val(new) &= ~PGSTE_GR_BIT; | |
1e133ab2 | 750 | |
a7e19ab5 DH |
751 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { |
752 | cc = page_reset_referenced(pte_val(*ptep) & PAGE_MASK); | |
753 | /* Merge real referenced bit into host-set */ | |
754 | pgste_val(new) |= ((unsigned long) cc << 53) & PGSTE_HR_BIT; | |
1e133ab2 | 755 | } |
a7e19ab5 DH |
756 | /* Reflect guest's logical view, not physical */ |
757 | cc |= (pgste_val(old) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 49; | |
758 | /* Changing the guest storage key is considered a change of the page */ | |
759 | if ((pgste_val(new) ^ pgste_val(old)) & PGSTE_GR_BIT) | |
760 | pgste_val(new) |= PGSTE_UC_BIT; | |
761 | ||
762 | pgste_set_unlock(ptep, new); | |
763 | pte_unmap_unlock(ptep, ptl); | |
4bead2a4 | 764 | return cc; |
a7e19ab5 DH |
765 | } |
766 | EXPORT_SYMBOL(reset_guest_reference_bit); | |
767 | ||
154c8c19 DH |
768 | int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, |
769 | unsigned char *key) | |
1e133ab2 | 770 | { |
1e133ab2 MS |
771 | spinlock_t *ptl; |
772 | pgste_t pgste; | |
773 | pte_t *ptep; | |
1e133ab2 | 774 | |
1e133ab2 | 775 | ptep = get_locked_pte(mm, addr, &ptl); |
d3ed1cee | 776 | if (unlikely(!ptep)) |
1e133ab2 | 777 | return -EFAULT; |
1e133ab2 | 778 | |
d3ed1cee | 779 | pgste = pgste_get_lock(ptep); |
154c8c19 | 780 | *key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; |
8d6037a7 | 781 | if (!(pte_val(*ptep) & _PAGE_INVALID)) |
154c8c19 | 782 | *key = page_get_storage_key(pte_val(*ptep) & PAGE_MASK); |
8d6037a7 | 783 | /* Reflect guest's logical view, not physical */ |
154c8c19 | 784 | *key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; |
1e133ab2 MS |
785 | pgste_set_unlock(ptep, pgste); |
786 | pte_unmap_unlock(ptep, ptl); | |
154c8c19 | 787 | return 0; |
1e133ab2 MS |
788 | } |
789 | EXPORT_SYMBOL(get_guest_storage_key); | |
790 | #endif |