1 // SPDX-License-Identifier: GPL-2.0
3 * High memory handling common code and variables.
5 * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de
6 * Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de
9 * Redesigned the x86 32-bit VM architecture to deal with
10 * 64-bit physical space. With current x86 CPUs this
11 * means up to 64 Gigabytes physical RAM.
13 * Rewrote high memory support to move the page cache into
14 * high memory. Implemented permanent (schedulable) kmaps
15 * based on Linus' idea.
17 * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
21 #include <linux/export.h>
22 #include <linux/swap.h>
23 #include <linux/bio.h>
24 #include <linux/pagemap.h>
25 #include <linux/mempool.h>
26 #include <linux/init.h>
27 #include <linux/hash.h>
28 #include <linux/highmem.h>
29 #include <linux/kgdb.h>
30 #include <asm/tlbflush.h>
31 #include <linux/vmalloc.h>
34 * Virtual_count is not a pure "count".
35 * 0 means that it is not mapped, and has not been mapped
36 * since a TLB flush - it is usable.
37 * 1 means that there are no users, but it has been mapped
38 * since the last TLB flush - so we can't use it.
39 * n means that there are (n-1) current users of it.
44 * Architecture with aliasing data cache may define the following family of
45 * helper functions in its asm/highmem.h to control cache color of virtual
46 * addresses where physical memory pages are mapped by kmap.
48 #ifndef get_pkmap_color
51 * Determine color of virtual address where the page should be mapped.
53 static inline unsigned int get_pkmap_color(struct page
*page
)
57 #define get_pkmap_color get_pkmap_color
60 * Get next index for mapping inside PKMAP region for page with given color.
62 static inline unsigned int get_next_pkmap_nr(unsigned int color
)
64 static unsigned int last_pkmap_nr
;
66 last_pkmap_nr
= (last_pkmap_nr
+ 1) & LAST_PKMAP_MASK
;
71 * Determine if page index inside PKMAP region (pkmap_nr) of given color
72 * has wrapped around PKMAP region end. When this happens an attempt to
73 * flush all unused PKMAP slots is made.
75 static inline int no_more_pkmaps(unsigned int pkmap_nr
, unsigned int color
)
81 * Get the number of PKMAP entries of the given color. If no free slot is
82 * found after checking that many entries, kmap will sleep waiting for
83 * someone to call kunmap and free PKMAP slot.
85 static inline int get_pkmap_entries_count(unsigned int color
)
91 * Get head of a wait queue for PKMAP entries of the given color.
92 * Wait queues for different mapping colors should be independent to avoid
93 * unnecessary wakeups caused by freeing of slots of other colors.
95 static inline wait_queue_head_t
*get_pkmap_wait_queue_head(unsigned int color
)
97 static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait
);
99 return &pkmap_map_wait
;
103 atomic_long_t _totalhigh_pages __read_mostly
;
104 EXPORT_SYMBOL(_totalhigh_pages
);
106 unsigned int __nr_free_highpages(void)
109 unsigned int pages
= 0;
111 for_each_populated_zone(zone
) {
112 if (is_highmem(zone
))
113 pages
+= zone_page_state(zone
, NR_FREE_PAGES
);
119 static int pkmap_count
[LAST_PKMAP
];
120 static __cacheline_aligned_in_smp
DEFINE_SPINLOCK(kmap_lock
);
122 pte_t
*pkmap_page_table
;
125 * Most architectures have no use for kmap_high_get(), so let's abstract
126 * the disabling of IRQ out of the locking in that case to save on a
127 * potential useless overhead.
129 #ifdef ARCH_NEEDS_KMAP_HIGH_GET
130 #define lock_kmap() spin_lock_irq(&kmap_lock)
131 #define unlock_kmap() spin_unlock_irq(&kmap_lock)
132 #define lock_kmap_any(flags) spin_lock_irqsave(&kmap_lock, flags)
133 #define unlock_kmap_any(flags) spin_unlock_irqrestore(&kmap_lock, flags)
135 #define lock_kmap() spin_lock(&kmap_lock)
136 #define unlock_kmap() spin_unlock(&kmap_lock)
137 #define lock_kmap_any(flags) \
138 do { spin_lock(&kmap_lock); (void)(flags); } while (0)
139 #define unlock_kmap_any(flags) \
140 do { spin_unlock(&kmap_lock); (void)(flags); } while (0)
143 struct page
*__kmap_to_page(void *vaddr
)
145 unsigned long addr
= (unsigned long)vaddr
;
147 if (addr
>= PKMAP_ADDR(0) && addr
< PKMAP_ADDR(LAST_PKMAP
)) {
148 int i
= PKMAP_NR(addr
);
150 return pte_page(pkmap_page_table
[i
]);
153 return virt_to_page(vaddr
);
155 EXPORT_SYMBOL(__kmap_to_page
);
157 static void flush_all_zero_pkmaps(void)
164 for (i
= 0; i
< LAST_PKMAP
; i
++) {
168 * zero means we don't have anything to do,
169 * >1 means that it is still in use. Only
170 * a count of 1 means that it is free but
171 * needs to be unmapped
173 if (pkmap_count
[i
] != 1)
178 BUG_ON(pte_none(pkmap_page_table
[i
]));
181 * Don't need an atomic fetch-and-clear op here;
182 * no-one has the page mapped, and cannot get at
183 * its virtual address (and hence PTE) without first
184 * getting the kmap_lock (which is held here).
185 * So no dangers, even with speculative execution.
187 page
= pte_page(pkmap_page_table
[i
]);
188 pte_clear(&init_mm
, PKMAP_ADDR(i
), &pkmap_page_table
[i
]);
190 set_page_address(page
, NULL
);
194 flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP
));
197 void __kmap_flush_unused(void)
200 flush_all_zero_pkmaps();
204 static inline unsigned long map_new_virtual(struct page
*page
)
208 unsigned int last_pkmap_nr
;
209 unsigned int color
= get_pkmap_color(page
);
212 count
= get_pkmap_entries_count(color
);
213 /* Find an empty entry */
215 last_pkmap_nr
= get_next_pkmap_nr(color
);
216 if (no_more_pkmaps(last_pkmap_nr
, color
)) {
217 flush_all_zero_pkmaps();
218 count
= get_pkmap_entries_count(color
);
220 if (!pkmap_count
[last_pkmap_nr
])
221 break; /* Found a usable entry */
226 * Sleep for somebody else to unmap their entries
229 DECLARE_WAITQUEUE(wait
, current
);
230 wait_queue_head_t
*pkmap_map_wait
=
231 get_pkmap_wait_queue_head(color
);
233 __set_current_state(TASK_UNINTERRUPTIBLE
);
234 add_wait_queue(pkmap_map_wait
, &wait
);
237 remove_wait_queue(pkmap_map_wait
, &wait
);
240 /* Somebody else might have mapped it while we slept */
241 if (page_address(page
))
242 return (unsigned long)page_address(page
);
248 vaddr
= PKMAP_ADDR(last_pkmap_nr
);
249 set_pte_at(&init_mm
, vaddr
,
250 &(pkmap_page_table
[last_pkmap_nr
]), mk_pte(page
, kmap_prot
));
252 pkmap_count
[last_pkmap_nr
] = 1;
253 set_page_address(page
, (void *)vaddr
);
259 * kmap_high - map a highmem page into memory
260 * @page: &struct page to map
262 * Returns the page's virtual memory address.
264 * We cannot call this from interrupts, as it may block.
266 void *kmap_high(struct page
*page
)
271 * For highmem pages, we can't trust "virtual" until
272 * after we have the lock.
275 vaddr
= (unsigned long)page_address(page
);
277 vaddr
= map_new_virtual(page
);
278 pkmap_count
[PKMAP_NR(vaddr
)]++;
279 BUG_ON(pkmap_count
[PKMAP_NR(vaddr
)] < 2);
281 return (void *) vaddr
;
283 EXPORT_SYMBOL(kmap_high
);
285 #ifdef ARCH_NEEDS_KMAP_HIGH_GET
287 * kmap_high_get - pin a highmem page into memory
288 * @page: &struct page to pin
290 * Returns the page's current virtual memory address, or NULL if no mapping
291 * exists. If and only if a non null address is returned then a
292 * matching call to kunmap_high() is necessary.
294 * This can be called from any context.
296 void *kmap_high_get(struct page
*page
)
298 unsigned long vaddr
, flags
;
300 lock_kmap_any(flags
);
301 vaddr
= (unsigned long)page_address(page
);
303 BUG_ON(pkmap_count
[PKMAP_NR(vaddr
)] < 1);
304 pkmap_count
[PKMAP_NR(vaddr
)]++;
306 unlock_kmap_any(flags
);
307 return (void *) vaddr
;
312 * kunmap_high - unmap a highmem page into memory
313 * @page: &struct page to unmap
315 * If ARCH_NEEDS_KMAP_HIGH_GET is not defined then this may be called
316 * only from user context.
318 void kunmap_high(struct page
*page
)
324 unsigned int color
= get_pkmap_color(page
);
325 wait_queue_head_t
*pkmap_map_wait
;
327 lock_kmap_any(flags
);
328 vaddr
= (unsigned long)page_address(page
);
330 nr
= PKMAP_NR(vaddr
);
333 * A count must never go down to zero
334 * without a TLB flush!
337 switch (--pkmap_count
[nr
]) {
342 * Avoid an unnecessary wake_up() function call.
343 * The common case is pkmap_count[] == 1, but
345 * The tasks queued in the wait-queue are guarded
346 * by both the lock in the wait-queue-head and by
347 * the kmap_lock. As the kmap_lock is held here,
348 * no need for the wait-queue-head's lock. Simply
349 * test if the queue is empty.
351 pkmap_map_wait
= get_pkmap_wait_queue_head(color
);
352 need_wakeup
= waitqueue_active(pkmap_map_wait
);
354 unlock_kmap_any(flags
);
356 /* do wake-up, if needed, race-free outside of the spin lock */
358 wake_up(pkmap_map_wait
);
360 EXPORT_SYMBOL(kunmap_high
);
362 void zero_user_segments(struct page
*page
, unsigned start1
, unsigned end1
,
363 unsigned start2
, unsigned end2
)
367 BUG_ON(end1
> page_size(page
) || end2
> page_size(page
));
374 for (i
= 0; i
< compound_nr(page
); i
++) {
377 if (start1
>= PAGE_SIZE
) {
381 unsigned this_end
= min_t(unsigned, end1
, PAGE_SIZE
);
384 kaddr
= kmap_local_page(page
+ i
);
385 memset(kaddr
+ start1
, 0, this_end
- start1
);
391 if (start2
>= PAGE_SIZE
) {
395 unsigned this_end
= min_t(unsigned, end2
, PAGE_SIZE
);
399 kaddr
= kmap_local_page(page
+ i
);
400 memset(kaddr
+ start2
, 0, this_end
- start2
);
408 flush_dcache_page(page
+ i
);
415 BUG_ON((start1
| start2
| end1
| end2
) != 0);
417 EXPORT_SYMBOL(zero_user_segments
);
418 #endif /* CONFIG_HIGHMEM */
420 #ifdef CONFIG_KMAP_LOCAL
422 #include <asm/kmap_size.h>
425 * With DEBUG_KMAP_LOCAL the stack depth is doubled and every second
426 * slot is unused which acts as a guard page
428 #ifdef CONFIG_DEBUG_KMAP_LOCAL
434 static inline int kmap_local_idx_push(void)
436 WARN_ON_ONCE(in_hardirq() && !irqs_disabled());
437 current
->kmap_ctrl
.idx
+= KM_INCR
;
438 BUG_ON(current
->kmap_ctrl
.idx
>= KM_MAX_IDX
);
439 return current
->kmap_ctrl
.idx
- 1;
442 static inline int kmap_local_idx(void)
444 return current
->kmap_ctrl
.idx
- 1;
447 static inline void kmap_local_idx_pop(void)
449 current
->kmap_ctrl
.idx
-= KM_INCR
;
450 BUG_ON(current
->kmap_ctrl
.idx
< 0);
453 #ifndef arch_kmap_local_post_map
454 # define arch_kmap_local_post_map(vaddr, pteval) do { } while (0)
457 #ifndef arch_kmap_local_pre_unmap
458 # define arch_kmap_local_pre_unmap(vaddr) do { } while (0)
461 #ifndef arch_kmap_local_post_unmap
462 # define arch_kmap_local_post_unmap(vaddr) do { } while (0)
465 #ifndef arch_kmap_local_map_idx
466 #define arch_kmap_local_map_idx(idx, pfn) kmap_local_calc_idx(idx)
469 #ifndef arch_kmap_local_unmap_idx
470 #define arch_kmap_local_unmap_idx(idx, vaddr) kmap_local_calc_idx(idx)
473 #ifndef arch_kmap_local_high_get
474 static inline void *arch_kmap_local_high_get(struct page
*page
)
480 #ifndef arch_kmap_local_set_pte
481 #define arch_kmap_local_set_pte(mm, vaddr, ptep, ptev) \
482 set_pte_at(mm, vaddr, ptep, ptev)
485 /* Unmap a local mapping which was obtained by kmap_high_get() */
486 static inline bool kmap_high_unmap_local(unsigned long vaddr
)
488 #ifdef ARCH_NEEDS_KMAP_HIGH_GET
489 if (vaddr
>= PKMAP_ADDR(0) && vaddr
< PKMAP_ADDR(LAST_PKMAP
)) {
490 kunmap_high(pte_page(pkmap_page_table
[PKMAP_NR(vaddr
)]));
497 static inline int kmap_local_calc_idx(int idx
)
499 return idx
+ KM_MAX_IDX
* smp_processor_id();
502 static pte_t
*__kmap_pte
;
504 static pte_t
*kmap_get_pte(unsigned long vaddr
, int idx
)
506 if (IS_ENABLED(CONFIG_KMAP_LOCAL_NON_LINEAR_PTE_ARRAY
))
508 * Set by the arch if __kmap_pte[-idx] does not produce
511 return virt_to_kpte(vaddr
);
513 __kmap_pte
= virt_to_kpte(__fix_to_virt(FIX_KMAP_BEGIN
));
514 return &__kmap_pte
[-idx
];
517 void *__kmap_local_pfn_prot(unsigned long pfn
, pgprot_t prot
)
519 pte_t pteval
, *kmap_pte
;
524 * Disable migration so resulting virtual address is stable
529 idx
= arch_kmap_local_map_idx(kmap_local_idx_push(), pfn
);
530 vaddr
= __fix_to_virt(FIX_KMAP_BEGIN
+ idx
);
531 kmap_pte
= kmap_get_pte(vaddr
, idx
);
532 BUG_ON(!pte_none(*kmap_pte
));
533 pteval
= pfn_pte(pfn
, prot
);
534 arch_kmap_local_set_pte(&init_mm
, vaddr
, kmap_pte
, pteval
);
535 arch_kmap_local_post_map(vaddr
, pteval
);
536 current
->kmap_ctrl
.pteval
[kmap_local_idx()] = pteval
;
539 return (void *)vaddr
;
541 EXPORT_SYMBOL_GPL(__kmap_local_pfn_prot
);
543 void *__kmap_local_page_prot(struct page
*page
, pgprot_t prot
)
548 * To broaden the usage of the actual kmap_local() machinery always map
549 * pages when debugging is enabled and the architecture has no problems
550 * with alias mappings.
552 if (!IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP
) && !PageHighMem(page
))
553 return page_address(page
);
555 /* Try kmap_high_get() if architecture has it enabled */
556 kmap
= arch_kmap_local_high_get(page
);
560 return __kmap_local_pfn_prot(page_to_pfn(page
), prot
);
562 EXPORT_SYMBOL(__kmap_local_page_prot
);
564 void kunmap_local_indexed(const void *vaddr
)
566 unsigned long addr
= (unsigned long) vaddr
& PAGE_MASK
;
570 if (addr
< __fix_to_virt(FIX_KMAP_END
) ||
571 addr
> __fix_to_virt(FIX_KMAP_BEGIN
)) {
572 if (IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP
)) {
573 /* This _should_ never happen! See above. */
578 * Handle mappings which were obtained by kmap_high_get()
579 * first as the virtual address of such mappings is below
580 * PAGE_OFFSET. Warn for all other addresses which are in
581 * the user space part of the virtual address space.
583 if (!kmap_high_unmap_local(addr
))
584 WARN_ON_ONCE(addr
< PAGE_OFFSET
);
589 idx
= arch_kmap_local_unmap_idx(kmap_local_idx(), addr
);
590 WARN_ON_ONCE(addr
!= __fix_to_virt(FIX_KMAP_BEGIN
+ idx
));
592 kmap_pte
= kmap_get_pte(addr
, idx
);
593 arch_kmap_local_pre_unmap(addr
);
594 pte_clear(&init_mm
, addr
, kmap_pte
);
595 arch_kmap_local_post_unmap(addr
);
596 current
->kmap_ctrl
.pteval
[kmap_local_idx()] = __pte(0);
597 kmap_local_idx_pop();
601 EXPORT_SYMBOL(kunmap_local_indexed
);
604 * Invoked before switch_to(). This is safe even when during or after
605 * clearing the maps an interrupt which needs a kmap_local happens because
606 * the task::kmap_ctrl.idx is not modified by the unmapping code so a
607 * nested kmap_local will use the next unused index and restore the index
608 * on unmap. The already cleared kmaps of the outgoing task are irrelevant
609 * because the interrupt context does not know about them. The same applies
610 * when scheduling back in for an interrupt which happens before the
611 * restore is complete.
613 void __kmap_local_sched_out(void)
615 struct task_struct
*tsk
= current
;
620 for (i
= 0; i
< tsk
->kmap_ctrl
.idx
; i
++) {
621 pte_t pteval
= tsk
->kmap_ctrl
.pteval
[i
];
625 /* With debug all even slots are unmapped and act as guard */
626 if (IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL
) && !(i
& 0x01)) {
627 WARN_ON_ONCE(pte_val(pteval
) != 0);
630 if (WARN_ON_ONCE(pte_none(pteval
)))
634 * This is a horrible hack for XTENSA to calculate the
635 * coloured PTE index. Uses the PFN encoded into the pteval
636 * and the map index calculation because the actual mapped
637 * virtual address is not stored in task::kmap_ctrl.
638 * For any sane architecture this is optimized out.
640 idx
= arch_kmap_local_map_idx(i
, pte_pfn(pteval
));
642 addr
= __fix_to_virt(FIX_KMAP_BEGIN
+ idx
);
643 kmap_pte
= kmap_get_pte(addr
, idx
);
644 arch_kmap_local_pre_unmap(addr
);
645 pte_clear(&init_mm
, addr
, kmap_pte
);
646 arch_kmap_local_post_unmap(addr
);
650 void __kmap_local_sched_in(void)
652 struct task_struct
*tsk
= current
;
657 for (i
= 0; i
< tsk
->kmap_ctrl
.idx
; i
++) {
658 pte_t pteval
= tsk
->kmap_ctrl
.pteval
[i
];
662 /* With debug all even slots are unmapped and act as guard */
663 if (IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL
) && !(i
& 0x01)) {
664 WARN_ON_ONCE(pte_val(pteval
) != 0);
667 if (WARN_ON_ONCE(pte_none(pteval
)))
670 /* See comment in __kmap_local_sched_out() */
671 idx
= arch_kmap_local_map_idx(i
, pte_pfn(pteval
));
672 addr
= __fix_to_virt(FIX_KMAP_BEGIN
+ idx
);
673 kmap_pte
= kmap_get_pte(addr
, idx
);
674 set_pte_at(&init_mm
, addr
, kmap_pte
, pteval
);
675 arch_kmap_local_post_map(addr
, pteval
);
679 void kmap_local_fork(struct task_struct
*tsk
)
681 if (WARN_ON_ONCE(tsk
->kmap_ctrl
.idx
))
682 memset(&tsk
->kmap_ctrl
, 0, sizeof(tsk
->kmap_ctrl
));
687 #if defined(HASHED_PAGE_VIRTUAL)
689 #define PA_HASH_ORDER 7
692 * Describes one page->virtual association
694 struct page_address_map
{
697 struct list_head list
;
700 static struct page_address_map page_address_maps
[LAST_PKMAP
];
705 static struct page_address_slot
{
706 struct list_head lh
; /* List of page_address_maps */
707 spinlock_t lock
; /* Protect this bucket's list */
708 } ____cacheline_aligned_in_smp page_address_htable
[1<<PA_HASH_ORDER
];
710 static struct page_address_slot
*page_slot(const struct page
*page
)
712 return &page_address_htable
[hash_ptr(page
, PA_HASH_ORDER
)];
716 * page_address - get the mapped virtual address of a page
717 * @page: &struct page to get the virtual address of
719 * Returns the page's virtual address.
721 void *page_address(const struct page
*page
)
725 struct page_address_slot
*pas
;
727 if (!PageHighMem(page
))
728 return lowmem_page_address(page
);
730 pas
= page_slot(page
);
732 spin_lock_irqsave(&pas
->lock
, flags
);
733 if (!list_empty(&pas
->lh
)) {
734 struct page_address_map
*pam
;
736 list_for_each_entry(pam
, &pas
->lh
, list
) {
737 if (pam
->page
== page
) {
744 spin_unlock_irqrestore(&pas
->lock
, flags
);
747 EXPORT_SYMBOL(page_address
);
750 * set_page_address - set a page's virtual address
751 * @page: &struct page to set
752 * @virtual: virtual address to use
754 void set_page_address(struct page
*page
, void *virtual)
757 struct page_address_slot
*pas
;
758 struct page_address_map
*pam
;
760 BUG_ON(!PageHighMem(page
));
762 pas
= page_slot(page
);
763 if (virtual) { /* Add */
764 pam
= &page_address_maps
[PKMAP_NR((unsigned long)virtual)];
766 pam
->virtual = virtual;
768 spin_lock_irqsave(&pas
->lock
, flags
);
769 list_add_tail(&pam
->list
, &pas
->lh
);
770 spin_unlock_irqrestore(&pas
->lock
, flags
);
771 } else { /* Remove */
772 spin_lock_irqsave(&pas
->lock
, flags
);
773 list_for_each_entry(pam
, &pas
->lh
, list
) {
774 if (pam
->page
== page
) {
775 list_del(&pam
->list
);
779 spin_unlock_irqrestore(&pas
->lock
, flags
);
785 void __init
page_address_init(void)
789 for (i
= 0; i
< ARRAY_SIZE(page_address_htable
); i
++) {
790 INIT_LIST_HEAD(&page_address_htable
[i
].lh
);
791 spin_lock_init(&page_address_htable
[i
].lock
);
795 #endif /* defined(HASHED_PAGE_VIRTUAL) */