]>
Commit | Line | Data |
---|---|---|
b9e9b117 GKH |
1 | From 3526741f0964c88bc2ce511e1078359052bf225b Mon Sep 17 00:00:00 2001 |
2 | From: Andrea Arcangeli <aarcange@redhat.com> | |
3 | Date: Wed, 2 Nov 2011 13:37:15 -0700 | |
4 | Subject: powerpc: gup_hugepte() support THP based tail recounting | |
5 | ||
6 | From: Andrea Arcangeli <aarcange@redhat.com> | |
7 | ||
8 | commit 3526741f0964c88bc2ce511e1078359052bf225b upstream. | |
9 | ||
10 | Up to this point the code assumed old refcounting for hugepages (pre-thp). | |
11 | This updates the code directly to the thp mapcount tail page refcounting. | |
12 | ||
13 | Signed-off-by: Andrea Arcangeli <aarcange@redhat.com> | |
14 | Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> | |
15 | Cc: Hugh Dickins <hughd@google.com> | |
16 | Cc: Johannes Weiner <jweiner@redhat.com> | |
17 | Cc: Rik van Riel <riel@redhat.com> | |
18 | Cc: Mel Gorman <mgorman@suse.de> | |
19 | Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> | |
20 | Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> | |
21 | Cc: David Gibson <david@gibson.dropbear.id.au> | |
22 | Signed-off-by: Andrew Morton <akpm@linux-foundation.org> | |
23 | Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> | |
24 | Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de> | |
25 | ||
26 | --- | |
27 | arch/powerpc/mm/hugetlbpage.c | 24 +++++++++++++++++++++++- | |
28 | 1 file changed, 23 insertions(+), 1 deletion(-) | |
29 | ||
30 | --- a/arch/powerpc/mm/hugetlbpage.c | |
31 | +++ b/arch/powerpc/mm/hugetlbpage.c | |
32 | @@ -385,12 +385,23 @@ follow_huge_pmd(struct mm_struct *mm, un | |
33 | return NULL; | |
34 | } | |
35 | ||
36 | +static inline void get_huge_page_tail(struct page *page) | |
37 | +{ | |
38 | + /* | |
39 | + * __split_huge_page_refcount() cannot run | |
40 | + * from under us. | |
41 | + */ | |
42 | + VM_BUG_ON(page_mapcount(page) < 0); | |
43 | + VM_BUG_ON(atomic_read(&page->_count) != 0); | |
44 | + atomic_inc(&page->_mapcount); | |
45 | +} | |
46 | + | |
47 | static noinline int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, | |
48 | unsigned long end, int write, struct page **pages, int *nr) | |
49 | { | |
50 | unsigned long mask; | |
51 | unsigned long pte_end; | |
52 | - struct page *head, *page; | |
53 | + struct page *head, *page, *tail; | |
54 | pte_t pte; | |
55 | int refs; | |
56 | ||
57 | @@ -413,6 +424,7 @@ static noinline int gup_hugepte(pte_t *p | |
58 | head = pte_page(pte); | |
59 | ||
60 | page = head + ((addr & (sz-1)) >> PAGE_SHIFT); | |
61 | + tail = page; | |
62 | do { | |
63 | VM_BUG_ON(compound_head(page) != head); | |
64 | pages[*nr] = page; | |
65 | @@ -431,6 +443,16 @@ static noinline int gup_hugepte(pte_t *p | |
66 | *nr -= refs; | |
67 | while (refs--) | |
68 | put_page(head); | |
69 | + } else { | |
70 | + /* | |
71 | + * Any tail page need their mapcount reference taken | |
72 | + * before we return. | |
73 | + */ | |
74 | + while (refs--) { | |
75 | + if (PageTail(tail)) | |
76 | + get_huge_page_tail(tail); | |
77 | + tail++; | |
78 | + } | |
79 | } | |
80 | ||
81 | return 1; |