mm/huge_memory.c: reorder operations in __split_huge_page_tail()

[thirdparty/kernel/stable.git] / mm / huge_memory.c

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 5a68730eebd62656e3c300581da9a9e71fd7ffc8..f0ae8d1d4329d5c72c9bcf9e460e16a7815a1059 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2356,26 +2356,13 @@ static void __split_huge_page_tail(struct page *head, int tail,
        struct page *page_tail = head + tail;
 
        VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
-       VM_BUG_ON_PAGE(page_ref_count(page_tail) != 0, page_tail);
 
        /*
-        * tail_page->_refcount is zero and not changing from under us. But
-        * get_page_unless_zero() may be running from under us on the
-        * tail_page. If we used atomic_set() below instead of atomic_inc() or
-        * atomic_add(), we would then run atomic_set() concurrently with
-        * get_page_unless_zero(), and atomic_set() is implemented in C not
-        * using locked ops. spin_unlock on x86 sometime uses locked ops
-        * because of PPro errata 66, 92, so unless somebody can guarantee
-        * atomic_set() here would be safe on all archs (and not only on x86),
-        * it's safer to use atomic_inc()/atomic_add().
+        * Clone page flags before unfreezing refcount.
+        *
+        * After successful get_page_unless_zero() might follow flags change,
+        * for exmaple lock_page() which set PG_waiters.
         */
-       if (PageAnon(head) && !PageSwapCache(head)) {
-               page_ref_inc(page_tail);
-       } else {
-               /* Additional pin to radix tree */
-               page_ref_add(page_tail, 2);
-       }
-
        page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
        page_tail->flags |= (head->flags &
                        ((1L << PG_referenced) |
@@ -2388,14 +2375,21 @@ static void __split_huge_page_tail(struct page *head, int tail,
                         (1L << PG_unevictable) |
                         (1L << PG_dirty)));
 
-       /*
-        * After clearing PageTail the gup refcount can be released.
-        * Page flags also must be visible before we make the page non-compound.
-        */
+       /* Page flags must be visible before we make the page non-compound. */
        smp_wmb();
 
+       /*
+        * Clear PageTail before unfreezing page refcount.
+        *
+        * After successful get_page_unless_zero() might follow put_page()
+        * which needs correct compound_head().
+        */
        clear_compound_head(page_tail);
 
+       /* Finally unfreeze refcount. Additional reference from page cache. */
+       page_ref_unfreeze(page_tail, 1 + (!PageAnon(head) ||
+                                         PageSwapCache(head)));
+
        if (page_is_young(head))
                set_page_young(page_tail);
        if (page_is_idle(head))