+++ /dev/null
-From 736b31e4394c8b483561e4ac1fd71eefb2bbec03 Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Thu, 26 Nov 2020 17:16:22 +0100
-Subject: mm: Fix pmd_read_atomic()
-
-From: Peter Zijlstra <peterz@infradead.org>
-
-[ Upstream commit 024d232ae4fcd7a7ce8ea239607d6c1246d7adc8 ]
-
-AFAICT there's no reason to do anything different than what we do for
-PTEs. Make it so (also affects SH).
-
-Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
-Link: https://lkml.kernel.org/r/20221022114424.711181252%40infradead.org
-Stable-dep-of: 71c186efc1b2 ("userfaultfd: fix checks for huge PMDs")
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- arch/x86/include/asm/pgtable-3level.h | 56 ---------------------------
- include/linux/pgtable.h | 47 +++++++++++++++++-----
- 2 files changed, 37 insertions(+), 66 deletions(-)
-
-diff --git a/arch/x86/include/asm/pgtable-3level.h b/arch/x86/include/asm/pgtable-3level.h
-index 28556d22feb8..94f50b0100a5 100644
---- a/arch/x86/include/asm/pgtable-3level.h
-+++ b/arch/x86/include/asm/pgtable-3level.h
-@@ -34,62 +34,6 @@ static inline void native_set_pte(pte_t *ptep, pte_t pte)
- ptep->pte_low = pte.pte_low;
- }
-
--#define pmd_read_atomic pmd_read_atomic
--/*
-- * pte_offset_map_lock() on 32-bit PAE kernels was reading the pmd_t with
-- * a "*pmdp" dereference done by GCC. Problem is, in certain places
-- * where pte_offset_map_lock() is called, concurrent page faults are
-- * allowed, if the mmap_lock is hold for reading. An example is mincore
-- * vs page faults vs MADV_DONTNEED. On the page fault side
-- * pmd_populate() rightfully does a set_64bit(), but if we're reading the
-- * pmd_t with a "*pmdp" on the mincore side, a SMP race can happen
-- * because GCC will not read the 64-bit value of the pmd atomically.
-- *
-- * To fix this all places running pte_offset_map_lock() while holding the
-- * mmap_lock in read mode, shall read the pmdp pointer using this
-- * function to know if the pmd is null or not, and in turn to know if
-- * they can run pte_offset_map_lock() or pmd_trans_huge() or other pmd
-- * operations.
-- *
-- * Without THP if the mmap_lock is held for reading, the pmd can only
-- * transition from null to not null while pmd_read_atomic() runs. So
-- * we can always return atomic pmd values with this function.
-- *
-- * With THP if the mmap_lock is held for reading, the pmd can become
-- * trans_huge or none or point to a pte (and in turn become "stable")
-- * at any time under pmd_read_atomic(). We could read it truly
-- * atomically here with an atomic64_read() for the THP enabled case (and
-- * it would be a whole lot simpler), but to avoid using cmpxchg8b we
-- * only return an atomic pmdval if the low part of the pmdval is later
-- * found to be stable (i.e. pointing to a pte). We are also returning a
-- * 'none' (zero) pmdval if the low part of the pmd is zero.
-- *
-- * In some cases the high and low part of the pmdval returned may not be
-- * consistent if THP is enabled (the low part may point to previously
-- * mapped hugepage, while the high part may point to a more recently
-- * mapped hugepage), but pmd_none_or_trans_huge_or_clear_bad() only
-- * needs the low part of the pmd to be read atomically to decide if the
-- * pmd is unstable or not, with the only exception when the low part
-- * of the pmd is zero, in which case we return a 'none' pmd.
-- */
--static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
--{
-- pmdval_t ret;
-- u32 *tmp = (u32 *)pmdp;
--
-- ret = (pmdval_t) (*tmp);
-- if (ret) {
-- /*
-- * If the low part is null, we must not read the high part
-- * or we can end up with a partial pmd.
-- */
-- smp_rmb();
-- ret |= ((pmdval_t)*(tmp + 1)) << 32;
-- }
--
-- return (pmd_t) { .pmd = ret };
--}
--
- static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
- {
- set_64bit((unsigned long long *)(ptep), native_pte_val(pte));
-diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
-index 5f0d7d0b9471..8f31e2ff6b58 100644
---- a/include/linux/pgtable.h
-+++ b/include/linux/pgtable.h
-@@ -316,6 +316,13 @@ static inline pte_t ptep_get(pte_t *ptep)
- }
- #endif
-
-+#ifndef __HAVE_ARCH_PMDP_GET
-+static inline pmd_t pmdp_get(pmd_t *pmdp)
-+{
-+ return READ_ONCE(*pmdp);
-+}
-+#endif
-+
- #ifdef CONFIG_GUP_GET_PTE_LOW_HIGH
- /*
- * WARNING: only to be used in the get_user_pages_fast() implementation.
-@@ -361,15 +368,42 @@ static inline pte_t ptep_get_lockless(pte_t *ptep)
-
- return pte;
- }
--#else /* CONFIG_GUP_GET_PTE_LOW_HIGH */
-+#define ptep_get_lockless ptep_get_lockless
-+
-+#if CONFIG_PGTABLE_LEVELS > 2
-+static inline pmd_t pmdp_get_lockless(pmd_t *pmdp)
-+{
-+ pmd_t pmd;
-+
-+ do {
-+ pmd.pmd_low = pmdp->pmd_low;
-+ smp_rmb();
-+ pmd.pmd_high = pmdp->pmd_high;
-+ smp_rmb();
-+ } while (unlikely(pmd.pmd_low != pmdp->pmd_low));
-+
-+ return pmd;
-+}
-+#define pmdp_get_lockless pmdp_get_lockless
-+#endif /* CONFIG_PGTABLE_LEVELS > 2 */
-+#endif /* CONFIG_GUP_GET_PTE_LOW_HIGH */
-+
- /*
- * We require that the PTE can be read atomically.
- */
-+#ifndef ptep_get_lockless
- static inline pte_t ptep_get_lockless(pte_t *ptep)
- {
- return ptep_get(ptep);
- }
--#endif /* CONFIG_GUP_GET_PTE_LOW_HIGH */
-+#endif
-+
-+#ifndef pmdp_get_lockless
-+static inline pmd_t pmdp_get_lockless(pmd_t *pmdp)
-+{
-+ return pmdp_get(pmdp);
-+}
-+#endif
-
- #ifdef CONFIG_TRANSPARENT_HUGEPAGE
- #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
-@@ -1339,17 +1373,10 @@ static inline int pud_trans_unstable(pud_t *pud)
- #endif
- }
-
--#ifndef pmd_read_atomic
- static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
- {
-- /*
-- * Depend on compiler for an atomic pmd read. NOTE: this is
-- * only going to work, if the pmdval_t isn't larger than
-- * an unsigned long.
-- */
-- return *pmdp;
-+ return pmdp_get_lockless(pmdp);
- }
--#endif
-
- #ifndef arch_needs_pgtable_deposit
- #define arch_needs_pgtable_deposit() (false)
---
-2.43.0
-
+++ /dev/null
-From ffc5769c6f6c603e01a725883e4d5760ee72939b Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Thu, 26 Nov 2020 17:20:28 +0100
-Subject: mm: Rename pmd_read_atomic()
-
-From: Peter Zijlstra <peterz@infradead.org>
-
-[ Upstream commit dab6e717429e5ec795d558a0e9a5337a1ed33a3d ]
-
-There's no point in having the identical routines for PTE/PMD have
-different names.
-
-Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
-Link: https://lkml.kernel.org/r/20221022114424.841277397%40infradead.org
-Stable-dep-of: 71c186efc1b2 ("userfaultfd: fix checks for huge PMDs")
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- include/linux/pgtable.h | 9 ++-------
- mm/hmm.c | 2 +-
- mm/khugepaged.c | 2 +-
- mm/mapping_dirty_helpers.c | 2 +-
- mm/mprotect.c | 2 +-
- mm/userfaultfd.c | 2 +-
- mm/vmscan.c | 4 ++--
- 7 files changed, 9 insertions(+), 14 deletions(-)
-
-diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
-index 8f31e2ff6b58..3e3c00e80b65 100644
---- a/include/linux/pgtable.h
-+++ b/include/linux/pgtable.h
-@@ -1373,11 +1373,6 @@ static inline int pud_trans_unstable(pud_t *pud)
- #endif
- }
-
--static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
--{
-- return pmdp_get_lockless(pmdp);
--}
--
- #ifndef arch_needs_pgtable_deposit
- #define arch_needs_pgtable_deposit() (false)
- #endif
-@@ -1404,13 +1399,13 @@ static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
- */
- static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd)
- {
-- pmd_t pmdval = pmd_read_atomic(pmd);
-+ pmd_t pmdval = pmdp_get_lockless(pmd);
- /*
- * The barrier will stabilize the pmdval in a register or on
- * the stack so that it will stop changing under the code.
- *
- * When CONFIG_TRANSPARENT_HUGEPAGE=y on x86 32bit PAE,
-- * pmd_read_atomic is allowed to return a not atomic pmdval
-+ * pmdp_get_lockless is allowed to return a not atomic pmdval
- * (for example pointing to an hugepage that has never been
- * mapped in the pmd). The below checks will only care about
- * the low part of the pmd with 32bit PAE x86 anyway, with the
-diff --git a/mm/hmm.c b/mm/hmm.c
-index 3850fb625dda..39cf50de76d7 100644
---- a/mm/hmm.c
-+++ b/mm/hmm.c
-@@ -361,7 +361,7 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
- * huge or device mapping one and compute corresponding pfn
- * values.
- */
-- pmd = pmd_read_atomic(pmdp);
-+ pmd = pmdp_get_lockless(pmdp);
- barrier();
- if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
- goto again;
-diff --git a/mm/khugepaged.c b/mm/khugepaged.c
-index 085fca1fa27a..47010c3b5c4d 100644
---- a/mm/khugepaged.c
-+++ b/mm/khugepaged.c
-@@ -866,7 +866,7 @@ static int find_pmd_or_thp_or_none(struct mm_struct *mm,
- if (!*pmd)
- return SCAN_PMD_NULL;
-
-- pmde = pmd_read_atomic(*pmd);
-+ pmde = pmdp_get_lockless(*pmd);
-
- #ifdef CONFIG_TRANSPARENT_HUGEPAGE
- /* See comments in pmd_none_or_trans_huge_or_clear_bad() */
-diff --git a/mm/mapping_dirty_helpers.c b/mm/mapping_dirty_helpers.c
-index 1b0ab8fcfd8b..175e424b9ab1 100644
---- a/mm/mapping_dirty_helpers.c
-+++ b/mm/mapping_dirty_helpers.c
-@@ -126,7 +126,7 @@ static int clean_record_pte(pte_t *pte, unsigned long addr,
- static int wp_clean_pmd_entry(pmd_t *pmd, unsigned long addr, unsigned long end,
- struct mm_walk *walk)
- {
-- pmd_t pmdval = pmd_read_atomic(pmd);
-+ pmd_t pmdval = pmdp_get_lockless(pmd);
-
- if (!pmd_trans_unstable(&pmdval))
- return 0;
-diff --git a/mm/mprotect.c b/mm/mprotect.c
-index 668bfaa6ed2a..f006bafe338f 100644
---- a/mm/mprotect.c
-+++ b/mm/mprotect.c
-@@ -294,7 +294,7 @@ static unsigned long change_pte_range(struct mmu_gather *tlb,
- */
- static inline int pmd_none_or_clear_bad_unless_trans_huge(pmd_t *pmd)
- {
-- pmd_t pmdval = pmd_read_atomic(pmd);
-+ pmd_t pmdval = pmdp_get_lockless(pmd);
-
- /* See pmd_none_or_trans_huge_or_clear_bad for info on barrier */
- #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
-index 992a0a16846f..5d873aadec76 100644
---- a/mm/userfaultfd.c
-+++ b/mm/userfaultfd.c
-@@ -641,7 +641,7 @@ static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm,
- break;
- }
-
-- dst_pmdval = pmd_read_atomic(dst_pmd);
-+ dst_pmdval = pmdp_get_lockless(dst_pmd);
- /*
- * If the dst_pmd is mapped as THP don't
- * override it and just be strict.
-diff --git a/mm/vmscan.c b/mm/vmscan.c
-index 4cd0cbf9c121..f5fa1c76d9e6 100644
---- a/mm/vmscan.c
-+++ b/mm/vmscan.c
-@@ -4068,9 +4068,9 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
- /* walk_pte_range() may call get_next_vma() */
- vma = args->vma;
- for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) {
-- pmd_t val = pmd_read_atomic(pmd + i);
-+ pmd_t val = pmdp_get_lockless(pmd + i);
-
-- /* for pmd_read_atomic() */
-+ /* for pmdp_get_lockless() */
- barrier();
-
- next = pmd_addr_end(addr, end);
---
-2.43.0
-
fuse-allow-non-extending-parallel-direct-writes-on-t.patch
fuse-add-request-extension.patch
fuse-fix-memory-leak-in-fuse_create_open.patch
-x86-mm-pae-make-pmd_t-similar-to-pte_t.patch
-mm-fix-pmd_read_atomic.patch
-mm-rename-pmd_read_atomic.patch
-userfaultfd-fix-checks-for-huge-pmds.patch
net-mana-fix-error-handling-in-mana_create_txq-rxq-s.patch
workqueue-wq_watchdog_touch-is-always-called-with-va.patch
workqueue-improve-scalability-of-workqueue-watchdog-.patch
+++ /dev/null
-From 170b9e3f04840e9c1a1e14c687b689e84d8f0d9d Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Tue, 13 Aug 2024 22:25:21 +0200
-Subject: userfaultfd: fix checks for huge PMDs
-
-From: Jann Horn <jannh@google.com>
-
-[ Upstream commit 71c186efc1b2cf1aeabfeff3b9bd5ac4c5ac14d8 ]
-
-Patch series "userfaultfd: fix races around pmd_trans_huge() check", v2.
-
-The pmd_trans_huge() code in mfill_atomic() is wrong in three different
-ways depending on kernel version:
-
-1. The pmd_trans_huge() check is racy and can lead to a BUG_ON() (if you hit
- the right two race windows) - I've tested this in a kernel build with
- some extra mdelay() calls. See the commit message for a description
- of the race scenario.
- On older kernels (before 6.5), I think the same bug can even
- theoretically lead to accessing transhuge page contents as a page table
- if you hit the right 5 narrow race windows (I haven't tested this case).
-2. As pointed out by Qi Zheng, pmd_trans_huge() is not sufficient for
- detecting PMDs that don't point to page tables.
- On older kernels (before 6.5), you'd just have to win a single fairly
- wide race to hit this.
- I've tested this on 6.1 stable by racing migration (with a mdelay()
- patched into try_to_migrate()) against UFFDIO_ZEROPAGE - on my x86
- VM, that causes a kernel oops in ptlock_ptr().
-3. On newer kernels (>=6.5), for shmem mappings, khugepaged is allowed
- to yank page tables out from under us (though I haven't tested that),
- so I think the BUG_ON() checks in mfill_atomic() are just wrong.
-
-I decided to write two separate fixes for these (one fix for bugs 1+2, one
-fix for bug 3), so that the first fix can be backported to kernels
-affected by bugs 1+2.
-
-This patch (of 2):
-
-This fixes two issues.
-
-I discovered that the following race can occur:
-
- mfill_atomic other thread
- ============ ============
- <zap PMD>
- pmdp_get_lockless() [reads none pmd]
- <bail if trans_huge>
- <if none:>
- <pagefault creates transhuge zeropage>
- __pte_alloc [no-op]
- <zap PMD>
- <bail if pmd_trans_huge(*dst_pmd)>
- BUG_ON(pmd_none(*dst_pmd))
-
-I have experimentally verified this in a kernel with extra mdelay() calls;
-the BUG_ON(pmd_none(*dst_pmd)) triggers.
-
-On kernels newer than commit 0d940a9b270b ("mm/pgtable: allow
-pte_offset_map[_lock]() to fail"), this can't lead to anything worse than
-a BUG_ON(), since the page table access helpers are actually designed to
-deal with page tables concurrently disappearing; but on older kernels
-(<=6.4), I think we could probably theoretically race past the two
-BUG_ON() checks and end up treating a hugepage as a page table.
-
-The second issue is that, as Qi Zheng pointed out, there are other types
-of huge PMDs that pmd_trans_huge() can't catch: devmap PMDs and swap PMDs
-(in particular, migration PMDs).
-
-On <=6.4, this is worse than the first issue: If mfill_atomic() runs on a
-PMD that contains a migration entry (which just requires winning a single,
-fairly wide race), it will pass the PMD to pte_offset_map_lock(), which
-assumes that the PMD points to a page table.
-
-Breakage follows: First, the kernel tries to take the PTE lock (which will
-crash or maybe worse if there is no "struct page" for the address bits in
-the migration entry PMD - I think at least on X86 there usually is no
-corresponding "struct page" thanks to the PTE inversion mitigation, amd64
-looks different).
-
-If that didn't crash, the kernel would next try to write a PTE into what
-it wrongly thinks is a page table.
-
-As part of fixing these issues, get rid of the check for pmd_trans_huge()
-before __pte_alloc() - that's redundant, we're going to have to check for
-that after the __pte_alloc() anyway.
-
-Backport note: pmdp_get_lockless() is pmd_read_atomic() in older kernels.
-
-Link: https://lkml.kernel.org/r/20240813-uffd-thp-flip-fix-v2-0-5efa61078a41@google.com
-Link: https://lkml.kernel.org/r/20240813-uffd-thp-flip-fix-v2-1-5efa61078a41@google.com
-Fixes: c1a4de99fada ("userfaultfd: mcopy_atomic|mfill_zeropage: UFFDIO_COPY|UFFDIO_ZEROPAGE preparation")
-Signed-off-by: Jann Horn <jannh@google.com>
-Acked-by: David Hildenbrand <david@redhat.com>
-Cc: Andrea Arcangeli <aarcange@redhat.com>
-Cc: Hugh Dickins <hughd@google.com>
-Cc: Jann Horn <jannh@google.com>
-Cc: Pavel Emelyanov <xemul@virtuozzo.com>
-Cc: Qi Zheng <zhengqi.arch@bytedance.com>
-Cc: <stable@vger.kernel.org>
-Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- mm/userfaultfd.c | 22 ++++++++++++----------
- 1 file changed, 12 insertions(+), 10 deletions(-)
-
-diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
-index 5d873aadec76..50c01a7eb705 100644
---- a/mm/userfaultfd.c
-+++ b/mm/userfaultfd.c
-@@ -642,21 +642,23 @@ static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm,
- }
-
- dst_pmdval = pmdp_get_lockless(dst_pmd);
-- /*
-- * If the dst_pmd is mapped as THP don't
-- * override it and just be strict.
-- */
-- if (unlikely(pmd_trans_huge(dst_pmdval))) {
-- err = -EEXIST;
-- break;
-- }
- if (unlikely(pmd_none(dst_pmdval)) &&
- unlikely(__pte_alloc(dst_mm, dst_pmd))) {
- err = -ENOMEM;
- break;
- }
-- /* If an huge pmd materialized from under us fail */
-- if (unlikely(pmd_trans_huge(*dst_pmd))) {
-+ dst_pmdval = pmdp_get_lockless(dst_pmd);
-+ /*
-+ * If the dst_pmd is THP don't override it and just be strict.
-+ * (This includes the case where the PMD used to be THP and
-+ * changed back to none after __pte_alloc().)
-+ */
-+ if (unlikely(!pmd_present(dst_pmdval) || pmd_trans_huge(dst_pmdval) ||
-+ pmd_devmap(dst_pmdval))) {
-+ err = -EEXIST;
-+ break;
-+ }
-+ if (unlikely(pmd_bad(dst_pmdval))) {
- err = -EFAULT;
- break;
- }
---
-2.43.0
-
+++ /dev/null
-From 4eae6e27b21e427bd48973eec754e0833489829f Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Thu, 26 Nov 2020 17:02:29 +0100
-Subject: x86/mm/pae: Make pmd_t similar to pte_t
-
-From: Peter Zijlstra <peterz@infradead.org>
-
-[ Upstream commit fbfdec9989e69e0b17aa3bf32fcb22d04cc33301 ]
-
-Instead of mucking about with at least 2 different ways of fudging
-it, do the same thing we do for pte_t.
-
-Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
-Link: https://lkml.kernel.org/r/20221022114424.580310787%40infradead.org
-Stable-dep-of: 71c186efc1b2 ("userfaultfd: fix checks for huge PMDs")
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- arch/x86/include/asm/pgtable-3level.h | 42 +++++++--------------
- arch/x86/include/asm/pgtable-3level_types.h | 7 ++++
- arch/x86/include/asm/pgtable_64_types.h | 1 +
- arch/x86/include/asm/pgtable_types.h | 4 +-
- 4 files changed, 23 insertions(+), 31 deletions(-)
-
-diff --git a/arch/x86/include/asm/pgtable-3level.h b/arch/x86/include/asm/pgtable-3level.h
-index 28421a887209..28556d22feb8 100644
---- a/arch/x86/include/asm/pgtable-3level.h
-+++ b/arch/x86/include/asm/pgtable-3level.h
-@@ -87,7 +87,7 @@ static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
- ret |= ((pmdval_t)*(tmp + 1)) << 32;
- }
-
-- return (pmd_t) { ret };
-+ return (pmd_t) { .pmd = ret };
- }
-
- static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
-@@ -121,12 +121,11 @@ static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr,
- ptep->pte_high = 0;
- }
-
--static inline void native_pmd_clear(pmd_t *pmd)
-+static inline void native_pmd_clear(pmd_t *pmdp)
- {
-- u32 *tmp = (u32 *)pmd;
-- *tmp = 0;
-+ pmdp->pmd_low = 0;
- smp_wmb();
-- *(tmp + 1) = 0;
-+ pmdp->pmd_high = 0;
- }
-
- static inline void native_pud_clear(pud_t *pudp)
-@@ -162,25 +161,17 @@ static inline pte_t native_ptep_get_and_clear(pte_t *ptep)
- #define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp)
- #endif
-
--union split_pmd {
-- struct {
-- u32 pmd_low;
-- u32 pmd_high;
-- };
-- pmd_t pmd;
--};
--
- #ifdef CONFIG_SMP
- static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp)
- {
-- union split_pmd res, *orig = (union split_pmd *)pmdp;
-+ pmd_t res;
-
- /* xchg acts as a barrier before setting of the high bits */
-- res.pmd_low = xchg(&orig->pmd_low, 0);
-- res.pmd_high = orig->pmd_high;
-- orig->pmd_high = 0;
-+ res.pmd_low = xchg(&pmdp->pmd_low, 0);
-+ res.pmd_high = READ_ONCE(pmdp->pmd_high);
-+ WRITE_ONCE(pmdp->pmd_high, 0);
-
-- return res.pmd;
-+ return res;
- }
- #else
- #define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)
-@@ -199,17 +190,12 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
- * anybody.
- */
- if (!(pmd_val(pmd) & _PAGE_PRESENT)) {
-- union split_pmd old, new, *ptr;
--
-- ptr = (union split_pmd *)pmdp;
--
-- new.pmd = pmd;
--
- /* xchg acts as a barrier before setting of the high bits */
-- old.pmd_low = xchg(&ptr->pmd_low, new.pmd_low);
-- old.pmd_high = ptr->pmd_high;
-- ptr->pmd_high = new.pmd_high;
-- return old.pmd;
-+ old.pmd_low = xchg(&pmdp->pmd_low, pmd.pmd_low);
-+ old.pmd_high = READ_ONCE(pmdp->pmd_high);
-+ WRITE_ONCE(pmdp->pmd_high, pmd.pmd_high);
-+
-+ return old;
- }
-
- do {
-diff --git a/arch/x86/include/asm/pgtable-3level_types.h b/arch/x86/include/asm/pgtable-3level_types.h
-index 56baf43befb4..80911349519e 100644
---- a/arch/x86/include/asm/pgtable-3level_types.h
-+++ b/arch/x86/include/asm/pgtable-3level_types.h
-@@ -18,6 +18,13 @@ typedef union {
- };
- pteval_t pte;
- } pte_t;
-+
-+typedef union {
-+ struct {
-+ unsigned long pmd_low, pmd_high;
-+ };
-+ pmdval_t pmd;
-+} pmd_t;
- #endif /* !__ASSEMBLY__ */
-
- #define SHARED_KERNEL_PMD (!static_cpu_has(X86_FEATURE_PTI))
-diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h
-index 6c7f7c526450..4ea3755f2444 100644
---- a/arch/x86/include/asm/pgtable_64_types.h
-+++ b/arch/x86/include/asm/pgtable_64_types.h
-@@ -19,6 +19,7 @@ typedef unsigned long pgdval_t;
- typedef unsigned long pgprotval_t;
-
- typedef struct { pteval_t pte; } pte_t;
-+typedef struct { pmdval_t pmd; } pmd_t;
-
- #ifdef CONFIG_X86_5LEVEL
- extern unsigned int __pgtable_l5_enabled;
-diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
-index e3028373f0b4..d0e9654d7272 100644
---- a/arch/x86/include/asm/pgtable_types.h
-+++ b/arch/x86/include/asm/pgtable_types.h
-@@ -363,11 +363,9 @@ static inline pudval_t native_pud_val(pud_t pud)
- #endif
-
- #if CONFIG_PGTABLE_LEVELS > 2
--typedef struct { pmdval_t pmd; } pmd_t;
--
- static inline pmd_t native_make_pmd(pmdval_t val)
- {
-- return (pmd_t) { val };
-+ return (pmd_t) { .pmd = val };
- }
-
- static inline pmdval_t native_pmd_val(pmd_t pmd)
---
-2.43.0
-
projects / thirdparty / kernel / stable-queue.git / commitdiff
