releases/4.5.7/mips-sync-icache-dcache-in-set_pte_at.patch

   1 From 37d22a0d798b5c938b277d32cfd86dc231381342 Mon Sep 17 00:00:00 2001
   2 From: Paul Burton <paul.burton@imgtec.com>
   3 Date: Tue, 1 Mar 2016 02:37:59 +0000
   4 Subject: MIPS: Sync icache & dcache in set_pte_at
   5
   6 From: Paul Burton <paul.burton@imgtec.com>
   7
   8 commit 37d22a0d798b5c938b277d32cfd86dc231381342 upstream.
   9
  10 It's possible for pages to become visible prior to update_mmu_cache
  11 running if a thread within the same address space preempts the current
  12 thread or runs simultaneously on another CPU. That is, the following
  13 scenario is possible:
  14
  15     CPU0                            CPU1
  16
  17     write to page
  18     flush_dcache_page
  19     flush_icache_page
  20     set_pte_at
  21                                     map page
  22     update_mmu_cache
  23
  24 If CPU1 maps the page in between CPU0's set_pte_at, which marks it valid
  25 & visible, and update_mmu_cache where the dcache flush occurs then CPU1s
  26 icache will fill from stale data (unless it fills from the dcache, in
  27 which case all is good, but most MIPS CPUs don't have this property).
  28 Commit 4d46a67a3eb8 ("MIPS: Fix race condition in lazy cache flushing.")
  29 attempted to fix that by performing the dcache flush in
  30 flush_icache_page such that it occurs before the set_pte_at call makes
  31 the page visible. However it has the problem that not all code that
  32 writes to pages exposed to userland call flush_icache_page. There are
  33 many callers of set_pte_at under mm/ and only 2 of them do call
  34 flush_icache_page. Thus the race window between a page becoming visible
  35 & being coherent between the icache & dcache remains open in some cases.
  36
  37 To illustrate some of the cases, a WARN was added to __update_cache with
  38 this patch applied that triggered in cases where a page about to be
  39 flushed from the dcache was not the last page provided to
  40 flush_icache_page. That is, backtraces were obtained for cases in which
  41 the race window is left open without this patch. The 2 standout examples
  42 follow.
  43
  44 When forking a process:
  45
  46 [   15.271842] [<80417630>] __update_cache+0xcc/0x188
  47 [   15.277274] [<80530394>] copy_page_range+0x56c/0x6ac
  48 [   15.282861] [<8042936c>] copy_process.part.54+0xd40/0x17ac
  49 [   15.289028] [<80429f80>] do_fork+0xe4/0x420
  50 [   15.293747] [<80413808>] handle_sys+0x128/0x14c
  51
  52 When exec'ing an ELF binary:
  53
  54 [   14.445964] [<80417630>] __update_cache+0xcc/0x188
  55 [   14.451369] [<80538d88>] move_page_tables+0x414/0x498
  56 [   14.457075] [<8055d848>] setup_arg_pages+0x220/0x318
  57 [   14.462685] [<805b0f38>] load_elf_binary+0x530/0x12a0
  58 [   14.468374] [<8055ec3c>] search_binary_handler+0xbc/0x214
  59 [   14.474444] [<8055f6c0>] do_execveat_common+0x43c/0x67c
  60 [   14.480324] [<8055f938>] do_execve+0x38/0x44
  61 [   14.485137] [<80413808>] handle_sys+0x128/0x14c
  62
  63 These code paths write into a page, call flush_dcache_page then call
  64 set_pte_at without flush_icache_page inbetween. The end result is that
  65 the icache can become corrupted & userland processes may execute
  66 unexpected or invalid code, typically resulting in a reserved
  67 instruction exception, a trap or a segfault.
  68
  69 Fix this race condition fully by performing any cache maintenance
  70 required to keep the icache & dcache in sync in set_pte_at, before the
  71 page is made valid. This has the added bonus of ensuring the cache
  72 maintenance always happens in one location, rather than being duplicated
  73 in flush_icache_page & update_mmu_cache. It also matches the way other
  74 architectures solve the same problem (see arm, ia64 & powerpc).
  75
  76 Signed-off-by: Paul Burton <paul.burton@imgtec.com>
  77 Reported-by: Ionela Voinescu <ionela.voinescu@imgtec.com>
  78 Cc: Lars Persson <lars.persson@axis.com>
  79 Fixes: 4d46a67a3eb8 ("MIPS: Fix race condition in lazy cache flushing.")
  80 Cc: Steven J. Hill <sjhill@realitydiluted.com>
  81 Cc: David Daney <david.daney@cavium.com>
  82 Cc: Huacai Chen <chenhc@lemote.com>
  83 Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
  84 Cc: Andrew Morton <akpm@linux-foundation.org>
  85 Cc: Jerome Marchand <jmarchan@redhat.com>
  86 Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
  87 Cc: linux-mips@linux-mips.org
  88 Cc: linux-kernel@vger.kernel.org
  89 Patchwork: https://patchwork.linux-mips.org/patch/12722/
  90 Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
  91 Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
  92
  93 ---
  94  arch/mips/include/asm/cacheflush.h |    6 ------
  95  arch/mips/include/asm/pgtable.h    |   26 +++++++++++++++++++++-----
  96  arch/mips/mm/cache.c               |   19 +++----------------
  97  3 files changed, 24 insertions(+), 27 deletions(-)
  98
  99 --- a/arch/mips/include/asm/cacheflush.h
 100 +++ b/arch/mips/include/asm/cacheflush.h
 101 @@ -51,7 +51,6 @@ extern void (*flush_cache_range)(struct
 102         unsigned long start, unsigned long end);
 103  extern void (*flush_cache_page)(struct vm_area_struct *vma, unsigned long page, unsigned long pfn);
 104  extern void __flush_dcache_page(struct page *page);
 105 -extern void __flush_icache_page(struct vm_area_struct *vma, struct page *page);
 106
 107  #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
 108  static inline void flush_dcache_page(struct page *page)
 109 @@ -77,11 +76,6 @@ static inline void flush_anon_page(struc
 110  static inline void flush_icache_page(struct vm_area_struct *vma,
 111         struct page *page)
 112  {
 113 -       if (!cpu_has_ic_fills_f_dc && (vma->vm_flags & VM_EXEC) &&
 114 -           Page_dcache_dirty(page)) {
 115 -               __flush_icache_page(vma, page);
 116 -               ClearPageDcacheDirty(page);
 117 -       }
 118  }
 119
 120  extern void (*flush_icache_range)(unsigned long start, unsigned long end);
 121 --- a/arch/mips/include/asm/pgtable.h
 122 +++ b/arch/mips/include/asm/pgtable.h
 123 @@ -127,10 +127,14 @@ do {                                                                      \
 124         }                                                               \
 125  } while(0)
 126
 127 +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
 128 +                             pte_t *ptep, pte_t pteval);
 129 +
 130  #if defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_CPU_MIPS32)
 131
 132  #define pte_none(pte)          (!(((pte).pte_high) & ~_PAGE_GLOBAL))
 133  #define pte_present(pte)       ((pte).pte_low & _PAGE_PRESENT)
 134 +#define pte_no_exec(pte)       ((pte).pte_low & _PAGE_NO_EXEC)
 135
 136  static inline void set_pte(pte_t *ptep, pte_t pte)
 137  {
 138 @@ -148,7 +152,6 @@ static inline void set_pte(pte_t *ptep,
 139                         buddy->pte_high |= _PAGE_GLOBAL;
 140         }
 141  }
 142 -#define set_pte_at(mm, addr, ptep, pteval) set_pte(ptep, pteval)
 143
 144  static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 145  {
 146 @@ -166,6 +169,7 @@ static inline void pte_clear(struct mm_s
 147
 148  #define pte_none(pte)          (!(pte_val(pte) & ~_PAGE_GLOBAL))
 149  #define pte_present(pte)       (pte_val(pte) & _PAGE_PRESENT)
 150 +#define pte_no_exec(pte)       (pte_val(pte) & _PAGE_NO_EXEC)
 151
 152  /*
 153   * Certain architectures need to do special things when pte's
 154 @@ -218,7 +222,6 @@ static inline void set_pte(pte_t *ptep,
 155         }
 156  #endif
 157  }
 158 -#define set_pte_at(mm, addr, ptep, pteval) set_pte(ptep, pteval)
 159
 160  static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 161  {
 162 @@ -234,6 +237,22 @@ static inline void pte_clear(struct mm_s
 163  }
 164  #endif
 165
 166 +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
 167 +                             pte_t *ptep, pte_t pteval)
 168 +{
 169 +       extern void __update_cache(unsigned long address, pte_t pte);
 170 +
 171 +       if (!pte_present(pteval))
 172 +               goto cache_sync_done;
 173 +
 174 +       if (pte_present(*ptep) && (pte_pfn(*ptep) == pte_pfn(pteval)))
 175 +               goto cache_sync_done;
 176 +
 177 +       __update_cache(addr, pteval);
 178 +cache_sync_done:
 179 +       set_pte(ptep, pteval);
 180 +}
 181 +
 182  /*
 183   * (pmds are folded into puds so this doesn't get actually called,
 184   * but the define is needed for a generic inline function.)
 185 @@ -430,15 +449,12 @@ static inline pte_t pte_modify(pte_t pte
 186
 187  extern void __update_tlb(struct vm_area_struct *vma, unsigned long address,
 188         pte_t pte);
 189 -extern void __update_cache(struct vm_area_struct *vma, unsigned long address,
 190 -       pte_t pte);
 191
 192  static inline void update_mmu_cache(struct vm_area_struct *vma,
 193         unsigned long address, pte_t *ptep)
 194  {
 195         pte_t pte = *ptep;
 196         __update_tlb(vma, address, pte);
 197 -       __update_cache(vma, address, pte);
 198  }
 199
 200  static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
 201 --- a/arch/mips/mm/cache.c
 202 +++ b/arch/mips/mm/cache.c
 203 @@ -125,30 +125,17 @@ void __flush_anon_page(struct page *page
 204
 205  EXPORT_SYMBOL(__flush_anon_page);
 206
 207 -void __flush_icache_page(struct vm_area_struct *vma, struct page *page)
 208 -{
 209 -       unsigned long addr;
 210 -
 211 -       if (PageHighMem(page))
 212 -               return;
 213 -
 214 -       addr = (unsigned long) page_address(page);
 215 -       flush_data_cache_page(addr);
 216 -}
 217 -EXPORT_SYMBOL_GPL(__flush_icache_page);
 218 -
 219 -void __update_cache(struct vm_area_struct *vma, unsigned long address,
 220 -       pte_t pte)
 221 +void __update_cache(unsigned long address, pte_t pte)
 222  {
 223         struct page *page;
 224         unsigned long pfn, addr;
 225 -       int exec = (vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc;
 226 +       int exec = !pte_no_exec(pte) && !cpu_has_ic_fills_f_dc;
 227
 228         pfn = pte_pfn(pte);
 229         if (unlikely(!pfn_valid(pfn)))
 230                 return;
 231         page = pfn_to_page(pfn);
 232 -       if (page_mapping(page) && Page_dcache_dirty(page)) {
 233 +       if (Page_dcache_dirty(page)) {
 234                 if (PageHighMem(page))
 235                         addr = (unsigned long)kmap_atomic(page);
 236                 else