arch/powerpc/include/asm/book3s/64/tlbflush.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H
   3 #define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H
   4
   5 #define MMU_NO_CONTEXT  ~0UL
   6
   7 #include <linux/mm_types.h>
   8 #include <asm/book3s/64/tlbflush-hash.h>
   9 #include <asm/book3s/64/tlbflush-radix.h>
  10
  11 /* TLB flush actions. Used as argument to tlbiel_all() */
  12 enum {
  13         TLB_INVAL_SCOPE_GLOBAL = 0,     /* invalidate all TLBs */
  14         TLB_INVAL_SCOPE_LPID = 1,       /* invalidate TLBs for current LPID */
  15 };
  16
  17 static inline void tlbiel_all(void)
  18 {
  19         /*
  20          * This is used for host machine check and bootup.
  21          *
  22          * This uses early_radix_enabled and implementations use
  23          * early_cpu_has_feature etc because that works early in boot
  24          * and this is the machine check path which is not performance
  25          * critical.
  26          */
  27         if (early_radix_enabled())
  28                 radix__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL);
  29         else
  30                 hash__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL);
  31 }
  32
  33 static inline void tlbiel_all_lpid(bool radix)
  34 {
  35         /*
  36          * This is used for guest machine check.
  37          */
  38         if (radix)
  39                 radix__tlbiel_all(TLB_INVAL_SCOPE_LPID);
  40         else
  41                 hash__tlbiel_all(TLB_INVAL_SCOPE_LPID);
  42 }
  43
  44
  45 #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
  46 static inline void flush_pmd_tlb_range(struct vm_area_struct *vma,
  47                                        unsigned long start, unsigned long end)
  48 {
  49         if (radix_enabled())
  50                 radix__flush_pmd_tlb_range(vma, start, end);
  51 }
  52
  53 #define __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
  54 static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
  55                                            unsigned long start,
  56                                            unsigned long end)
  57 {
  58         if (radix_enabled())
  59                 radix__flush_hugetlb_tlb_range(vma, start, end);
  60 }
  61
  62 static inline void flush_tlb_range(struct vm_area_struct *vma,
  63                                    unsigned long start, unsigned long end)
  64 {
  65         if (radix_enabled())
  66                 radix__flush_tlb_range(vma, start, end);
  67 }
  68
  69 static inline void flush_tlb_kernel_range(unsigned long start,
  70                                           unsigned long end)
  71 {
  72         if (radix_enabled())
  73                 radix__flush_tlb_kernel_range(start, end);
  74 }
  75
  76 static inline void local_flush_tlb_mm(struct mm_struct *mm)
  77 {
  78         if (radix_enabled())
  79                 radix__local_flush_tlb_mm(mm);
  80 }
  81
  82 static inline void local_flush_tlb_page(struct vm_area_struct *vma,
  83                                         unsigned long vmaddr)
  84 {
  85         if (radix_enabled())
  86                 radix__local_flush_tlb_page(vma, vmaddr);
  87 }
  88
  89 static inline void local_flush_tlb_page_psize(struct mm_struct *mm,
  90                                               unsigned long vmaddr, int psize)
  91 {
  92         if (radix_enabled())
  93                 radix__local_flush_tlb_page_psize(mm, vmaddr, psize);
  94 }
  95
  96 static inline void tlb_flush(struct mmu_gather *tlb)
  97 {
  98         if (radix_enabled())
  99                 radix__tlb_flush(tlb);
 100         else
 101                 hash__tlb_flush(tlb);
 102 }
 103
 104 #ifdef CONFIG_SMP
 105 static inline void flush_tlb_mm(struct mm_struct *mm)
 106 {
 107         if (radix_enabled())
 108                 radix__flush_tlb_mm(mm);
 109 }
 110
 111 static inline void flush_tlb_page(struct vm_area_struct *vma,
 112                                   unsigned long vmaddr)
 113 {
 114         if (radix_enabled())
 115                 radix__flush_tlb_page(vma, vmaddr);
 116 }
 117 #else
 118 #define flush_tlb_mm(mm)                local_flush_tlb_mm(mm)
 119 #define flush_tlb_page(vma, addr)       local_flush_tlb_page(vma, addr)
 120 #endif /* CONFIG_SMP */
 121
 122 #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault
 123 static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma,
 124                                                 unsigned long address,
 125                                                 pte_t *ptep)
 126 {
 127         /*
 128          * Book3S 64 does not require spurious fault flushes because the PTE
 129          * must be re-fetched in case of an access permission problem. So the
 130          * only reason for a spurious fault should be concurrent modification
 131          * to the PTE, in which case the PTE will eventually be re-fetched by
 132          * the MMU when it attempts the access again.
 133          *
 134          * See: Power ISA Version 3.1B, 6.10.1.2 Modifying a Translation Table
 135          * Entry, Setting a Reference or Change Bit or Upgrading Access
 136          * Authority (PTE Subject to Atomic Hardware Updates):
 137          *
 138          * "If the only change being made to a valid PTE that is subject to
 139          *  atomic hardware updates is to set the Reference or Change bit to
 140          *  1 or to upgrade access authority, a simpler sequence suffices
 141          *  because the translation hardware will refetch the PTE if an
 142          *  access is attempted for which the only problems were reference
 143          *  and/or change bits needing to be set or insufficient access
 144          *  authority."
 145          *
 146          * The nest MMU in POWER9 does not perform this PTE re-fetch, but
 147          * it avoids the spurious fault problem by flushing the TLB before
 148          * upgrading PTE permissions, see radix__ptep_set_access_flags.
 149          */
 150 }
 151
 152 static inline bool __pte_protnone(unsigned long pte)
 153 {
 154         return (pte & (pgprot_val(PAGE_NONE) | _PAGE_RWX)) == pgprot_val(PAGE_NONE);
 155 }
 156
 157 static inline bool __pte_flags_need_flush(unsigned long oldval,
 158                                           unsigned long newval)
 159 {
 160         unsigned long delta = oldval ^ newval;
 161
 162         /*
 163          * The return value of this function doesn't matter for hash,
 164          * ptep_modify_prot_start() does a pte_update() which does or schedules
 165          * any necessary hash table update and flush.
 166          */
 167         if (!radix_enabled())
 168                 return true;
 169
 170         /*
 171          * We do not expect kernel mappings or non-PTEs or not-present PTEs.
 172          */
 173         VM_WARN_ON_ONCE(!__pte_protnone(oldval) && oldval & _PAGE_PRIVILEGED);
 174         VM_WARN_ON_ONCE(!__pte_protnone(newval) && newval & _PAGE_PRIVILEGED);
 175         VM_WARN_ON_ONCE(!(oldval & _PAGE_PTE));
 176         VM_WARN_ON_ONCE(!(newval & _PAGE_PTE));
 177         VM_WARN_ON_ONCE(!(oldval & _PAGE_PRESENT));
 178         VM_WARN_ON_ONCE(!(newval & _PAGE_PRESENT));
 179
 180         /*
 181         *  Must flush on any change except READ, WRITE, EXEC, DIRTY, ACCESSED.
 182         *
 183          * In theory, some changed software bits could be tolerated, in
 184          * practice those should rarely if ever matter.
 185          */
 186
 187         if (delta & ~(_PAGE_RWX | _PAGE_DIRTY | _PAGE_ACCESSED))
 188                 return true;
 189
 190         /*
 191          * If any of the above was present in old but cleared in new, flush.
 192          * With the exception of _PAGE_ACCESSED, don't worry about flushing
 193          * if that was cleared (see the comment in ptep_clear_flush_young()).
 194          */
 195         if ((delta & ~_PAGE_ACCESSED) & oldval)
 196                 return true;
 197
 198         return false;
 199 }
 200
 201 static inline bool pte_needs_flush(pte_t oldpte, pte_t newpte)
 202 {
 203         return __pte_flags_need_flush(pte_val(oldpte), pte_val(newpte));
 204 }
 205 #define pte_needs_flush pte_needs_flush
 206
 207 static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd)
 208 {
 209         return __pte_flags_need_flush(pmd_val(oldpmd), pmd_val(newpmd));
 210 }
 211 #define huge_pmd_needs_flush huge_pmd_needs_flush
 212
 213 extern bool tlbie_capable;
 214 extern bool tlbie_enabled;
 215
 216 static inline bool cputlb_use_tlbie(void)
 217 {
 218         return tlbie_enabled;
 219 }
 220
 221 #endif /*  _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H */