arch/x86/hyperv/mmu.c

   1 #define pr_fmt(fmt)  "Hyper-V: " fmt
   2
   3 #include <linux/hyperv.h>
   4 #include <linux/log2.h>
   5 #include <linux/slab.h>
   6 #include <linux/types.h>
   7
   8 #include <asm/fpu/api.h>
   9 #include <asm/mshyperv.h>
  10 #include <asm/msr.h>
  11 #include <asm/tlbflush.h>
  12
  13 #define CREATE_TRACE_POINTS
  14 #include <asm/trace/hyperv.h>
  15
  16 /* HvFlushVirtualAddressSpace, HvFlushVirtualAddressList hypercalls */
  17 struct hv_flush_pcpu {
  18         u64 address_space;
  19         u64 flags;
  20         u64 processor_mask;
  21         u64 gva_list[];
  22 };
  23
  24 /* HvFlushVirtualAddressSpaceEx, HvFlushVirtualAddressListEx hypercalls */
  25 struct hv_flush_pcpu_ex {
  26         u64 address_space;
  27         u64 flags;
  28         struct {
  29                 u64 format;
  30                 u64 valid_bank_mask;
  31                 u64 bank_contents[];
  32         } hv_vp_set;
  33         u64 gva_list[];
  34 };
  35
  36 /* Each gva in gva_list encodes up to 4096 pages to flush */
  37 #define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE)
  38
  39 static struct hv_flush_pcpu __percpu *pcpu_flush;
  40
  41 static struct hv_flush_pcpu_ex __percpu *pcpu_flush_ex;
  42
  43 /*
  44  * Fills in gva_list starting from offset. Returns the number of items added.
  45  */
  46 static inline int fill_gva_list(u64 gva_list[], int offset,
  47                                 unsigned long start, unsigned long end)
  48 {
  49         int gva_n = offset;
  50         unsigned long cur = start, diff;
  51
  52         do {
  53                 diff = end > cur ? end - cur : 0;
  54
  55                 gva_list[gva_n] = cur & PAGE_MASK;
  56                 /*
  57                  * Lower 12 bits encode the number of additional
  58                  * pages to flush (in addition to the 'cur' page).
  59                  */
  60                 if (diff >= HV_TLB_FLUSH_UNIT)
  61                         gva_list[gva_n] |= ~PAGE_MASK;
  62                 else if (diff)
  63                         gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT;
  64
  65                 cur += HV_TLB_FLUSH_UNIT;
  66                 gva_n++;
  67
  68         } while (cur < end);
  69
  70         return gva_n - offset;
  71 }
  72
  73 /* Return the number of banks in the resulting vp_set */
  74 static inline int cpumask_to_vp_set(struct hv_flush_pcpu_ex *flush,
  75                                     const struct cpumask *cpus)
  76 {
  77         int cpu, vcpu, vcpu_bank, vcpu_offset, nr_bank = 1;
  78
  79         /* valid_bank_mask can represent up to 64 banks */
  80         if (hv_max_vp_index / 64 >= 64)
  81                 return 0;
  82
  83         /*
  84          * Clear all banks up to the maximum possible bank as hv_flush_pcpu_ex
  85          * structs are not cleared between calls, we risk flushing unneeded
  86          * vCPUs otherwise.
  87          */
  88         for (vcpu_bank = 0; vcpu_bank <= hv_max_vp_index / 64; vcpu_bank++)
  89                 flush->hv_vp_set.bank_contents[vcpu_bank] = 0;
  90
  91         /*
  92          * Some banks may end up being empty but this is acceptable.
  93          */
  94         for_each_cpu(cpu, cpus) {
  95                 vcpu = hv_cpu_number_to_vp_number(cpu);
  96                 vcpu_bank = vcpu / 64;
  97                 vcpu_offset = vcpu % 64;
  98                 __set_bit(vcpu_offset, (unsigned long *)
  99                           &flush->hv_vp_set.bank_contents[vcpu_bank]);
 100                 if (vcpu_bank >= nr_bank)
 101                         nr_bank = vcpu_bank + 1;
 102         }
 103         flush->hv_vp_set.valid_bank_mask = GENMASK_ULL(nr_bank - 1, 0);
 104
 105         return nr_bank;
 106 }
 107
 108 static void hyperv_flush_tlb_others(const struct cpumask *cpus,
 109                                     const struct flush_tlb_info *info)
 110 {
 111         int cpu, vcpu, gva_n, max_gvas;
 112         struct hv_flush_pcpu *flush;
 113         u64 status = U64_MAX;
 114         unsigned long flags;
 115
 116         trace_hyperv_mmu_flush_tlb_others(cpus, info);
 117
 118         if (!pcpu_flush || !hv_hypercall_pg)
 119                 goto do_native;
 120
 121         if (cpumask_empty(cpus))
 122                 return;
 123
 124         local_irq_save(flags);
 125
 126         flush = this_cpu_ptr(pcpu_flush);
 127
 128         if (info->mm) {
 129                 flush->address_space = virt_to_phys(info->mm->pgd);
 130                 flush->flags = 0;
 131         } else {
 132                 flush->address_space = 0;
 133                 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
 134         }
 135
 136         flush->processor_mask = 0;
 137         if (cpumask_equal(cpus, cpu_present_mask)) {
 138                 flush->flags |= HV_FLUSH_ALL_PROCESSORS;
 139         } else {
 140                 for_each_cpu(cpu, cpus) {
 141                         vcpu = hv_cpu_number_to_vp_number(cpu);
 142                         if (vcpu >= 64)
 143                                 goto do_native;
 144
 145                         __set_bit(vcpu, (unsigned long *)
 146                                   &flush->processor_mask);
 147                 }
 148         }
 149
 150         /*
 151          * We can flush not more than max_gvas with one hypercall. Flush the
 152          * whole address space if we were asked to do more.
 153          */
 154         max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]);
 155
 156         if (info->end == TLB_FLUSH_ALL) {
 157                 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
 158                 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
 159                                          flush, NULL);
 160         } else if (info->end &&
 161                    ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
 162                 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
 163                                          flush, NULL);
 164         } else {
 165                 gva_n = fill_gva_list(flush->gva_list, 0,
 166                                       info->start, info->end);
 167                 status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST,
 168                                              gva_n, 0, flush, NULL);
 169         }
 170
 171         local_irq_restore(flags);
 172
 173         if (!(status & HV_HYPERCALL_RESULT_MASK))
 174                 return;
 175 do_native:
 176         native_flush_tlb_others(cpus, info);
 177 }
 178
 179 static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
 180                                        const struct flush_tlb_info *info)
 181 {
 182         int nr_bank = 0, max_gvas, gva_n;
 183         struct hv_flush_pcpu_ex *flush;
 184         u64 status = U64_MAX;
 185         unsigned long flags;
 186
 187         trace_hyperv_mmu_flush_tlb_others(cpus, info);
 188
 189         if (!pcpu_flush_ex || !hv_hypercall_pg)
 190                 goto do_native;
 191
 192         if (cpumask_empty(cpus))
 193                 return;
 194
 195         local_irq_save(flags);
 196
 197         flush = this_cpu_ptr(pcpu_flush_ex);
 198
 199         if (info->mm) {
 200                 flush->address_space = virt_to_phys(info->mm->pgd);
 201                 flush->flags = 0;
 202         } else {
 203                 flush->address_space = 0;
 204                 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
 205         }
 206
 207         flush->hv_vp_set.valid_bank_mask = 0;
 208
 209         if (!cpumask_equal(cpus, cpu_present_mask)) {
 210                 flush->hv_vp_set.format = HV_GENERIC_SET_SPARCE_4K;
 211                 nr_bank = cpumask_to_vp_set(flush, cpus);
 212         }
 213
 214         if (!nr_bank) {
 215                 flush->hv_vp_set.format = HV_GENERIC_SET_ALL;
 216                 flush->flags |= HV_FLUSH_ALL_PROCESSORS;
 217         }
 218
 219         /*
 220          * We can flush not more than max_gvas with one hypercall. Flush the
 221          * whole address space if we were asked to do more.
 222          */
 223         max_gvas =
 224                 (PAGE_SIZE - sizeof(*flush) - nr_bank *
 225                  sizeof(flush->hv_vp_set.bank_contents[0])) /
 226                 sizeof(flush->gva_list[0]);
 227
 228         if (info->end == TLB_FLUSH_ALL) {
 229                 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
 230                 status = hv_do_rep_hypercall(
 231                         HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
 232                         0, nr_bank + 2, flush, NULL);
 233         } else if (info->end &&
 234                    ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
 235                 status = hv_do_rep_hypercall(
 236                         HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
 237                         0, nr_bank + 2, flush, NULL);
 238         } else {
 239                 gva_n = fill_gva_list(flush->gva_list, nr_bank,
 240                                       info->start, info->end);
 241                 status = hv_do_rep_hypercall(
 242                         HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX,
 243                         gva_n, nr_bank + 2, flush, NULL);
 244         }
 245
 246         local_irq_restore(flags);
 247
 248         if (!(status & HV_HYPERCALL_RESULT_MASK))
 249                 return;
 250 do_native:
 251         native_flush_tlb_others(cpus, info);
 252 }
 253
 254 void hyperv_setup_mmu_ops(void)
 255 {
 256         if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED))
 257                 return;
 258
 259         setup_clear_cpu_cap(X86_FEATURE_PCID);
 260
 261         if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) {
 262                 pr_info("Using hypercall for remote TLB flush\n");
 263                 pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others;
 264         } else {
 265                 pr_info("Using ext hypercall for remote TLB flush\n");
 266                 pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others_ex;
 267         }
 268 }
 269
 270 void hyper_alloc_mmu(void)
 271 {
 272         if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED))
 273                 return;
 274
 275         if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
 276                 pcpu_flush = __alloc_percpu(PAGE_SIZE, PAGE_SIZE);
 277         else
 278                 pcpu_flush_ex = __alloc_percpu(PAGE_SIZE, PAGE_SIZE);
 279 }