arch/x86/lib/msr-smp.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/export.h>
   3 #include <linux/preempt.h>
   4 #include <linux/smp.h>
   5 #include <asm/msr.h>
   6
   7 static void __rdmsr_on_cpu(void *info)
   8 {
   9         struct msr_info *rv = info;
  10         struct msr *reg;
  11         int this_cpu = raw_smp_processor_id();
  12
  13         if (rv->msrs)
  14                 reg = per_cpu_ptr(rv->msrs, this_cpu);
  15         else
  16                 reg = &rv->reg;
  17
  18         rdmsr(rv->msr_no, reg->l, reg->h);
  19 }
  20
  21 static void __wrmsr_on_cpu(void *info)
  22 {
  23         struct msr_info *rv = info;
  24         struct msr *reg;
  25         int this_cpu = raw_smp_processor_id();
  26
  27         if (rv->msrs)
  28                 reg = per_cpu_ptr(rv->msrs, this_cpu);
  29         else
  30                 reg = &rv->reg;
  31
  32         wrmsr(rv->msr_no, reg->l, reg->h);
  33 }
  34
  35 int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
  36 {
  37         int err;
  38         struct msr_info rv;
  39
  40         memset(&rv, 0, sizeof(rv));
  41
  42         rv.msr_no = msr_no;
  43         err = smp_call_function_single(cpu, __rdmsr_on_cpu, &rv, 1);
  44         *l = rv.reg.l;
  45         *h = rv.reg.h;
  46
  47         return err;
  48 }
  49 EXPORT_SYMBOL(rdmsr_on_cpu);
  50
  51 int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
  52 {
  53         int err;
  54         struct msr_info rv;
  55
  56         memset(&rv, 0, sizeof(rv));
  57
  58         rv.msr_no = msr_no;
  59         err = smp_call_function_single(cpu, __rdmsr_on_cpu, &rv, 1);
  60         *q = rv.reg.q;
  61
  62         return err;
  63 }
  64 EXPORT_SYMBOL(rdmsrl_on_cpu);
  65
  66 int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
  67 {
  68         int err;
  69         struct msr_info rv;
  70
  71         memset(&rv, 0, sizeof(rv));
  72
  73         rv.msr_no = msr_no;
  74         rv.reg.l = l;
  75         rv.reg.h = h;
  76         err = smp_call_function_single(cpu, __wrmsr_on_cpu, &rv, 1);
  77
  78         return err;
  79 }
  80 EXPORT_SYMBOL(wrmsr_on_cpu);
  81
  82 int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
  83 {
  84         int err;
  85         struct msr_info rv;
  86
  87         memset(&rv, 0, sizeof(rv));
  88
  89         rv.msr_no = msr_no;
  90         rv.reg.q = q;
  91
  92         err = smp_call_function_single(cpu, __wrmsr_on_cpu, &rv, 1);
  93
  94         return err;
  95 }
  96 EXPORT_SYMBOL(wrmsrl_on_cpu);
  97
  98 static void __rwmsr_on_cpus(const struct cpumask *mask, u32 msr_no,
  99                             struct msr *msrs,
 100                             void (*msr_func) (void *info))
 101 {
 102         struct msr_info rv;
 103         int this_cpu;
 104
 105         memset(&rv, 0, sizeof(rv));
 106
 107         rv.msrs   = msrs;
 108         rv.msr_no = msr_no;
 109
 110         this_cpu = get_cpu();
 111
 112         if (cpumask_test_cpu(this_cpu, mask))
 113                 msr_func(&rv);
 114
 115         smp_call_function_many(mask, msr_func, &rv, 1);
 116         put_cpu();
 117 }
 118
 119 /* rdmsr on a bunch of CPUs
 120  *
 121  * @mask:       which CPUs
 122  * @msr_no:     which MSR
 123  * @msrs:       array of MSR values
 124  *
 125  */
 126 void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs)
 127 {
 128         __rwmsr_on_cpus(mask, msr_no, msrs, __rdmsr_on_cpu);
 129 }
 130 EXPORT_SYMBOL(rdmsr_on_cpus);
 131
 132 /*
 133  * wrmsr on a bunch of CPUs
 134  *
 135  * @mask:       which CPUs
 136  * @msr_no:     which MSR
 137  * @msrs:       array of MSR values
 138  *
 139  */
 140 void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs)
 141 {
 142         __rwmsr_on_cpus(mask, msr_no, msrs, __wrmsr_on_cpu);
 143 }
 144 EXPORT_SYMBOL(wrmsr_on_cpus);
 145
 146 /* These "safe" variants are slower and should be used when the target MSR
 147    may not actually exist. */
 148 static void __rdmsr_safe_on_cpu(void *info)
 149 {
 150         struct msr_info *rv = info;
 151
 152         rv->err = rdmsr_safe(rv->msr_no, &rv->reg.l, &rv->reg.h);
 153 }
 154
 155 static void __wrmsr_safe_on_cpu(void *info)
 156 {
 157         struct msr_info *rv = info;
 158
 159         rv->err = wrmsr_safe(rv->msr_no, rv->reg.l, rv->reg.h);
 160 }
 161
 162 int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
 163 {
 164         int err;
 165         struct msr_info rv;
 166
 167         memset(&rv, 0, sizeof(rv));
 168
 169         rv.msr_no = msr_no;
 170         err = smp_call_function_single(cpu, __rdmsr_safe_on_cpu, &rv, 1);
 171         *l = rv.reg.l;
 172         *h = rv.reg.h;
 173
 174         return err ? err : rv.err;
 175 }
 176 EXPORT_SYMBOL(rdmsr_safe_on_cpu);
 177
 178 int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
 179 {
 180         int err;
 181         struct msr_info rv;
 182
 183         memset(&rv, 0, sizeof(rv));
 184
 185         rv.msr_no = msr_no;
 186         rv.reg.l = l;
 187         rv.reg.h = h;
 188         err = smp_call_function_single(cpu, __wrmsr_safe_on_cpu, &rv, 1);
 189
 190         return err ? err : rv.err;
 191 }
 192 EXPORT_SYMBOL(wrmsr_safe_on_cpu);
 193
 194 int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
 195 {
 196         int err;
 197         struct msr_info rv;
 198
 199         memset(&rv, 0, sizeof(rv));
 200
 201         rv.msr_no = msr_no;
 202         rv.reg.q = q;
 203
 204         err = smp_call_function_single(cpu, __wrmsr_safe_on_cpu, &rv, 1);
 205
 206         return err ? err : rv.err;
 207 }
 208 EXPORT_SYMBOL(wrmsrl_safe_on_cpu);
 209
 210 int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
 211 {
 212         int err;
 213         struct msr_info rv;
 214
 215         memset(&rv, 0, sizeof(rv));
 216
 217         rv.msr_no = msr_no;
 218         err = smp_call_function_single(cpu, __rdmsr_safe_on_cpu, &rv, 1);
 219         *q = rv.reg.q;
 220
 221         return err ? err : rv.err;
 222 }
 223 EXPORT_SYMBOL(rdmsrl_safe_on_cpu);
 224
 225 /*
 226  * These variants are significantly slower, but allows control over
 227  * the entire 32-bit GPR set.
 228  */
 229 static void __rdmsr_safe_regs_on_cpu(void *info)
 230 {
 231         struct msr_regs_info *rv = info;
 232
 233         rv->err = rdmsr_safe_regs(rv->regs);
 234 }
 235
 236 static void __wrmsr_safe_regs_on_cpu(void *info)
 237 {
 238         struct msr_regs_info *rv = info;
 239
 240         rv->err = wrmsr_safe_regs(rv->regs);
 241 }
 242
 243 int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 *regs)
 244 {
 245         int err;
 246         struct msr_regs_info rv;
 247
 248         rv.regs   = regs;
 249         rv.err    = -EIO;
 250         err = smp_call_function_single(cpu, __rdmsr_safe_regs_on_cpu, &rv, 1);
 251
 252         return err ? err : rv.err;
 253 }
 254 EXPORT_SYMBOL(rdmsr_safe_regs_on_cpu);
 255
 256 int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 *regs)
 257 {
 258         int err;
 259         struct msr_regs_info rv;
 260
 261         rv.regs = regs;
 262         rv.err  = -EIO;
 263         err = smp_call_function_single(cpu, __wrmsr_safe_regs_on_cpu, &rv, 1);
 264
 265         return err ? err : rv.err;
 266 }
 267 EXPORT_SYMBOL(wrmsr_safe_regs_on_cpu);