1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Paravirtualization interfaces
3 Copyright (C) 2006 Rusty Russell IBM Corporation
6 2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc
9 #include <linux/errno.h>
10 #include <linux/init.h>
11 #include <linux/export.h>
12 #include <linux/efi.h>
13 #include <linux/bcd.h>
14 #include <linux/highmem.h>
15 #include <linux/kprobes.h>
16 #include <linux/pgtable.h>
17 #include <linux/static_call.h>
20 #include <asm/paravirt.h>
21 #include <asm/debugreg.h>
23 #include <asm/setup.h>
25 #include <asm/pgalloc.h>
27 #include <asm/delay.h>
28 #include <asm/fixmap.h>
30 #include <asm/tlbflush.h>
31 #include <asm/timer.h>
32 #include <asm/special_insns.h>
34 #include <asm/io_bitmap.h>
35 #include <asm/gsseg.h>
38 * nop stub, which must not clobber anything *including the stack* to
39 * avoid confusing the entry prologues.
41 DEFINE_PARAVIRT_ASM(_paravirt_nop
, "", .entry
.text
);
43 /* stub always returning 0. */
44 DEFINE_PARAVIRT_ASM(paravirt_ret0
, "xor %eax,%eax", .entry
.text
);
46 void __init
default_banner(void)
48 printk(KERN_INFO
"Booting paravirtualized kernel on %s\n",
52 /* Undefined instruction for dealing with missing ops pointers. */
53 noinstr
void paravirt_BUG(void)
58 static unsigned paravirt_patch_call(void *insn_buff
, const void *target
,
59 unsigned long addr
, unsigned len
)
61 __text_gen_insn(insn_buff
, CALL_INSN_OPCODE
,
62 (void *)addr
, target
, CALL_INSN_SIZE
);
63 return CALL_INSN_SIZE
;
66 #ifdef CONFIG_PARAVIRT_XXL
67 DEFINE_PARAVIRT_ASM(_paravirt_ident_64
, "mov %rdi, %rax", .text
);
68 DEFINE_PARAVIRT_ASM(pv_native_save_fl
, "pushf; pop %rax", .noinstr
.text
);
69 DEFINE_PARAVIRT_ASM(pv_native_irq_disable
, "cli", .noinstr
.text
);
70 DEFINE_PARAVIRT_ASM(pv_native_irq_enable
, "sti", .noinstr
.text
);
71 DEFINE_PARAVIRT_ASM(pv_native_read_cr2
, "mov %cr2, %rax", .noinstr
.text
);
74 DEFINE_STATIC_KEY_TRUE(virt_spin_lock_key
);
76 void __init
native_pv_lock_init(void)
78 if (IS_ENABLED(CONFIG_PARAVIRT_SPINLOCKS
) &&
79 !boot_cpu_has(X86_FEATURE_HYPERVISOR
))
80 static_branch_disable(&virt_spin_lock_key
);
83 static void native_tlb_remove_table(struct mmu_gather
*tlb
, void *table
)
85 tlb_remove_page(tlb
, table
);
88 unsigned int paravirt_patch(u8 type
, void *insn_buff
, unsigned long addr
,
92 * Neat trick to map patch type back to the call within the
93 * corresponding structure.
95 void *opfunc
= *((void **)&pv_ops
+ type
);
99 /* If there's no function, patch it with paravirt_BUG() */
100 ret
= paravirt_patch_call(insn_buff
, paravirt_BUG
, addr
, len
);
101 else if (opfunc
== _paravirt_nop
)
104 /* Otherwise call the function. */
105 ret
= paravirt_patch_call(insn_buff
, opfunc
, addr
, len
);
110 struct static_key paravirt_steal_enabled
;
111 struct static_key paravirt_steal_rq_enabled
;
113 static u64
native_steal_clock(int cpu
)
118 DEFINE_STATIC_CALL(pv_steal_clock
, native_steal_clock
);
119 DEFINE_STATIC_CALL(pv_sched_clock
, native_sched_clock
);
121 void paravirt_set_sched_clock(u64 (*func
)(void))
123 static_call_update(pv_sched_clock
, func
);
126 /* These are in entry.S */
127 static struct resource reserve_ioports
= {
129 .end
= IO_SPACE_LIMIT
,
130 .name
= "paravirt-ioport",
131 .flags
= IORESOURCE_IO
| IORESOURCE_BUSY
,
135 * Reserve the whole legacy IO space to prevent any legacy drivers
136 * from wasting time probing for their hardware. This is a fairly
137 * brute-force approach to disabling all non-virtual drivers.
139 * Note that this must be called very early to have any effect.
141 int paravirt_disable_iospace(void)
143 return request_resource(&ioport_resource
, &reserve_ioports
);
146 #ifdef CONFIG_PARAVIRT_XXL
147 static noinstr
void pv_native_write_cr2(unsigned long val
)
149 native_write_cr2(val
);
152 static noinstr
unsigned long pv_native_get_debugreg(int regno
)
154 return native_get_debugreg(regno
);
157 static noinstr
void pv_native_set_debugreg(int regno
, unsigned long val
)
159 native_set_debugreg(regno
, val
);
162 noinstr
void pv_native_wbinvd(void)
167 static noinstr
void pv_native_safe_halt(void)
173 struct pv_info pv_info
= {
174 .name
= "bare hardware",
175 #ifdef CONFIG_PARAVIRT_XXL
176 .extra_user_64bit_cs
= __USER_CS
,
180 /* 64-bit pagetable entries */
181 #define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64)
183 struct paravirt_patch_template pv_ops
= {
185 .cpu
.io_delay
= native_io_delay
,
187 #ifdef CONFIG_PARAVIRT_XXL
188 .cpu
.cpuid
= native_cpuid
,
189 .cpu
.get_debugreg
= pv_native_get_debugreg
,
190 .cpu
.set_debugreg
= pv_native_set_debugreg
,
191 .cpu
.read_cr0
= native_read_cr0
,
192 .cpu
.write_cr0
= native_write_cr0
,
193 .cpu
.write_cr4
= native_write_cr4
,
194 .cpu
.wbinvd
= pv_native_wbinvd
,
195 .cpu
.read_msr
= native_read_msr
,
196 .cpu
.write_msr
= native_write_msr
,
197 .cpu
.read_msr_safe
= native_read_msr_safe
,
198 .cpu
.write_msr_safe
= native_write_msr_safe
,
199 .cpu
.read_pmc
= native_read_pmc
,
200 .cpu
.load_tr_desc
= native_load_tr_desc
,
201 .cpu
.set_ldt
= native_set_ldt
,
202 .cpu
.load_gdt
= native_load_gdt
,
203 .cpu
.load_idt
= native_load_idt
,
204 .cpu
.store_tr
= native_store_tr
,
205 .cpu
.load_tls
= native_load_tls
,
206 .cpu
.load_gs_index
= native_load_gs_index
,
207 .cpu
.write_ldt_entry
= native_write_ldt_entry
,
208 .cpu
.write_gdt_entry
= native_write_gdt_entry
,
209 .cpu
.write_idt_entry
= native_write_idt_entry
,
211 .cpu
.alloc_ldt
= paravirt_nop
,
212 .cpu
.free_ldt
= paravirt_nop
,
214 .cpu
.load_sp0
= native_load_sp0
,
216 #ifdef CONFIG_X86_IOPL_IOPERM
217 .cpu
.invalidate_io_bitmap
= native_tss_invalidate_io_bitmap
,
218 .cpu
.update_io_bitmap
= native_tss_update_io_bitmap
,
221 .cpu
.start_context_switch
= paravirt_nop
,
222 .cpu
.end_context_switch
= paravirt_nop
,
225 .irq
.save_fl
= __PV_IS_CALLEE_SAVE(pv_native_save_fl
),
226 .irq
.irq_disable
= __PV_IS_CALLEE_SAVE(pv_native_irq_disable
),
227 .irq
.irq_enable
= __PV_IS_CALLEE_SAVE(pv_native_irq_enable
),
228 .irq
.safe_halt
= pv_native_safe_halt
,
229 .irq
.halt
= native_halt
,
230 #endif /* CONFIG_PARAVIRT_XXL */
233 .mmu
.flush_tlb_user
= native_flush_tlb_local
,
234 .mmu
.flush_tlb_kernel
= native_flush_tlb_global
,
235 .mmu
.flush_tlb_one_user
= native_flush_tlb_one_user
,
236 .mmu
.flush_tlb_multi
= native_flush_tlb_multi
,
237 .mmu
.tlb_remove_table
= native_tlb_remove_table
,
239 .mmu
.exit_mmap
= paravirt_nop
,
240 .mmu
.notify_page_enc_status_changed
= paravirt_nop
,
242 #ifdef CONFIG_PARAVIRT_XXL
243 .mmu
.read_cr2
= __PV_IS_CALLEE_SAVE(pv_native_read_cr2
),
244 .mmu
.write_cr2
= pv_native_write_cr2
,
245 .mmu
.read_cr3
= __native_read_cr3
,
246 .mmu
.write_cr3
= native_write_cr3
,
248 .mmu
.pgd_alloc
= __paravirt_pgd_alloc
,
249 .mmu
.pgd_free
= paravirt_nop
,
251 .mmu
.alloc_pte
= paravirt_nop
,
252 .mmu
.alloc_pmd
= paravirt_nop
,
253 .mmu
.alloc_pud
= paravirt_nop
,
254 .mmu
.alloc_p4d
= paravirt_nop
,
255 .mmu
.release_pte
= paravirt_nop
,
256 .mmu
.release_pmd
= paravirt_nop
,
257 .mmu
.release_pud
= paravirt_nop
,
258 .mmu
.release_p4d
= paravirt_nop
,
260 .mmu
.set_pte
= native_set_pte
,
261 .mmu
.set_pmd
= native_set_pmd
,
263 .mmu
.ptep_modify_prot_start
= __ptep_modify_prot_start
,
264 .mmu
.ptep_modify_prot_commit
= __ptep_modify_prot_commit
,
266 .mmu
.set_pud
= native_set_pud
,
268 .mmu
.pmd_val
= PTE_IDENT
,
269 .mmu
.make_pmd
= PTE_IDENT
,
271 .mmu
.pud_val
= PTE_IDENT
,
272 .mmu
.make_pud
= PTE_IDENT
,
274 .mmu
.set_p4d
= native_set_p4d
,
276 #if CONFIG_PGTABLE_LEVELS >= 5
277 .mmu
.p4d_val
= PTE_IDENT
,
278 .mmu
.make_p4d
= PTE_IDENT
,
280 .mmu
.set_pgd
= native_set_pgd
,
281 #endif /* CONFIG_PGTABLE_LEVELS >= 5 */
283 .mmu
.pte_val
= PTE_IDENT
,
284 .mmu
.pgd_val
= PTE_IDENT
,
286 .mmu
.make_pte
= PTE_IDENT
,
287 .mmu
.make_pgd
= PTE_IDENT
,
289 .mmu
.enter_mmap
= paravirt_nop
,
292 .enter
= paravirt_nop
,
293 .leave
= paravirt_nop
,
294 .flush
= paravirt_nop
,
297 .mmu
.set_fixmap
= native_set_fixmap
,
298 #endif /* CONFIG_PARAVIRT_XXL */
300 #if defined(CONFIG_PARAVIRT_SPINLOCKS)
303 .lock
.queued_spin_lock_slowpath
= native_queued_spin_lock_slowpath
,
304 .lock
.queued_spin_unlock
=
305 PV_CALLEE_SAVE(__native_queued_spin_unlock
),
306 .lock
.wait
= paravirt_nop
,
307 .lock
.kick
= paravirt_nop
,
308 .lock
.vcpu_is_preempted
=
309 PV_CALLEE_SAVE(__native_vcpu_is_preempted
),
314 #ifdef CONFIG_PARAVIRT_XXL
315 NOKPROBE_SYMBOL(native_load_idt
);
318 EXPORT_SYMBOL(pv_ops
);
319 EXPORT_SYMBOL_GPL(pv_info
);