git.ipfire.org Git - thirdparty/kernel/stable-queue.git/blob

   1 From 928a4c39484281f8ca366f53a1db79330d058401 Mon Sep 17 00:00:00 2001
   2 From: David Woodhouse <dwmw@amazon.co.uk>
   3 Date: Sat, 10 Feb 2018 23:39:24 +0000
   4 Subject: KVM/x86: Reduce retpoline performance impact in slot_handle_level_range(), by always inlining iterator helper methods
   5 MIME-Version: 1.0
   6 Content-Type: text/plain; charset=UTF-8
   7 Content-Transfer-Encoding: 8bit
   8
   9 From: David Woodhouse <dwmw@amazon.co.uk>
  10
  11 commit 928a4c39484281f8ca366f53a1db79330d058401 upstream.
  12
  13 With retpoline, tight loops of "call this function for every XXX" are
  14 very much pessimised by taking a prediction miss *every* time. This one
  15 is by far the biggest contributor to the guest launch time with retpoline.
  16
  17 By marking the iterator slot_handle_…() functions always_inline, we can
  18 ensure that the indirect function call can be optimised away into a
  19 direct call and it actually generates slightly smaller code because
  20 some of the other conditionals can get optimised away too.
  21
  22 Performance is now pretty close to what we see with nospectre_v2 on
  23 the command line.
  24
  25 Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
  26 Tested-by: Filippo Sironi <sironi@amazon.de>
  27 Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
  28 Reviewed-by: Filippo Sironi <sironi@amazon.de>
  29 Acked-by: Paolo Bonzini <pbonzini@redhat.com>
  30 Cc: Andy Lutomirski <luto@kernel.org>
  31 Cc: Arjan van de Ven <arjan@linux.intel.com>
  32 Cc: Borislav Petkov <bp@alien8.de>
  33 Cc: Dan Williams <dan.j.williams@intel.com>
  34 Cc: Dave Hansen <dave.hansen@linux.intel.com>
  35 Cc: David Woodhouse <dwmw2@infradead.org>
  36 Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
  37 Cc: Josh Poimboeuf <jpoimboe@redhat.com>
  38 Cc: Peter Zijlstra <peterz@infradead.org>
  39 Cc: Thomas Gleixner <tglx@linutronix.de>
  40 Cc: arjan.van.de.ven@intel.com
  41 Cc: dave.hansen@intel.com
  42 Cc: jmattson@google.com
  43 Cc: karahmed@amazon.de
  44 Cc: kvm@vger.kernel.org
  45 Cc: rkrcmar@redhat.com
  46 Link: http://lkml.kernel.org/r/1518305967-31356-4-git-send-email-dwmw@amazon.co.uk
  47 Signed-off-by: Ingo Molnar <mingo@kernel.org>
  48 Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
  49
  50 ---
  51  arch/x86/kvm/mmu.c |   10 +++++-----
  52  1 file changed, 5 insertions(+), 5 deletions(-)
  53
  54 --- a/arch/x86/kvm/mmu.c
  55 +++ b/arch/x86/kvm/mmu.c
  56 @@ -5063,7 +5063,7 @@ void kvm_mmu_uninit_vm(struct kvm *kvm)
  57  typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head);
  58
  59  /* The caller should hold mmu-lock before calling this function. */
  60 -static bool
  61 +static __always_inline bool
  62  slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
  63                         slot_level_handler fn, int start_level, int end_level,
  64                         gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
  65 @@ -5093,7 +5093,7 @@ slot_handle_level_range(struct kvm *kvm,
  66         return flush;
  67  }
  68
  69 -static bool
  70 +static __always_inline bool
  71  slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
  72                   slot_level_handler fn, int start_level, int end_level,
  73                   bool lock_flush_tlb)
  74 @@ -5104,7 +5104,7 @@ slot_handle_level(struct kvm *kvm, struc
  75                         lock_flush_tlb);
  76  }
  77
  78 -static bool
  79 +static __always_inline bool
  80  slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
  81                       slot_level_handler fn, bool lock_flush_tlb)
  82  {
  83 @@ -5112,7 +5112,7 @@ slot_handle_all_level(struct kvm *kvm, s
  84                                  PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
  85  }
  86
  87 -static bool
  88 +static __always_inline bool
  89  slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
  90                         slot_level_handler fn, bool lock_flush_tlb)
  91  {
  92 @@ -5120,7 +5120,7 @@ slot_handle_large_level(struct kvm *kvm,
  93                                  PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
  94  }
  95
  96 -static bool
  97 +static __always_inline bool
  98  slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
  99                  slot_level_handler fn, bool lock_flush_tlb)
 100  {