git.ipfire.org Git - thirdparty/kernel/stable-queue.git/blob

   1 From a82eee7424525e34e98d821dd059ce14560a1e35 Mon Sep 17 00:00:00 2001
   2 From: Toshi Kani <toshi.kani@hpe.com>
   3 Date: Thu, 11 Feb 2016 14:24:17 -0700
   4 Subject: x86/uaccess/64: Handle the caching of 4-byte nocache copies properly in __copy_user_nocache()
   5
   6 From: Toshi Kani <toshi.kani@hpe.com>
   7
   8 commit a82eee7424525e34e98d821dd059ce14560a1e35 upstream.
   9
  10 Data corruption issues were observed in tests which initiated
  11 a system crash/reset while accessing BTT devices.  This problem
  12 is reproducible.
  13
  14 The BTT driver calls pmem_rw_bytes() to update data in pmem
  15 devices.  This interface calls __copy_user_nocache(), which
  16 uses non-temporal stores so that the stores to pmem are
  17 persistent.
  18
  19 __copy_user_nocache() uses non-temporal stores when a request
  20 size is 8 bytes or larger (and is aligned by 8 bytes).  The
  21 BTT driver updates the BTT map table, which entry size is
  22 4 bytes.  Therefore, updates to the map table entries remain
  23 cached, and are not written to pmem after a crash.
  24
  25 Change __copy_user_nocache() to use non-temporal store when
  26 a request size is 4 bytes.  The change extends the current
  27 byte-copy path for a less-than-8-bytes request, and does not
  28 add any overhead to the regular path.
  29
  30 Reported-and-tested-by: Micah Parrish <micah.parrish@hpe.com>
  31 Reported-and-tested-by: Brian Boylston <brian.boylston@hpe.com>
  32 Signed-off-by: Toshi Kani <toshi.kani@hpe.com>
  33 Cc: Andrew Morton <akpm@linux-foundation.org>
  34 Cc: Andy Lutomirski <luto@amacapital.net>
  35 Cc: Borislav Petkov <bp@alien8.de>
  36 Cc: Borislav Petkov <bp@suse.de>
  37 Cc: Brian Gerst <brgerst@gmail.com>
  38 Cc: Dan Williams <dan.j.williams@intel.com>
  39 Cc: Denys Vlasenko <dvlasenk@redhat.com>
  40 Cc: H. Peter Anvin <hpa@zytor.com>
  41 Cc: Linus Torvalds <torvalds@linux-foundation.org>
  42 Cc: Luis R. Rodriguez <mcgrof@suse.com>
  43 Cc: Peter Zijlstra <peterz@infradead.org>
  44 Cc: Ross Zwisler <ross.zwisler@linux.intel.com>
  45 Cc: Thomas Gleixner <tglx@linutronix.de>
  46 Cc: Toshi Kani <toshi.kani@hp.com>
  47 Cc: Vishal Verma <vishal.l.verma@intel.com>
  48 Cc: linux-nvdimm@lists.01.org
  49 Link: http://lkml.kernel.org/r/1455225857-12039-3-git-send-email-toshi.kani@hpe.com
  50 [ Small readability edits. ]
  51 Signed-off-by: Ingo Molnar <mingo@kernel.org>
  52 Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
  53
  54 ---
  55  arch/x86/lib/copy_user_64.S |   36 ++++++++++++++++++++++++++++++++----
  56  1 file changed, 32 insertions(+), 4 deletions(-)
  57
  58 --- a/arch/x86/lib/copy_user_64.S
  59 +++ b/arch/x86/lib/copy_user_64.S
  60 @@ -237,13 +237,14 @@ ENDPROC(copy_user_enhanced_fast_string)
  61   * Note: Cached memory copy is used when destination or size is not
  62   * naturally aligned. That is:
  63   *  - Require 8-byte alignment when size is 8 bytes or larger.
  64 + *  - Require 4-byte alignment when size is 4 bytes.
  65   */
  66  ENTRY(__copy_user_nocache)
  67         ASM_STAC
  68
  69 -       /* If size is less than 8 bytes, go to byte copy */
  70 +       /* If size is less than 8 bytes, go to 4-byte copy */
  71         cmpl $8,%edx
  72 -       jb .L_1b_cache_copy_entry
  73 +       jb .L_4b_nocache_copy_entry
  74
  75         /* If destination is not 8-byte aligned, "cache" copy to align it */
  76         ALIGN_DESTINATION
  77 @@ -282,7 +283,7 @@ ENTRY(__copy_user_nocache)
  78         movl %edx,%ecx
  79         andl $7,%edx
  80         shrl $3,%ecx
  81 -       jz .L_1b_cache_copy_entry       /* jump if count is 0 */
  82 +       jz .L_4b_nocache_copy_entry     /* jump if count is 0 */
  83
  84         /* Perform 8-byte nocache loop-copy */
  85  .L_8b_nocache_copy_loop:
  86 @@ -294,11 +295,33 @@ ENTRY(__copy_user_nocache)
  87         jnz .L_8b_nocache_copy_loop
  88
  89         /* If no byte left, we're done */
  90 -.L_1b_cache_copy_entry:
  91 +.L_4b_nocache_copy_entry:
  92 +       andl %edx,%edx
  93 +       jz .L_finish_copy
  94 +
  95 +       /* If destination is not 4-byte aligned, go to byte copy: */
  96 +       movl %edi,%ecx
  97 +       andl $3,%ecx
  98 +       jnz .L_1b_cache_copy_entry
  99 +
 100 +       /* Set 4-byte copy count (1 or 0) and remainder */
 101 +       movl %edx,%ecx
 102 +       andl $3,%edx
 103 +       shrl $2,%ecx
 104 +       jz .L_1b_cache_copy_entry       /* jump if count is 0 */
 105 +
 106 +       /* Perform 4-byte nocache copy: */
 107 +30:    movl (%rsi),%r8d
 108 +31:    movnti %r8d,(%rdi)
 109 +       leaq 4(%rsi),%rsi
 110 +       leaq 4(%rdi),%rdi
 111 +
 112 +       /* If no bytes left, we're done: */
 113         andl %edx,%edx
 114         jz .L_finish_copy
 115
 116         /* Perform byte "cache" loop-copy for the remainder */
 117 +.L_1b_cache_copy_entry:
 118         movl %edx,%ecx
 119  .L_1b_cache_copy_loop:
 120  40:    movb (%rsi),%al
 121 @@ -323,6 +346,9 @@ ENTRY(__copy_user_nocache)
 122  .L_fixup_8b_copy:
 123         lea (%rdx,%rcx,8),%rdx
 124         jmp .L_fixup_handle_tail
 125 +.L_fixup_4b_copy:
 126 +       lea (%rdx,%rcx,4),%rdx
 127 +       jmp .L_fixup_handle_tail
 128  .L_fixup_1b_copy:
 129         movl %ecx,%edx
 130  .L_fixup_handle_tail:
 131 @@ -348,6 +374,8 @@ ENTRY(__copy_user_nocache)
 132         _ASM_EXTABLE(16b,.L_fixup_4x8b_copy)
 133         _ASM_EXTABLE(20b,.L_fixup_8b_copy)
 134         _ASM_EXTABLE(21b,.L_fixup_8b_copy)
 135 +       _ASM_EXTABLE(30b,.L_fixup_4b_copy)
 136 +       _ASM_EXTABLE(31b,.L_fixup_4b_copy)
 137         _ASM_EXTABLE(40b,.L_fixup_1b_copy)
 138         _ASM_EXTABLE(41b,.L_fixup_1b_copy)
 139  ENDPROC(__copy_user_nocache)