arch/x86/lib/copy_user_64.S

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com>
   4  * Copyright 2002 Andi Kleen, SuSE Labs.
   5  *
   6  * Functions to copy from and to user space.
   7  */
   8
   9 #include <linux/export.h>
  10 #include <linux/linkage.h>
  11 #include <asm/cpufeatures.h>
  12 #include <asm/alternative.h>
  13 #include <asm/asm.h>
  14
  15 /*
  16  * rep_movs_alternative - memory copy with exception handling.
  17  * This version is for CPUs that don't have FSRM (Fast Short Rep Movs)
  18  *
  19  * Input:
  20  * rdi destination
  21  * rsi source
  22  * rcx count
  23  *
  24  * Output:
  25  * rcx uncopied bytes or 0 if successful.
  26  *
  27  * NOTE! The calling convention is very intentionally the same as
  28  * for 'rep movs', so that we can rewrite the function call with
  29  * just a plain 'rep movs' on machines that have FSRM.  But to make
  30  * it simpler for us, we can clobber rsi/rdi and rax freely.
  31  */
  32 SYM_FUNC_START(rep_movs_alternative)
  33         cmpq $64,%rcx
  34         jae .Llarge
  35
  36         cmp $8,%ecx
  37         jae .Lword
  38
  39         testl %ecx,%ecx
  40         je .Lexit
  41
  42 .Lcopy_user_tail:
  43 0:      movb (%rsi),%al
  44 1:      movb %al,(%rdi)
  45         inc %rdi
  46         inc %rsi
  47         dec %rcx
  48         jne .Lcopy_user_tail
  49 .Lexit:
  50         RET
  51
  52         _ASM_EXTABLE_UA( 0b, .Lexit)
  53         _ASM_EXTABLE_UA( 1b, .Lexit)
  54
  55         .p2align 4
  56 .Lword:
  57 2:      movq (%rsi),%rax
  58 3:      movq %rax,(%rdi)
  59         addq $8,%rsi
  60         addq $8,%rdi
  61         sub $8,%ecx
  62         je .Lexit
  63         cmp $8,%ecx
  64         jae .Lword
  65         jmp .Lcopy_user_tail
  66
  67         _ASM_EXTABLE_UA( 2b, .Lcopy_user_tail)
  68         _ASM_EXTABLE_UA( 3b, .Lcopy_user_tail)
  69
  70 .Llarge:
  71 0:      ALTERNATIVE "jmp .Llarge_movsq", "rep movsb", X86_FEATURE_ERMS
  72 1:      RET
  73
  74         _ASM_EXTABLE_UA( 0b, 1b)
  75
  76 .Llarge_movsq:
  77         movq %rcx,%rax
  78         shrq $3,%rcx
  79         andl $7,%eax
  80 0:      rep movsq
  81         movl %eax,%ecx
  82         testl %ecx,%ecx
  83         jne .Lcopy_user_tail
  84         RET
  85
  86 1:      leaq (%rax,%rcx,8),%rcx
  87         jmp .Lcopy_user_tail
  88
  89         _ASM_EXTABLE_UA( 0b, 1b)
  90 SYM_FUNC_END(rep_movs_alternative)
  91 EXPORT_SYMBOL(rep_movs_alternative)