arch/x86/lib/copy_user_64.S

   1 /*
   2  * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com>
   3  * Copyright 2002 Andi Kleen, SuSE Labs.
   4  * Subject to the GNU Public License v2.
   5  *
   6  * Functions to copy from and to user space.
   7  */
   8
   9 #include <linux/linkage.h>
  10 #include <asm/dwarf2.h>
  11
  12 #define FIX_ALIGNMENT 1
  13
  14 #include <asm/current.h>
  15 #include <asm/asm-offsets.h>
  16 #include <asm/thread_info.h>
  17 #include <asm/cpufeatures.h>
  18 #include <asm/alternative-asm.h>
  19 #include <asm/asm.h>
  20 #include <asm/smap.h>
  21
  22 /*
  23  * By placing feature2 after feature1 in altinstructions section, we logically
  24  * implement:
  25  * If CPU has feature2, jmp to alt2 is used
  26  * else if CPU has feature1, jmp to alt1 is used
  27  * else jmp to orig is used.
  28  */
  29         .macro ALTERNATIVE_JUMP feature1,feature2,orig,alt1,alt2
  30 0:
  31         jmp \orig
  32 1:
  33         .section .altinstr_replacement,"ax"
  34 2:
  35         jmp \alt1
  36 3:
  37         jmp \alt2
  38         .previous
  39
  40         .section .altinstructions,"a"
  41         altinstruction_entry 0b,2b,\feature1,5,5,0
  42         altinstruction_entry 0b,3b,\feature2,5,5,0
  43         .previous
  44         .endm
  45
  46         .macro ALIGN_DESTINATION
  47 #ifdef FIX_ALIGNMENT
  48         /* check for bad alignment of destination */
  49         movl %edi,%ecx
  50         andl $7,%ecx
  51         jz 102f                         /* already aligned */
  52         subl $8,%ecx
  53         negl %ecx
  54         subl %ecx,%edx
  55 100:    movb (%rsi),%al
  56 101:    movb %al,(%rdi)
  57         incq %rsi
  58         incq %rdi
  59         decl %ecx
  60         jnz 100b
  61 102:
  62         .section .fixup,"ax"
  63 103:    addl %ecx,%edx                  /* ecx is zerorest also */
  64         jmp copy_user_handle_tail
  65         .previous
  66
  67         _ASM_EXTABLE(100b,103b)
  68         _ASM_EXTABLE(101b,103b)
  69 #endif
  70         .endm
  71
  72 /* Standard copy_to_user with segment limit checking */
  73 ENTRY(_copy_to_user)
  74         CFI_STARTPROC
  75         GET_THREAD_INFO(%rax)
  76         movq %rdi,%rcx
  77         addq %rdx,%rcx
  78         jc bad_to_user
  79         cmpq TI_addr_limit(%rax),%rcx
  80         ja bad_to_user
  81         ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \
  82                 copy_user_generic_unrolled,copy_user_generic_string,    \
  83                 copy_user_enhanced_fast_string
  84         CFI_ENDPROC
  85 ENDPROC(_copy_to_user)
  86
  87 /* Standard copy_from_user with segment limit checking */
  88 ENTRY(_copy_from_user)
  89         CFI_STARTPROC
  90         GET_THREAD_INFO(%rax)
  91         movq %rsi,%rcx
  92         addq %rdx,%rcx
  93         jc bad_from_user
  94         cmpq TI_addr_limit(%rax),%rcx
  95         ja bad_from_user
  96         ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \
  97                 copy_user_generic_unrolled,copy_user_generic_string,    \
  98                 copy_user_enhanced_fast_string
  99         CFI_ENDPROC
 100 ENDPROC(_copy_from_user)
 101
 102         .section .fixup,"ax"
 103         /* must zero dest */
 104 ENTRY(bad_from_user)
 105 bad_from_user:
 106         CFI_STARTPROC
 107         movl %edx,%ecx
 108         xorl %eax,%eax
 109         rep
 110         stosb
 111 bad_to_user:
 112         movl %edx,%eax
 113         ret
 114         CFI_ENDPROC
 115 ENDPROC(bad_from_user)
 116         .previous
 117
 118 /*
 119  * copy_user_generic_unrolled - memory copy with exception handling.
 120  * This version is for CPUs like P4 that don't have efficient micro
 121  * code for rep movsq
 122  *
 123  * Input:
 124  * rdi destination
 125  * rsi source
 126  * rdx count
 127  *
 128  * Output:
 129  * eax uncopied bytes or 0 if successful.
 130  */
 131 ENTRY(copy_user_generic_unrolled)
 132         CFI_STARTPROC
 133         ASM_STAC
 134         cmpl $8,%edx
 135         jb 20f          /* less then 8 bytes, go to byte copy loop */
 136         ALIGN_DESTINATION
 137         movl %edx,%ecx
 138         andl $63,%edx
 139         shrl $6,%ecx
 140         jz 17f
 141 1:      movq (%rsi),%r8
 142 2:      movq 1*8(%rsi),%r9
 143 3:      movq 2*8(%rsi),%r10
 144 4:      movq 3*8(%rsi),%r11
 145 5:      movq %r8,(%rdi)
 146 6:      movq %r9,1*8(%rdi)
 147 7:      movq %r10,2*8(%rdi)
 148 8:      movq %r11,3*8(%rdi)
 149 9:      movq 4*8(%rsi),%r8
 150 10:     movq 5*8(%rsi),%r9
 151 11:     movq 6*8(%rsi),%r10
 152 12:     movq 7*8(%rsi),%r11
 153 13:     movq %r8,4*8(%rdi)
 154 14:     movq %r9,5*8(%rdi)
 155 15:     movq %r10,6*8(%rdi)
 156 16:     movq %r11,7*8(%rdi)
 157         leaq 64(%rsi),%rsi
 158         leaq 64(%rdi),%rdi
 159         decl %ecx
 160         jnz 1b
 161 17:     movl %edx,%ecx
 162         andl $7,%edx
 163         shrl $3,%ecx
 164         jz 20f
 165 18:     movq (%rsi),%r8
 166 19:     movq %r8,(%rdi)
 167         leaq 8(%rsi),%rsi
 168         leaq 8(%rdi),%rdi
 169         decl %ecx
 170         jnz 18b
 171 20:     andl %edx,%edx
 172         jz 23f
 173         movl %edx,%ecx
 174 21:     movb (%rsi),%al
 175 22:     movb %al,(%rdi)
 176         incq %rsi
 177         incq %rdi
 178         decl %ecx
 179         jnz 21b
 180 23:     xor %eax,%eax
 181         ASM_CLAC
 182         ret
 183
 184         .section .fixup,"ax"
 185 30:     shll $6,%ecx
 186         addl %ecx,%edx
 187         jmp 60f
 188 40:     leal (%rdx,%rcx,8),%edx
 189         jmp 60f
 190 50:     movl %ecx,%edx
 191 60:     jmp copy_user_handle_tail /* ecx is zerorest also */
 192         .previous
 193
 194         _ASM_EXTABLE(1b,30b)
 195         _ASM_EXTABLE(2b,30b)
 196         _ASM_EXTABLE(3b,30b)
 197         _ASM_EXTABLE(4b,30b)
 198         _ASM_EXTABLE(5b,30b)
 199         _ASM_EXTABLE(6b,30b)
 200         _ASM_EXTABLE(7b,30b)
 201         _ASM_EXTABLE(8b,30b)
 202         _ASM_EXTABLE(9b,30b)
 203         _ASM_EXTABLE(10b,30b)
 204         _ASM_EXTABLE(11b,30b)
 205         _ASM_EXTABLE(12b,30b)
 206         _ASM_EXTABLE(13b,30b)
 207         _ASM_EXTABLE(14b,30b)
 208         _ASM_EXTABLE(15b,30b)
 209         _ASM_EXTABLE(16b,30b)
 210         _ASM_EXTABLE(18b,40b)
 211         _ASM_EXTABLE(19b,40b)
 212         _ASM_EXTABLE(21b,50b)
 213         _ASM_EXTABLE(22b,50b)
 214         CFI_ENDPROC
 215 ENDPROC(copy_user_generic_unrolled)
 216
 217 /* Some CPUs run faster using the string copy instructions.
 218  * This is also a lot simpler. Use them when possible.
 219  *
 220  * Only 4GB of copy is supported. This shouldn't be a problem
 221  * because the kernel normally only writes from/to page sized chunks
 222  * even if user space passed a longer buffer.
 223  * And more would be dangerous because both Intel and AMD have
 224  * errata with rep movsq > 4GB. If someone feels the need to fix
 225  * this please consider this.
 226  *
 227  * Input:
 228  * rdi destination
 229  * rsi source
 230  * rdx count
 231  *
 232  * Output:
 233  * eax uncopied bytes or 0 if successful.
 234  */
 235 ENTRY(copy_user_generic_string)
 236         CFI_STARTPROC
 237         ASM_STAC
 238         cmpl $8,%edx
 239         jb 2f           /* less than 8 bytes, go to byte copy loop */
 240         ALIGN_DESTINATION
 241         movl %edx,%ecx
 242         shrl $3,%ecx
 243         andl $7,%edx
 244 1:      rep
 245         movsq
 246 2:      movl %edx,%ecx
 247 3:      rep
 248         movsb
 249         xorl %eax,%eax
 250         ASM_CLAC
 251         ret
 252
 253         .section .fixup,"ax"
 254 11:     leal (%rdx,%rcx,8),%ecx
 255 12:     movl %ecx,%edx          /* ecx is zerorest also */
 256         jmp copy_user_handle_tail
 257         .previous
 258
 259         _ASM_EXTABLE(1b,11b)
 260         _ASM_EXTABLE(3b,12b)
 261         CFI_ENDPROC
 262 ENDPROC(copy_user_generic_string)
 263
 264 /*
 265  * Some CPUs are adding enhanced REP MOVSB/STOSB instructions.
 266  * It's recommended to use enhanced REP MOVSB/STOSB if it's enabled.
 267  *
 268  * Input:
 269  * rdi destination
 270  * rsi source
 271  * rdx count
 272  *
 273  * Output:
 274  * eax uncopied bytes or 0 if successful.
 275  */
 276 ENTRY(copy_user_enhanced_fast_string)
 277         CFI_STARTPROC
 278         ASM_STAC
 279         movl %edx,%ecx
 280 1:      rep
 281         movsb
 282         xorl %eax,%eax
 283         ASM_CLAC
 284         ret
 285
 286         .section .fixup,"ax"
 287 12:     movl %ecx,%edx          /* ecx is zerorest also */
 288         jmp copy_user_handle_tail
 289         .previous
 290
 291         _ASM_EXTABLE(1b,12b)
 292         CFI_ENDPROC
 293 ENDPROC(copy_user_enhanced_fast_string)