arch/arm/lib/memcpy.S

   1 /*
   2  *  linux/arch/arm/lib/memcpy.S
   3  *
   4  *  Author:     Nicolas Pitre
   5  *  Created:    Sep 28, 2005
   6  *  Copyright:  MontaVista Software, Inc.
   7  *
   8  *  This program is free software; you can redistribute it and/or modify
   9  *  it under the terms of the GNU General Public License version 2 as
  10  *  published by the Free Software Foundation.
  11  */
  12
  13 #include <linux/linkage.h>
  14 #include <asm/assembler.h>
  15
  16 #define LDR1W_SHIFT     0
  17 #define STR1W_SHIFT     0
  18
  19         .macro ldr1w ptr reg abort
  20         W(ldr) \reg, [\ptr], #4
  21         .endm
  22
  23         .macro ldr4w ptr reg1 reg2 reg3 reg4 abort
  24         ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4}
  25         .endm
  26
  27         .macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
  28         ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
  29         .endm
  30
  31         .macro ldr1b ptr reg cond=al abort
  32         ldrb\cond\() \reg, [\ptr], #1
  33         .endm
  34
  35         .macro str1w ptr reg abort
  36         W(str) \reg, [\ptr], #4
  37         .endm
  38
  39         .macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
  40         stmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
  41         .endm
  42
  43         .macro str1b ptr reg cond=al abort
  44         strb\cond\() \reg, [\ptr], #1
  45         .endm
  46
  47         .macro enter reg1 reg2
  48         stmdb sp!, {r0, \reg1, \reg2}
  49         .endm
  50
  51         .macro exit reg1 reg2
  52         ldmfd sp!, {r0, \reg1, \reg2}
  53         .endm
  54
  55         .text
  56
  57 /* Prototype: void *memcpy(void *dest, const void *src, size_t n); */
  58         .syntax unified
  59 #if defined(CONFIG_SYS_THUMB_BUILD) && !defined(MEMCPY_NO_THUMB_BUILD)
  60         .thumb
  61         .thumb_func
  62 #endif
  63 ENTRY(memcpy)
  64                 cmp     r0, r1
  65                 moveq   pc, lr
  66
  67                 enter   r4, lr
  68
  69                 subs    r2, r2, #4
  70                 blt     8f
  71                 ands    ip, r0, #3
  72         PLD(    pld     [r1, #0]                )
  73                 bne     9f
  74                 ands    ip, r1, #3
  75                 bne     10f
  76
  77 1:              subs    r2, r2, #(28)
  78                 stmfd   sp!, {r5 - r8}
  79                 blt     5f
  80
  81         CALGN(  ands    ip, r0, #31             )
  82         CALGN(  rsb     r3, ip, #32             )
  83         CALGN(  sbcsne  r4, r3, r2              )  @ C is always set here
  84         CALGN(  bcs     2f                      )
  85         CALGN(  adr     r4, 6f                  )
  86         CALGN(  subs    r2, r2, r3              )  @ C gets set
  87         CALGN(  add     pc, r4, ip              )
  88
  89         PLD(    pld     [r1, #0]                )
  90 2:      PLD(    subs    r2, r2, #96             )
  91         PLD(    pld     [r1, #28]               )
  92         PLD(    blt     4f                      )
  93         PLD(    pld     [r1, #60]               )
  94         PLD(    pld     [r1, #92]               )
  95
  96 3:      PLD(    pld     [r1, #124]              )
  97 4:              ldr8w   r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
  98                 subs    r2, r2, #32
  99                 str8w   r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
 100                 bge     3b
 101         PLD(    cmn     r2, #96                 )
 102         PLD(    bge     4b                      )
 103
 104 5:              ands    ip, r2, #28
 105                 rsb     ip, ip, #32
 106 #if LDR1W_SHIFT > 0
 107                 lsl     ip, ip, #LDR1W_SHIFT
 108 #endif
 109                 addne   pc, pc, ip              @ C is always clear here
 110                 b       7f
 111 6:
 112                 .rept   (1 << LDR1W_SHIFT)
 113                 W(nop)
 114                 .endr
 115                 ldr1w   r1, r3, abort=20f
 116                 ldr1w   r1, r4, abort=20f
 117                 ldr1w   r1, r5, abort=20f
 118                 ldr1w   r1, r6, abort=20f
 119                 ldr1w   r1, r7, abort=20f
 120                 ldr1w   r1, r8, abort=20f
 121                 ldr1w   r1, lr, abort=20f
 122
 123 #if LDR1W_SHIFT < STR1W_SHIFT
 124                 lsl     ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
 125 #elif LDR1W_SHIFT > STR1W_SHIFT
 126                 lsr     ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
 127 #endif
 128                 add     pc, pc, ip
 129                 nop
 130                 .rept   (1 << STR1W_SHIFT)
 131                 W(nop)
 132                 .endr
 133                 str1w   r0, r3, abort=20f
 134                 str1w   r0, r4, abort=20f
 135                 str1w   r0, r5, abort=20f
 136                 str1w   r0, r6, abort=20f
 137                 str1w   r0, r7, abort=20f
 138                 str1w   r0, r8, abort=20f
 139                 str1w   r0, lr, abort=20f
 140
 141         CALGN(  bcs     2b                      )
 142
 143 7:              ldmfd   sp!, {r5 - r8}
 144
 145 8:              movs    r2, r2, lsl #31
 146                 ldr1b   r1, r3, ne, abort=21f
 147                 ldr1b   r1, r4, cs, abort=21f
 148                 ldr1b   r1, ip, cs, abort=21f
 149                 str1b   r0, r3, ne, abort=21f
 150                 str1b   r0, r4, cs, abort=21f
 151                 str1b   r0, ip, cs, abort=21f
 152
 153                 exit    r4, pc
 154
 155 9:              rsb     ip, ip, #4
 156                 cmp     ip, #2
 157                 ldr1b   r1, r3, gt, abort=21f
 158                 ldr1b   r1, r4, ge, abort=21f
 159                 ldr1b   r1, lr, abort=21f
 160                 str1b   r0, r3, gt, abort=21f
 161                 str1b   r0, r4, ge, abort=21f
 162                 subs    r2, r2, ip
 163                 str1b   r0, lr, abort=21f
 164                 blt     8b
 165                 ands    ip, r1, #3
 166                 beq     1b
 167
 168 10:             bic     r1, r1, #3
 169                 cmp     ip, #2
 170                 ldr1w   r1, lr, abort=21f
 171                 beq     17f
 172                 bgt     18f
 173
 174
 175                 .macro  forward_copy_shift pull push
 176
 177                 subs    r2, r2, #28
 178                 blt     14f
 179
 180         CALGN(  ands    ip, r0, #31             )
 181         CALGN(  rsb     ip, ip, #32             )
 182         CALGN(  sbcsne  r4, ip, r2              )  @ C is always set here
 183         CALGN(  subcc   r2, r2, ip              )
 184         CALGN(  bcc     15f                     )
 185
 186 11:             stmfd   sp!, {r5 - r9}
 187
 188         PLD(    pld     [r1, #0]                )
 189         PLD(    subs    r2, r2, #96             )
 190         PLD(    pld     [r1, #28]               )
 191         PLD(    blt     13f                     )
 192         PLD(    pld     [r1, #60]               )
 193         PLD(    pld     [r1, #92]               )
 194
 195 12:     PLD(    pld     [r1, #124]              )
 196 13:             ldr4w   r1, r4, r5, r6, r7, abort=19f
 197                 mov     r3, lr, lspull #\pull
 198                 subs    r2, r2, #32
 199                 ldr4w   r1, r8, r9, ip, lr, abort=19f
 200                 orr     r3, r3, r4, lspush #\push
 201                 mov     r4, r4, lspull #\pull
 202                 orr     r4, r4, r5, lspush #\push
 203                 mov     r5, r5, lspull #\pull
 204                 orr     r5, r5, r6, lspush #\push
 205                 mov     r6, r6, lspull #\pull
 206                 orr     r6, r6, r7, lspush #\push
 207                 mov     r7, r7, lspull #\pull
 208                 orr     r7, r7, r8, lspush #\push
 209                 mov     r8, r8, lspull #\pull
 210                 orr     r8, r8, r9, lspush #\push
 211                 mov     r9, r9, lspull #\pull
 212                 orr     r9, r9, ip, lspush #\push
 213                 mov     ip, ip, lspull #\pull
 214                 orr     ip, ip, lr, lspush #\push
 215                 str8w   r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f
 216                 bge     12b
 217         PLD(    cmn     r2, #96                 )
 218         PLD(    bge     13b                     )
 219
 220                 ldmfd   sp!, {r5 - r9}
 221
 222 14:             ands    ip, r2, #28
 223                 beq     16f
 224
 225 15:             mov     r3, lr, lspull #\pull
 226                 ldr1w   r1, lr, abort=21f
 227                 subs    ip, ip, #4
 228                 orr     r3, r3, lr, lspush #\push
 229                 str1w   r0, r3, abort=21f
 230                 bgt     15b
 231         CALGN(  cmp     r2, #0                  )
 232         CALGN(  bge     11b                     )
 233
 234 16:             sub     r1, r1, #(\push / 8)
 235                 b       8b
 236
 237                 .endm
 238
 239
 240                 forward_copy_shift      pull=8  push=24
 241
 242 17:             forward_copy_shift      pull=16 push=16
 243
 244 18:             forward_copy_shift      pull=24 push=8
 245
 246
 247 /*
 248  * Abort preamble and completion macros.
 249  * If a fixup handler is required then those macros must surround it.
 250  * It is assumed that the fixup code will handle the private part of
 251  * the exit macro.
 252  */
 253
 254         .macro  copy_abort_preamble
 255 19:     ldmfd   sp!, {r5 - r9}
 256         b       21f
 257 20:     ldmfd   sp!, {r5 - r8}
 258 21:
 259         .endm
 260
 261         .macro  copy_abort_end
 262         ldmfd   sp!, {r4, pc}
 263         .endm
 264
 265 ENDPROC(memcpy)