ports/sysdeps/mips/memcpy.S

   1 /* Copyright (C) 2012-2014 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3
   4    The GNU C Library is free software; you can redistribute it and/or
   5    modify it under the terms of the GNU Lesser General Public
   6    License as published by the Free Software Foundation; either
   7    version 2.1 of the License, or (at your option) any later version.
   8
   9    The GNU C Library is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12    Lesser General Public License for more details.
  13
  14    You should have received a copy of the GNU Lesser General Public
  15    License along with the GNU C Library.  If not, see
  16    <http://www.gnu.org/licenses/>.  */
  17
  18 #ifdef ANDROID_CHANGES
  19 #include "machine/asm.h"
  20 #include "machine/regdef.h"
  21 #define USE_MEMMOVE_FOR_OVERLAP
  22 #define PREFETCH_LOAD_HINT PREFETCH_HINT_LOAD_STREAMED
  23 #define PREFETCH_STORE_HINT PREFETCH_HINT_PREPAREFORSTORE
  24 #elif _LIBC
  25 #include <sysdep.h>
  26 #include <regdef.h>
  27 #include <sys/asm.h>
  28 #define PREFETCH_LOAD_HINT PREFETCH_HINT_LOAD_STREAMED
  29 #define PREFETCH_STORE_HINT PREFETCH_HINT_PREPAREFORSTORE
  30 #elif _COMPILING_NEWLIB
  31 #include "machine/asm.h"
  32 #include "machine/regdef.h"
  33 #define PREFETCH_LOAD_HINT PREFETCH_HINT_LOAD_STREAMED
  34 #define PREFETCH_STORE_HINT PREFETCH_HINT_PREPAREFORSTORE
  35 #else
  36 #include <regdef.h>
  37 #include <sys/asm.h>
  38 #endif
  39
  40 #if (_MIPS_ISA == _MIPS_ISA_MIPS4) || (_MIPS_ISA == _MIPS_ISA_MIPS5) || \
  41     (_MIPS_ISA == _MIPS_ISA_MIPS32) || (_MIPS_ISA == _MIPS_ISA_MIPS64)
  42 #ifndef DISABLE_PREFETCH
  43 #define USE_PREFETCH
  44 #endif
  45 #endif
  46
  47 #if defined(_MIPS_SIM) && ((_MIPS_SIM == _ABI64) || (_MIPS_SIM == _ABIN32))
  48 #ifndef DISABLE_DOUBLE
  49 #define USE_DOUBLE
  50 #endif
  51 #endif
  52
  53
  54
  55 /* Some asm.h files do not have the L macro definition.  */
  56 #ifndef L
  57 #if _MIPS_SIM == _ABIO32
  58 # define L(label) $L ## label
  59 #else
  60 # define L(label) .L ## label
  61 #endif
  62 #endif
  63
  64 /* Some asm.h files do not have the PTR_ADDIU macro definition.  */
  65 #ifndef PTR_ADDIU
  66 #ifdef USE_DOUBLE
  67 #define PTR_ADDIU       daddiu
  68 #else
  69 #define PTR_ADDIU       addiu
  70 #endif
  71 #endif
  72
  73 /* Some asm.h files do not have the PTR_SRA macro definition.  */
  74 #ifndef PTR_SRA
  75 #ifdef USE_DOUBLE
  76 #define PTR_SRA         dsra
  77 #else
  78 #define PTR_SRA         sra
  79 #endif
  80 #endif
  81
  82
  83 /*
  84  * Using PREFETCH_HINT_LOAD_STREAMED instead of PREFETCH_LOAD on load
  85  * prefetches appears to offer a slight preformance advantage.
  86  *
  87  * Using PREFETCH_HINT_PREPAREFORSTORE instead of PREFETCH_STORE
  88  * or PREFETCH_STORE_STREAMED offers a large performance advantage
  89  * but PREPAREFORSTORE has some special restrictions to consider.
  90  *
  91  * Prefetch with the 'prepare for store' hint does not copy a memory
  92  * location into the cache, it just allocates a cache line and zeros
  93  * it out.  This means that if you do not write to the entire cache
  94  * line before writing it out to memory some data will get zero'ed out
  95  * when the cache line is written back to memory and data will be lost.
  96  *
  97  * Also if you are using this memcpy to copy overlapping buffers it may
  98  * not behave correctly when using the 'prepare for store' hint.  If you
  99  * use the 'prepare for store' prefetch on a memory area that is in the
 100  * memcpy source (as well as the memcpy destination), then you will get
 101  * some data zero'ed out before you have a chance to read it and data will
 102  * be lost.
 103  *
 104  * If you are going to use this memcpy routine with the 'prepare for store'
 105  * prefetch you may want to set USE_MEMMOVE_FOR_OVERLAP in order to avoid
 106  * the problem of running memcpy on overlapping buffers.
 107  *
 108  * There are ifdef'ed sections of this memcpy to make sure that it does not
 109  * do prefetches on cache lines that are not going to be completely written.
 110  * This code is only needed and only used when PREFETCH_STORE_HINT is set to
 111  * PREFETCH_HINT_PREPAREFORSTORE.  This code assumes that cache lines are
 112  * 32 bytes and if the cache line is larger it will not work correctly.
 113  */
 114
 115 #ifdef USE_PREFETCH
 116 # define PREFETCH_HINT_LOAD             0
 117 # define PREFETCH_HINT_STORE            1
 118 # define PREFETCH_HINT_LOAD_STREAMED    4
 119 # define PREFETCH_HINT_STORE_STREAMED   5
 120 # define PREFETCH_HINT_LOAD_RETAINED    6
 121 # define PREFETCH_HINT_STORE_RETAINED   7
 122 # define PREFETCH_HINT_WRITEBACK_INVAL  25
 123 # define PREFETCH_HINT_PREPAREFORSTORE  30
 124
 125 /*
 126  * If we have not picked out what hints to use at this point use the
 127  * standard load and store prefetch hints.
 128  */
 129 #ifndef PREFETCH_STORE_HINT
 130 # define PREFETCH_STORE_HINT PREFETCH_HINT_STORE
 131 #endif
 132 #ifndef PREFETCH_LOAD_HINT
 133 # define PREFETCH_LOAD_HINT PREFETCH_HINT_LOAD
 134 #endif
 135
 136 /*
 137  * We double everything when USE_DOUBLE is true so we do 2 prefetches to
 138  * get 64 bytes in that case.  The assumption is that each individual
 139  * prefetch brings in 32 bytes.
 140  */
 141
 142 #ifdef USE_DOUBLE
 143 # define PREFETCH_CHUNK 64
 144 # define PREFETCH_FOR_LOAD(chunk, reg) \
 145  pref PREFETCH_LOAD_HINT, (chunk)*64(reg); \
 146  pref PREFETCH_LOAD_HINT, ((chunk)*64)+32(reg)
 147 # define PREFETCH_FOR_STORE(chunk, reg) \
 148  pref PREFETCH_STORE_HINT, (chunk)*64(reg); \
 149  pref PREFETCH_STORE_HINT, ((chunk)*64)+32(reg)
 150 #else
 151 # define PREFETCH_CHUNK 32
 152 # define PREFETCH_FOR_LOAD(chunk, reg) \
 153  pref PREFETCH_LOAD_HINT, (chunk)*32(reg)
 154 # define PREFETCH_FOR_STORE(chunk, reg) \
 155  pref PREFETCH_STORE_HINT, (chunk)*32(reg)
 156 #endif
 157 /* MAX_PREFETCH_SIZE is the maximum size of a prefetch, it must not be less
 158  * than PREFETCH_CHUNK, the assumed size of each prefetch.  If the real size
 159  * of a prefetch is greater than MAX_PREFETCH_SIZE and the PREPAREFORSTORE
 160  * hint is used, the code will not work correctly.  If PREPAREFORSTORE is not
 161  * used then MAX_PREFETCH_SIZE does not matter.  */
 162 #define MAX_PREFETCH_SIZE 128
 163 /* PREFETCH_LIMIT is set based on the fact that we never use an offset greater
 164  * than 5 on a STORE prefetch and that a single prefetch can never be larger
 165  * than MAX_PREFETCH_SIZE.  We add the extra 32 when USE_DOUBLE is set because
 166  * we actually do two prefetches in that case, one 32 bytes after the other.  */
 167 #ifdef USE_DOUBLE
 168 # define PREFETCH_LIMIT (5 * PREFETCH_CHUNK) + 32 + MAX_PREFETCH_SIZE
 169 #else
 170 # define PREFETCH_LIMIT (5 * PREFETCH_CHUNK) + MAX_PREFETCH_SIZE
 171 #endif
 172 #if (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE) \
 173     && ((PREFETCH_CHUNK * 4) < MAX_PREFETCH_SIZE)
 174 /* We cannot handle this because the initial prefetches may fetch bytes that
 175  * are before the buffer being copied.  We start copies with an offset
 176  * of 4 so avoid this situation when using PREPAREFORSTORE.  */
 177 #error "PREFETCH_CHUNK is too large and/or MAX_PREFETCH_SIZE is too small."
 178 #endif
 179 #else /* USE_PREFETCH not defined */
 180 # define PREFETCH_FOR_LOAD(offset, reg)
 181 # define PREFETCH_FOR_STORE(offset, reg)
 182 #endif
 183
 184 /* Allow the routine to be named something else if desired.  */
 185 #ifndef MEMCPY_NAME
 186 #define MEMCPY_NAME memcpy
 187 #endif
 188
 189 /* We use these 32/64 bit registers as temporaries to do the copying.  */
 190 #define REG0 t0
 191 #define REG1 t1
 192 #define REG2 t2
 193 #define REG3 t3
 194 #if defined(_MIPS_SIM) && ((_MIPS_SIM == _ABIO32) || (_MIPS_SIM == _ABIO64))
 195 #  define REG4 t4
 196 #  define REG5 t5
 197 #  define REG6 t6
 198 #  define REG7 t7
 199 #else
 200 #  define REG4 ta0
 201 #  define REG5 ta1
 202 #  define REG6 ta2
 203 #  define REG7 ta3
 204 #endif
 205
 206 /* We load/store 64 bits at a time when USE_DOUBLE is true.
 207  * The C_ prefix stands for CHUNK and is used to avoid macro name
 208  * conflicts with system header files.  */
 209
 210 #ifdef USE_DOUBLE
 211 #  define C_ST  sd
 212 #  define C_LD  ld
 213 #if __MIPSEB
 214 #  define C_LDHI        ldl     /* high part is left in big-endian      */
 215 #  define C_STHI        sdl     /* high part is left in big-endian      */
 216 #  define C_LDLO        ldr     /* low part is right in big-endian      */
 217 #  define C_STLO        sdr     /* low part is right in big-endian      */
 218 #else
 219 #  define C_LDHI        ldr     /* high part is right in little-endian  */
 220 #  define C_STHI        sdr     /* high part is right in little-endian  */
 221 #  define C_LDLO        ldl     /* low part is left in little-endian    */
 222 #  define C_STLO        sdl     /* low part is left in little-endian    */
 223 #endif
 224 #else
 225 #  define C_ST  sw
 226 #  define C_LD  lw
 227 #if __MIPSEB
 228 #  define C_LDHI        lwl     /* high part is left in big-endian      */
 229 #  define C_STHI        swl     /* high part is left in big-endian      */
 230 #  define C_LDLO        lwr     /* low part is right in big-endian      */
 231 #  define C_STLO        swr     /* low part is right in big-endian      */
 232 #else
 233 #  define C_LDHI        lwr     /* high part is right in little-endian  */
 234 #  define C_STHI        swr     /* high part is right in little-endian  */
 235 #  define C_LDLO        lwl     /* low part is left in little-endian    */
 236 #  define C_STLO        swl     /* low part is left in little-endian    */
 237 #endif
 238 #endif
 239
 240 /* Bookkeeping values for 32 vs. 64 bit mode.  */
 241 #ifdef USE_DOUBLE
 242 #  define NSIZE 8
 243 #  define NSIZEMASK 0x3f
 244 #  define NSIZEDMASK 0x7f
 245 #else
 246 #  define NSIZE 4
 247 #  define NSIZEMASK 0x1f
 248 #  define NSIZEDMASK 0x3f
 249 #endif
 250 #define UNIT(unit) ((unit)*NSIZE)
 251 #define UNITM1(unit) (((unit)*NSIZE)-1)
 252
 253 #ifdef ANDROID_CHANGES
 254 LEAF(MEMCPY_NAME, 0)
 255 #else
 256 LEAF(MEMCPY_NAME)
 257 #endif
 258         .set    nomips16
 259         .set    noreorder
 260 /*
 261  * Below we handle the case where memcpy is called with overlapping src and dst.
 262  * Although memcpy is not required to handle this case, some parts of Android
 263  * like Skia rely on such usage. We call memmove to handle such cases.
 264  */
 265 #ifdef USE_MEMMOVE_FOR_OVERLAP
 266         PTR_SUBU t0,a0,a1
 267         PTR_SRA t2,t0,31
 268         xor     t1,t0,t2
 269         PTR_SUBU t0,t1,t2
 270         sltu    t2,t0,a2
 271         beq     t2,zero,L(memcpy)
 272         la      t9,memmove
 273         jr      t9
 274          nop
 275 L(memcpy):
 276 #endif
 277 /*
 278  * If the size is less than 2*NSIZE (8 or 16), go to L(lastb).  Regardless of
 279  * size, copy dst pointer to v0 for the return value.
 280  */
 281         slti    t2,a2,(2 * NSIZE)
 282         bne     t2,zero,L(lastb)
 283 #if defined(RETURN_FIRST_PREFETCH) || defined(RETURN_LAST_PREFETCH)
 284         move    v0,zero
 285 #else
 286         move    v0,a0
 287 #endif
 288 /*
 289  * If src and dst have different alignments, go to L(unaligned), if they
 290  * have the same alignment (but are not actually aligned) do a partial
 291  * load/store to make them aligned.  If they are both already aligned
 292  * we can start copying at L(aligned).
 293  */
 294         xor     t8,a1,a0
 295         andi    t8,t8,(NSIZE-1)         /* t8 is a0/a1 word-displacement */
 296         bne     t8,zero,L(unaligned)
 297         PTR_SUBU a3, zero, a0
 298
 299         andi    a3,a3,(NSIZE-1)         /* copy a3 bytes to align a0/a1   */
 300         beq     a3,zero,L(aligned)      /* if a3=0, it is already aligned */
 301         PTR_SUBU a2,a2,a3               /* a2 is the remining bytes count */
 302
 303         C_LDHI  t8,0(a1)
 304         PTR_ADDU a1,a1,a3
 305         C_STHI  t8,0(a0)
 306         PTR_ADDU a0,a0,a3
 307
 308 /*
 309  * Now dst/src are both aligned to (word or double word) aligned addresses
 310  * Set a2 to count how many bytes we have to copy after all the 64/128 byte
 311  * chunks are copied and a3 to the dst pointer after all the 64/128 byte
 312  * chunks have been copied.  We will loop, incrementing a0 and a1 until a0
 313  * equals a3.
 314  */
 315
 316 L(aligned):
 317         andi    t8,a2,NSIZEDMASK /* any whole 64-byte/128-byte chunks? */
 318         beq     a2,t8,L(chkw)    /* if a2==t8, no 64-byte/128-byte chunks */
 319         PTR_SUBU a3,a2,t8        /* subtract from a2 the reminder */
 320         PTR_ADDU a3,a0,a3        /* Now a3 is the final dst after loop */
 321
 322 /* When in the loop we may prefetch with the 'prepare to store' hint,
 323  * in this case the a0+x should not be past the "t0-32" address.  This
 324  * means: for x=128 the last "safe" a0 address is "t0-160".  Alternatively,
 325  * for x=64 the last "safe" a0 address is "t0-96" In the current version we
 326  * will use "prefetch hint,128(a0)", so "t0-160" is the limit.
 327  */
 328 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
 329         PTR_ADDU t0,a0,a2               /* t0 is the "past the end" address */
 330         PTR_SUBU t9,t0,PREFETCH_LIMIT   /* t9 is the "last safe pref" address */
 331 #endif
 332         PREFETCH_FOR_LOAD  (0, a1)
 333         PREFETCH_FOR_LOAD  (1, a1)
 334         PREFETCH_FOR_LOAD  (2, a1)
 335         PREFETCH_FOR_LOAD  (3, a1)
 336 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT != PREFETCH_HINT_PREPAREFORSTORE)
 337         PREFETCH_FOR_STORE (1, a0)
 338         PREFETCH_FOR_STORE (2, a0)
 339         PREFETCH_FOR_STORE (3, a0)
 340 #endif
 341 #if defined(RETURN_FIRST_PREFETCH) && defined(USE_PREFETCH)
 342 #if PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE
 343         sltu    v1,t9,a0
 344         bgtz    v1,L(skip_set)
 345         nop
 346         PTR_ADDIU v0,a0,(PREFETCH_CHUNK*4)
 347 L(skip_set):
 348 #else
 349         PTR_ADDIU v0,a0,(PREFETCH_CHUNK*1)
 350 #endif
 351 #endif
 352 #if defined(RETURN_LAST_PREFETCH) && defined(USE_PREFETCH) \
 353     && (PREFETCH_STORE_HINT != PREFETCH_HINT_PREPAREFORSTORE)
 354         PTR_ADDIU v0,a0,(PREFETCH_CHUNK*3)
 355 #ifdef USE_DOUBLE
 356         PTR_ADDIU v0,v0,32
 357 #endif
 358 #endif
 359 L(loop16w):
 360         C_LD    t0,UNIT(0)(a1)
 361 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
 362         sltu    v1,t9,a0                /* If a0 > t9 don't use next prefetch */
 363         bgtz    v1,L(skip_pref)
 364 #endif
 365         C_LD    t1,UNIT(1)(a1)
 366         PREFETCH_FOR_STORE (4, a0)
 367         PREFETCH_FOR_STORE (5, a0)
 368 #if defined(RETURN_LAST_PREFETCH) && defined(USE_PREFETCH)
 369         PTR_ADDIU v0,a0,(PREFETCH_CHUNK*5)
 370 #ifdef USE_DOUBLE
 371         PTR_ADDIU v0,v0,32
 372 #endif
 373 #endif
 374 L(skip_pref):
 375         C_LD    REG2,UNIT(2)(a1)
 376         C_LD    REG3,UNIT(3)(a1)
 377         C_LD    REG4,UNIT(4)(a1)
 378         C_LD    REG5,UNIT(5)(a1)
 379         C_LD    REG6,UNIT(6)(a1)
 380         C_LD    REG7,UNIT(7)(a1)
 381         PREFETCH_FOR_LOAD (4, a1)
 382
 383         C_ST    t0,UNIT(0)(a0)
 384         C_ST    t1,UNIT(1)(a0)
 385         C_ST    REG2,UNIT(2)(a0)
 386         C_ST    REG3,UNIT(3)(a0)
 387         C_ST    REG4,UNIT(4)(a0)
 388         C_ST    REG5,UNIT(5)(a0)
 389         C_ST    REG6,UNIT(6)(a0)
 390         C_ST    REG7,UNIT(7)(a0)
 391
 392         C_LD    t0,UNIT(8)(a1)
 393         C_LD    t1,UNIT(9)(a1)
 394         C_LD    REG2,UNIT(10)(a1)
 395         C_LD    REG3,UNIT(11)(a1)
 396         C_LD    REG4,UNIT(12)(a1)
 397         C_LD    REG5,UNIT(13)(a1)
 398         C_LD    REG6,UNIT(14)(a1)
 399         C_LD    REG7,UNIT(15)(a1)
 400         PREFETCH_FOR_LOAD (5, a1)
 401         C_ST    t0,UNIT(8)(a0)
 402         C_ST    t1,UNIT(9)(a0)
 403         C_ST    REG2,UNIT(10)(a0)
 404         C_ST    REG3,UNIT(11)(a0)
 405         C_ST    REG4,UNIT(12)(a0)
 406         C_ST    REG5,UNIT(13)(a0)
 407         C_ST    REG6,UNIT(14)(a0)
 408         C_ST    REG7,UNIT(15)(a0)
 409         PTR_ADDIU a0,a0,UNIT(16)        /* adding 64/128 to dest */
 410         bne     a0,a3,L(loop16w)
 411         PTR_ADDIU a1,a1,UNIT(16)        /* adding 64/128 to src */
 412         move    a2,t8
 413
 414 /* Here we have src and dest word-aligned but less than 64-bytes or
 415  * 128 bytes to go.  Check for a 32(64) byte chunk and copy if if there
 416  * is one.  Otherwise jump down to L(chk1w) to handle the tail end of
 417  * the copy.
 418  */
 419
 420 L(chkw):
 421         PREFETCH_FOR_LOAD (0, a1)
 422         andi    t8,a2,NSIZEMASK /* Is there a 32-byte/64-byte chunk.  */
 423                                 /* The t8 is the reminder count past 32-bytes */
 424         beq     a2,t8,L(chk1w)  /* When a2=t8, no 32-byte chunk  */
 425         nop
 426         C_LD    t0,UNIT(0)(a1)
 427         C_LD    t1,UNIT(1)(a1)
 428         C_LD    REG2,UNIT(2)(a1)
 429         C_LD    REG3,UNIT(3)(a1)
 430         C_LD    REG4,UNIT(4)(a1)
 431         C_LD    REG5,UNIT(5)(a1)
 432         C_LD    REG6,UNIT(6)(a1)
 433         C_LD    REG7,UNIT(7)(a1)
 434         PTR_ADDIU a1,a1,UNIT(8)
 435         C_ST    t0,UNIT(0)(a0)
 436         C_ST    t1,UNIT(1)(a0)
 437         C_ST    REG2,UNIT(2)(a0)
 438         C_ST    REG3,UNIT(3)(a0)
 439         C_ST    REG4,UNIT(4)(a0)
 440         C_ST    REG5,UNIT(5)(a0)
 441         C_ST    REG6,UNIT(6)(a0)
 442         C_ST    REG7,UNIT(7)(a0)
 443         PTR_ADDIU a0,a0,UNIT(8)
 444
 445 /*
 446  * Here we have less than 32(64) bytes to copy.  Set up for a loop to
 447  * copy one word (or double word) at a time.  Set a2 to count how many
 448  * bytes we have to copy after all the word (or double word) chunks are
 449  * copied and a3 to the dst pointer after all the (d)word chunks have
 450  * been copied.  We will loop, incrementing a0 and a1 until a0 equals a3.
 451  */
 452 L(chk1w):
 453         andi    a2,t8,(NSIZE-1) /* a2 is the reminder past one (d)word chunks */
 454         beq     a2,t8,L(lastb)
 455         PTR_SUBU a3,t8,a2       /* a3 is count of bytes in one (d)word chunks */
 456         PTR_ADDU a3,a0,a3       /* a3 is the dst address after loop */
 457
 458 /* copying in words (4-byte or 8-byte chunks) */
 459 L(wordCopy_loop):
 460         C_LD    REG3,UNIT(0)(a1)
 461         PTR_ADDIU a0,a0,UNIT(1)
 462         PTR_ADDIU a1,a1,UNIT(1)
 463         bne     a0,a3,L(wordCopy_loop)
 464         C_ST    REG3,UNIT(-1)(a0)
 465
 466 /* Copy the last 8 (or 16) bytes */
 467 L(lastb):
 468         blez    a2,L(leave)
 469         PTR_ADDU a3,a0,a2       /* a3 is the last dst address */
 470 L(lastbloop):
 471         lb      v1,0(a1)
 472         PTR_ADDIU a0,a0,1
 473         PTR_ADDIU a1,a1,1
 474         bne     a0,a3,L(lastbloop)
 475         sb      v1,-1(a0)
 476 L(leave):
 477         j       ra
 478         nop
 479 /*
 480  * UNALIGNED case, got here with a3 = "negu a0"
 481  * This code is nearly identical to the aligned code above
 482  * but only the destination (not the source) gets aligned
 483  * so we need to do partial loads of the source followed
 484  * by normal stores to the destination (once we have aligned
 485  * the destination).
 486  */
 487
 488 L(unaligned):
 489         andi    a3,a3,(NSIZE-1) /* copy a3 bytes to align a0/a1 */
 490         beqz    a3,L(ua_chk16w) /* if a3=0, it is already aligned */
 491         PTR_SUBU a2,a2,a3       /* a2 is the remining bytes count */
 492
 493         C_LDHI  v1,UNIT(0)(a1)
 494         C_LDLO  v1,UNITM1(1)(a1)
 495         PTR_ADDU a1,a1,a3
 496         C_STHI  v1,UNIT(0)(a0)
 497         PTR_ADDU a0,a0,a3
 498
 499 /*
 500  *  Now the destination (but not the source) is aligned
 501  * Set a2 to count how many bytes we have to copy after all the 64/128 byte
 502  * chunks are copied and a3 to the dst pointer after all the 64/128 byte
 503  * chunks have been copied.  We will loop, incrementing a0 and a1 until a0
 504  * equals a3.
 505  */
 506
 507 L(ua_chk16w):
 508         andi    t8,a2,NSIZEDMASK /* any whole 64-byte/128-byte chunks? */
 509         beq     a2,t8,L(ua_chkw) /* if a2==t8, no 64-byte/128-byte chunks */
 510         PTR_SUBU a3,a2,t8        /* subtract from a2 the reminder */
 511         PTR_ADDU a3,a0,a3        /* Now a3 is the final dst after loop */
 512
 513 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
 514         PTR_ADDU t0,a0,a2         /* t0 is the "past the end" address */
 515         PTR_SUBU t9,t0,PREFETCH_LIMIT /* t9 is the "last safe pref" address */
 516 #endif
 517         PREFETCH_FOR_LOAD  (0, a1)
 518         PREFETCH_FOR_LOAD  (1, a1)
 519         PREFETCH_FOR_LOAD  (2, a1)
 520 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT != PREFETCH_HINT_PREPAREFORSTORE)
 521         PREFETCH_FOR_STORE (1, a0)
 522         PREFETCH_FOR_STORE (2, a0)
 523         PREFETCH_FOR_STORE (3, a0)
 524 #endif
 525 #if defined(RETURN_FIRST_PREFETCH) && defined(USE_PREFETCH)
 526 #if (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
 527         sltu    v1,t9,a0
 528         bgtz    v1,L(ua_skip_set)
 529         nop
 530         PTR_ADDIU v0,a0,(PREFETCH_CHUNK*4)
 531 L(ua_skip_set):
 532 #else
 533         PTR_ADDIU v0,a0,(PREFETCH_CHUNK*1)
 534 #endif
 535 #endif
 536 L(ua_loop16w):
 537         PREFETCH_FOR_LOAD  (3, a1)
 538         C_LDHI  t0,UNIT(0)(a1)
 539         C_LDHI  t1,UNIT(1)(a1)
 540         C_LDHI  REG2,UNIT(2)(a1)
 541 #if defined(USE_PREFETCH) && (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE)
 542         sltu    v1,t9,a0
 543         bgtz    v1,L(ua_skip_pref)
 544 #endif
 545         C_LDHI  REG3,UNIT(3)(a1)
 546         PREFETCH_FOR_STORE (4, a0)
 547         PREFETCH_FOR_STORE (5, a0)
 548 L(ua_skip_pref):
 549         C_LDHI  REG4,UNIT(4)(a1)
 550         C_LDHI  REG5,UNIT(5)(a1)
 551         C_LDHI  REG6,UNIT(6)(a1)
 552         C_LDHI  REG7,UNIT(7)(a1)
 553         C_LDLO  t0,UNITM1(1)(a1)
 554         C_LDLO  t1,UNITM1(2)(a1)
 555         C_LDLO  REG2,UNITM1(3)(a1)
 556         C_LDLO  REG3,UNITM1(4)(a1)
 557         C_LDLO  REG4,UNITM1(5)(a1)
 558         C_LDLO  REG5,UNITM1(6)(a1)
 559         C_LDLO  REG6,UNITM1(7)(a1)
 560         C_LDLO  REG7,UNITM1(8)(a1)
 561         PREFETCH_FOR_LOAD (4, a1)
 562         C_ST    t0,UNIT(0)(a0)
 563         C_ST    t1,UNIT(1)(a0)
 564         C_ST    REG2,UNIT(2)(a0)
 565         C_ST    REG3,UNIT(3)(a0)
 566         C_ST    REG4,UNIT(4)(a0)
 567         C_ST    REG5,UNIT(5)(a0)
 568         C_ST    REG6,UNIT(6)(a0)
 569         C_ST    REG7,UNIT(7)(a0)
 570         C_LDHI  t0,UNIT(8)(a1)
 571         C_LDHI  t1,UNIT(9)(a1)
 572         C_LDHI  REG2,UNIT(10)(a1)
 573         C_LDHI  REG3,UNIT(11)(a1)
 574         C_LDHI  REG4,UNIT(12)(a1)
 575         C_LDHI  REG5,UNIT(13)(a1)
 576         C_LDHI  REG6,UNIT(14)(a1)
 577         C_LDHI  REG7,UNIT(15)(a1)
 578         C_LDLO  t0,UNITM1(9)(a1)
 579         C_LDLO  t1,UNITM1(10)(a1)
 580         C_LDLO  REG2,UNITM1(11)(a1)
 581         C_LDLO  REG3,UNITM1(12)(a1)
 582         C_LDLO  REG4,UNITM1(13)(a1)
 583         C_LDLO  REG5,UNITM1(14)(a1)
 584         C_LDLO  REG6,UNITM1(15)(a1)
 585         C_LDLO  REG7,UNITM1(16)(a1)
 586         PREFETCH_FOR_LOAD (5, a1)
 587         C_ST    t0,UNIT(8)(a0)
 588         C_ST    t1,UNIT(9)(a0)
 589         C_ST    REG2,UNIT(10)(a0)
 590         C_ST    REG3,UNIT(11)(a0)
 591         C_ST    REG4,UNIT(12)(a0)
 592         C_ST    REG5,UNIT(13)(a0)
 593         C_ST    REG6,UNIT(14)(a0)
 594         C_ST    REG7,UNIT(15)(a0)
 595         PTR_ADDIU a0,a0,UNIT(16)        /* adding 64/128 to dest */
 596         bne     a0,a3,L(ua_loop16w)
 597         PTR_ADDIU a1,a1,UNIT(16)        /* adding 64/128 to src */
 598         move    a2,t8
 599
 600 /* Here we have src and dest word-aligned but less than 64-bytes or
 601  * 128 bytes to go.  Check for a 32(64) byte chunk and copy if if there
 602  * is one.  Otherwise jump down to L(ua_chk1w) to handle the tail end of
 603  * the copy.  */
 604
 605 L(ua_chkw):
 606         PREFETCH_FOR_LOAD (0, a1)
 607         andi    t8,a2,NSIZEMASK   /* Is there a 32-byte/64-byte chunk.  */
 608                                   /* t8 is the reminder count past 32-bytes */
 609         beq     a2,t8,L(ua_chk1w) /* When a2=t8, no 32-byte chunk */
 610         nop
 611         C_LDHI  t0,UNIT(0)(a1)
 612         C_LDHI  t1,UNIT(1)(a1)
 613         C_LDHI  REG2,UNIT(2)(a1)
 614         C_LDHI  REG3,UNIT(3)(a1)
 615         C_LDHI  REG4,UNIT(4)(a1)
 616         C_LDHI  REG5,UNIT(5)(a1)
 617         C_LDHI  REG6,UNIT(6)(a1)
 618         C_LDHI  REG7,UNIT(7)(a1)
 619         C_LDLO  t0,UNITM1(1)(a1)
 620         C_LDLO  t1,UNITM1(2)(a1)
 621         C_LDLO  REG2,UNITM1(3)(a1)
 622         C_LDLO  REG3,UNITM1(4)(a1)
 623         C_LDLO  REG4,UNITM1(5)(a1)
 624         C_LDLO  REG5,UNITM1(6)(a1)
 625         C_LDLO  REG6,UNITM1(7)(a1)
 626         C_LDLO  REG7,UNITM1(8)(a1)
 627         PTR_ADDIU a1,a1,UNIT(8)
 628         C_ST    t0,UNIT(0)(a0)
 629         C_ST    t1,UNIT(1)(a0)
 630         C_ST    REG2,UNIT(2)(a0)
 631         C_ST    REG3,UNIT(3)(a0)
 632         C_ST    REG4,UNIT(4)(a0)
 633         C_ST    REG5,UNIT(5)(a0)
 634         C_ST    REG6,UNIT(6)(a0)
 635         C_ST    REG7,UNIT(7)(a0)
 636         PTR_ADDIU a0,a0,UNIT(8)
 637 /*
 638  * Here we have less than 32(64) bytes to copy.  Set up for a loop to
 639  * copy one word (or double word) at a time.
 640  */
 641 L(ua_chk1w):
 642         andi    a2,t8,(NSIZE-1) /* a2 is the reminder past one (d)word chunks */
 643         beq     a2,t8,L(ua_smallCopy)
 644         PTR_SUBU a3,t8,a2       /* a3 is count of bytes in one (d)word chunks */
 645         PTR_ADDU a3,a0,a3       /* a3 is the dst address after loop */
 646
 647 /* copying in words (4-byte or 8-byte chunks) */
 648 L(ua_wordCopy_loop):
 649         C_LDHI  v1,UNIT(0)(a1)
 650         C_LDLO  v1,UNITM1(1)(a1)
 651         PTR_ADDIU a0,a0,UNIT(1)
 652         PTR_ADDIU a1,a1,UNIT(1)
 653         bne     a0,a3,L(ua_wordCopy_loop)
 654         C_ST    v1,UNIT(-1)(a0)
 655
 656 /* Copy the last 8 (or 16) bytes */
 657 L(ua_smallCopy):
 658         beqz    a2,L(leave)
 659         PTR_ADDU a3,a0,a2       /* a3 is the last dst address */
 660 L(ua_smallCopy_loop):
 661         lb      v1,0(a1)
 662         PTR_ADDIU a0,a0,1
 663         PTR_ADDIU a1,a1,1
 664         bne     a0,a3,L(ua_smallCopy_loop)
 665         sb      v1,-1(a0)
 666
 667         j       ra
 668         nop
 669
 670         .set    at
 671         .set    reorder
 672 END(MEMCPY_NAME)
 673 #ifndef ANDROID_CHANGES
 674 #ifdef _LIBC
 675 libc_hidden_builtin_def (MEMCPY_NAME)
 676 #endif
 677 #endif