sysdeps/powerpc/powerpc64/power8/strcpy.S

   1 /* Optimized strcpy/stpcpy implementation for PowerPC64/POWER8.
   2    Copyright (C) 2015-2016 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <sysdep.h>
  20
  21 #ifdef USE_AS_STPCPY
  22 # define FUNC_NAME __stpcpy
  23 #else
  24 # define FUNC_NAME strcpy
  25 #endif
  26
  27 /* Implements the function
  28
  29    char * [r3] strcpy (char *dest [r3], const char *src [r4])
  30
  31    or
  32
  33    char * [r3] stpcpy (char *dest [r3], const char *src [r4])
  34
  35    if USE_AS_STPCPY is defined.
  36
  37    The implementation uses unaligned doubleword access to avoid specialized
  38    code paths depending of data alignment.  Although recent powerpc64 uses
  39    64K as default, the page cross handling assumes minimum page size of
  40    4k.  */
  41
  42         .machine  power7
  43 EALIGN (FUNC_NAME, 4, 0)
  44         li      r0,0          /* Doubleword with null chars to use
  45                                  with cmpb.  */
  46
  47         /* Check if the [src]+15 will cross a 4K page by checking if the bit
  48            indicating the page size changes.  Basically:
  49
  50            uint64_t srcin = (uint64_t)src;
  51            uint64_t ob = srcin & 4096UL;
  52            uint64_t nb = (srcin+15UL) & 4096UL;
  53            if (ob ^ nb)
  54              goto pagecross;  */
  55
  56         addi    r9,r4,15
  57         xor     r9,r9,r4
  58         rlwinm. r9,r9,0,19,19
  59         bne     L(pagecross)
  60
  61         /* For short string (less than 16 bytes), just calculate its size as
  62            strlen and issues a memcpy if null is found.  */
  63         mr      r7,r4
  64         ld      r12,0(r7)     /* Load doubleword from memory.  */
  65         cmpb    r10,r12,r0    /* Check for null bytes in DWORD1.  */
  66         cmpdi   cr7,r10,0     /* If r10 == 0, no null's have been found.  */
  67         bne     cr7,L(done)
  68
  69         ldu     r8,8(r7)
  70         cmpb    r10,r8,r0
  71         cmpdi   cr7,r10,0
  72         bne     cr7,L(done)
  73
  74         b       L(loop_before)
  75
  76         .align  4
  77 L(pagecross):
  78         clrrdi  r7,r4,3       /* Align the address to doubleword boundary.  */
  79         rlwinm  r6,r4,3,26,28 /* Calculate padding.  */
  80         li      r5,-1         /* MASK = 0xffffffffffffffff.  */
  81         ld      r12,0(r7)     /* Load doubleword from memory.  */
  82 #ifdef __LITTLE_ENDIAN__
  83         sld     r5,r5,r6
  84 #else
  85         srd     r5,r5,r6      /* MASK = MASK >> padding.  */
  86 #endif
  87         orc     r9,r12,r5     /* Mask bits that are not part of the string.  */
  88         cmpb    r10,r9,r0     /* Check for null bytes in DWORD1.  */
  89         cmpdi   cr7,r10,0     /* If r10 == 0, no null's have been found.  */
  90         bne     cr7,L(done)
  91
  92         ldu     r6,8(r7)
  93         cmpb    r10,r6,r0
  94         cmpdi   cr7,r10,0
  95         bne     cr7,L(done)
  96
  97         ld      r12,0(r7)
  98         cmpb    r10,r12,r0
  99         cmpdi   cr7,r10,0
 100         bne     cr7,L(done)
 101
 102         ldu     r6,8(r7)
 103         cmpb    r10,r6,r0
 104         cmpdi   cr7,r10,0
 105         bne     cr7,L(done)
 106
 107         /* We checked for 24 - x bytes, with x being the source alignment
 108            (0 <= x <= 16), and no zero has been found.  Start the loop
 109            copy with doubleword aligned address.  */
 110         mr      r7,r4
 111         ld      r12, 0(r7)
 112         ldu     r8, 8(r7)
 113
 114 L(loop_before):
 115         /* Save the two doublewords readed from source and align the source
 116            to 16 bytes for the loop.  */
 117         mr      r11,r3
 118         std     r12,0(r11)
 119         std     r8,8(r11)
 120         addi    r11,r11,16
 121         rldicl  r9,r4,0,60
 122         subf    r7,r9,r7
 123         subf    r11,r9,r11
 124         b       L(loop_start)
 125
 126         .align  5
 127 L(loop):
 128         std     r12, 0(r11)
 129         std     r6, 8(r11)
 130         addi    r11,r11,16
 131 L(loop_start):
 132         /* Load two doublewords, compare and merge in a
 133            single register for speed.  This is an attempt
 134            to speed up the null-checking process for bigger strings.  */
 135
 136         ld      r12, 8(r7)
 137         ldu     r6, 16(r7)
 138         cmpb    r10,r12,r0
 139         cmpb    r9,r6,r0
 140         or      r8,r9,r10     /* Merge everything in one doubleword.  */
 141         cmpdi   cr7,r8,0
 142         beq     cr7,L(loop)
 143
 144
 145         /* OK, one (or both) of the doublewords contains a null byte.  Check
 146            the first doubleword and decrement the address in case the first
 147            doubleword really contains a null byte.  */
 148
 149         addi    r4,r7,-8
 150         cmpdi   cr6,r10,0
 151         addi    r7,r7,-8
 152         bne     cr6,L(done2)
 153
 154         /* The null byte must be in the second doubleword.  Adjust the address
 155            again and move the result of cmpb to r10 so we can calculate the
 156            length.  */
 157
 158         mr      r10,r9
 159         addi    r7,r7,8
 160         b       L(done2)
 161
 162         /* r10 has the output of the cmpb instruction, that is, it contains
 163            0xff in the same position as the null byte in the original
 164            doubleword from the string.  Use that to calculate the length.  */
 165 L(done):
 166         mr      r11,r3
 167 L(done2):
 168 #ifdef __LITTLE_ENDIAN__
 169         addi    r9, r10, -1   /* Form a mask from trailing zeros.  */
 170         andc    r9, r9, r10
 171         popcntd r6, r9        /* Count the bits in the mask.  */
 172 #else
 173         cntlzd  r6,r10        /* Count leading zeros before the match.  */
 174 #endif
 175         subf    r5,r4,r7
 176         srdi    r6,r6,3       /* Convert leading/trailing zeros to bytes.  */
 177         add     r8,r5,r6      /* Compute final length.  */
 178 #ifdef USE_AS_STPCPY
 179         /* stpcpy returns the dest address plus the size not counting the
 180            final '\0'.  */
 181         add     r3,r11,r8
 182 #endif
 183         addi    r8,r8,1       /* Final '/0'.  */
 184
 185         cmpldi  cr6,r8,8
 186         mtocrf  0x01,r8
 187         ble     cr6,L(copy_LE_8)
 188
 189         cmpldi  cr1,r8,16
 190         blt     cr1,8f
 191
 192         /* Handle copies of 0~31 bytes.  */
 193         .align  4
 194 L(copy_LT_32):
 195         /* At least 6 bytes to go.  */
 196         blt     cr1,8f
 197
 198         /* Copy 16 bytes.  */
 199         ld      r6,0(r4)
 200         ld      r8,8(r4)
 201         addi    r4,r4,16
 202         std     r6,0(r11)
 203         std     r8,8(r11)
 204         addi    r11,r11,16
 205 8:      /* Copy 8 bytes.  */
 206         bf      28,L(tail4)
 207         ld      r6,0(r4)
 208         addi    r4,r4,8
 209         std     r6,0(r11)
 210         addi    r11,r11,8
 211
 212         .align  4
 213 /* Copies 4~7 bytes.  */
 214 L(tail4):
 215         bf      29,L(tail2)
 216         lwz     r6,0(r4)
 217         stw     r6,0(r11)
 218         bf      30,L(tail5)
 219         lhz     r7,4(r4)
 220         sth     r7,4(r11)
 221         bflr    31
 222         lbz     r8,6(r4)
 223         stb     r8,6(r11)
 224         blr
 225
 226         .align  4
 227 /* Copies 2~3 bytes.  */
 228 L(tail2):
 229         bf      30,1f
 230         lhz     r6,0(r4)
 231         sth     r6,0(r11)
 232         bflr    31
 233         lbz     r7,2(r4)
 234         stb     r7,2(r11)
 235         blr
 236
 237         .align  4
 238 L(tail5):
 239         bf      31,1f
 240         lbz     r6,4(r4)
 241         stb     r6,4(r11)
 242         blr
 243
 244         .align  4
 245 1:
 246         bflr    31
 247         lbz     r6,0(r4)
 248         stb     r6,0(r11)
 249         blr
 250
 251 /* Handles copies of 0~8 bytes.  */
 252         .align  4
 253 L(copy_LE_8):
 254         bne     cr6,L(tail4)
 255         ld      r6,0(r4)
 256         std     r6,0(r11)
 257         blr
 258 END (FUNC_NAME)
 259
 260 #ifndef USE_AS_STPCPY
 261 libc_hidden_builtin_def (strcpy)
 262 #endif