sysdeps/powerpc/powerpc64/power7/memrchr.S

   1 /* Optimized memrchr implementation for PowerPC64/POWER7 using cmpb insn.
   2    Copyright (C) 2010-2016 Free Software Foundation, Inc.
   3    Contributed by Luis Machado <luisgpm@br.ibm.com>.
   4    This file is part of the GNU C Library.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, see
  18    <http://www.gnu.org/licenses/>.  */
  19
  20 #include <sysdep.h>
  21
  22 /* int [r3] memrchr (char *s [r3], int byte [r4], int size [r5])  */
  23         .machine  power7
  24 ENTRY (__memrchr)
  25         CALL_MCOUNT 3
  26         add     r7,r3,r5      /* Calculate the last acceptable address.  */
  27         neg     r0,r7
  28         addi    r7,r7,-1
  29         mr      r10,r3
  30         clrrdi  r6,r7,7
  31         li      r9,3<<5
  32         dcbt    r9,r6,8       /* Stream hint, decreasing addresses.  */
  33
  34         /* Replicate BYTE to doubleword.  */
  35         insrdi  r4,r4,8,48
  36         insrdi  r4,r4,16,32
  37         insrdi  r4,r4,32,0
  38         li      r6,-8
  39         li      r9,-1
  40         rlwinm  r0,r0,3,26,28 /* Calculate padding.  */
  41         clrrdi  r8,r7,3
  42         srd     r9,r9,r0
  43         cmpldi  r5,32
  44         clrrdi  r0,r10,3
  45         ble     L(small_range)
  46
  47 #ifdef __LITTLE_ENDIAN__
  48         ldx     r12,0,r8
  49 #else
  50         ldbrx   r12,0,r8      /* Load reversed doubleword from memory.  */
  51 #endif
  52         cmpb    r3,r12,r4     /* Check for BYTE in DWORD1.  */
  53         and     r3,r3,r9
  54         cmpldi  cr7,r3,0      /* If r3 == 0, no BYTEs have been found.  */
  55         bne     cr7,L(done)
  56
  57         mtcrf   0x01,r8
  58         /* Are we now aligned to a quadword boundary?  If so, skip to
  59            the main loop.  Otherwise, go through the alignment code.  */
  60         bf      28,L(loop_setup)
  61
  62         /* Handle DWORD2 of pair.  */
  63 #ifdef __LITTLE_ENDIAN__
  64         ldx     r12,r8,r6
  65 #else
  66         ldbrx   r12,r8,r6
  67 #endif
  68         addi    r8,r8,-8
  69         cmpb    r3,r12,r4
  70         cmpldi  cr7,r3,0
  71         bne     cr7,L(done)
  72
  73 L(loop_setup):
  74         /* The last dword we want to read in the loop below is the one
  75            containing the first byte of the string, ie. the dword at
  76            s & ~7, or r0.  The first dword read is at r8 - 8, we
  77            read 2 * cnt dwords, so the last dword read will be at
  78            r8 - 8 - 16 * cnt + 8.  Solving for cnt gives
  79            cnt = (r8 - r0) / 16  */
  80         sub     r5,r8,r0
  81         addi    r8,r8,-8
  82         srdi    r9,r5,4       /* Number of loop iterations.  */
  83         mtctr   r9            /* Setup the counter.  */
  84
  85         /* Main loop to look for BYTE backwards in the string.
  86            FIXME: Investigate whether 32 byte align helps with this
  87            9 instruction loop.  */
  88         .align  5
  89 L(loop):
  90         /* Load two doublewords, compare and merge in a
  91            single register for speed.  This is an attempt
  92            to speed up the byte-checking process for bigger strings.  */
  93
  94 #ifdef __LITTLE_ENDIAN__
  95         ldx     r12,0,r8
  96         ldx     r11,r8,r6
  97 #else
  98         ldbrx   r12,0,r8
  99         ldbrx   r11,r8,r6
 100 #endif
 101         cmpb    r3,r12,r4
 102         cmpb    r9,r11,r4
 103         or      r5,r9,r3      /* Merge everything in one doubleword.  */
 104         cmpldi  cr7,r5,0
 105         bne     cr7,L(found)
 106         addi    r8,r8,-16
 107         bdnz    L(loop)
 108
 109         /* We may have one more word to read.  */
 110         cmpld   r8,r0
 111         bnelr
 112
 113 #ifdef __LITTLE_ENDIAN__
 114         ldx     r12,0,r8
 115 #else
 116         ldbrx   r12,0,r8
 117 #endif
 118         cmpb    r3,r12,r4
 119         cmpldi  cr7,r3,0
 120         bne     cr7,L(done)
 121         blr
 122
 123         .align  4
 124 L(found):
 125         /* OK, one (or both) of the dwords contains BYTE.  Check
 126            the first dword.  */
 127         cmpldi  cr6,r3,0
 128         bne     cr6,L(done)
 129
 130         /* BYTE must be in the second word.  Adjust the address
 131            again and move the result of cmpb to r3 so we can calculate the
 132            pointer.  */
 133
 134         mr      r3,r9
 135         addi    r8,r8,-8
 136
 137         /* r3 has the output of the cmpb instruction, that is, it contains
 138            0xff in the same position as BYTE in the original
 139            word from the string.  Use that to calculate the pointer.
 140            We need to make sure BYTE is *before* the end of the
 141            range.  */
 142 L(done):
 143         cntlzd  r9,r3         /* Count leading zeros before the match.  */
 144         cmpld   r8,r0         /* Are we on the last word?  */
 145         srdi    r6,r9,3       /* Convert leading zeros to bytes.  */
 146         addi    r0,r6,-7
 147         sub     r3,r8,r0
 148         cmpld   cr7,r3,r10
 149         bnelr
 150         bgelr   cr7
 151         li      r3,0
 152         blr
 153
 154         .align  4
 155 L(null):
 156         li      r3,0
 157         blr
 158
 159 /* Deals with size <= 32.  */
 160         .align  4
 161 L(small_range):
 162         cmpldi  r5,0
 163         beq     L(null)
 164
 165 #ifdef __LITTLE_ENDIAN__
 166         ldx     r12,0,r8
 167 #else
 168         ldbrx   r12,0,r8      /* Load reversed doubleword from memory.  */
 169 #endif
 170         cmpb    r3,r12,r4     /* Check for BYTE in DWORD1.  */
 171         and     r3,r3,r9
 172         cmpldi  cr7,r3,0
 173         bne     cr7,L(done)
 174
 175         /* Are we done already?  */
 176         cmpld   r8,r0
 177         addi    r8,r8,-8
 178         beqlr
 179
 180         .align  5
 181 L(loop_small):
 182 #ifdef __LITTLE_ENDIAN__
 183         ldx     r12,0,r8
 184 #else
 185         ldbrx   r12,0,r8
 186 #endif
 187         cmpb    r3,r12,r4
 188         cmpld   r8,r0
 189         cmpldi  cr7,r3,0
 190         bne     cr7,L(done)
 191         addi    r8,r8,-8
 192         bne     L(loop_small)
 193         blr
 194
 195 END (__memrchr)
 196 weak_alias (__memrchr, memrchr)
 197 libc_hidden_builtin_def (memrchr)