sysdeps/powerpc/powerpc64/power7/memrchr.S

   1 /* Optimized memrchr implementation for PowerPC64/POWER7 using cmpb insn.
   2    Copyright (C) 2010-2019 Free Software Foundation, Inc.
   3    Contributed by Luis Machado <luisgpm@br.ibm.com>.
   4    This file is part of the GNU C Library.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, see
  18    <http://www.gnu.org/licenses/>.  */
  19
  20 #include <sysdep.h>
  21
  22 /* int [r3] memrchr (char *s [r3], int byte [r4], int size [r5])  */
  23
  24 #ifndef MEMRCHR
  25 # define MEMRCHR __memrchr
  26 #endif
  27         .machine  power7
  28 ENTRY_TOCLESS (MEMRCHR)
  29         CALL_MCOUNT 3
  30         add     r7,r3,r5      /* Calculate the last acceptable address.  */
  31         neg     r0,r7
  32         addi    r7,r7,-1
  33         mr      r10,r3
  34         clrrdi  r6,r7,7
  35         li      r9,3<<5
  36         dcbt    r9,r6,8       /* Stream hint, decreasing addresses.  */
  37
  38         /* Replicate BYTE to doubleword.  */
  39         insrdi  r4,r4,8,48
  40         insrdi  r4,r4,16,32
  41         insrdi  r4,r4,32,0
  42         li      r6,-8
  43         li      r9,-1
  44         rlwinm  r0,r0,3,26,28 /* Calculate padding.  */
  45         clrrdi  r8,r7,3
  46         srd     r9,r9,r0
  47         cmpldi  r5,32
  48         clrrdi  r0,r10,3
  49         ble     L(small_range)
  50
  51 #ifdef __LITTLE_ENDIAN__
  52         ldx     r12,0,r8
  53 #else
  54         ldbrx   r12,0,r8      /* Load reversed doubleword from memory.  */
  55 #endif
  56         cmpb    r3,r12,r4     /* Check for BYTE in DWORD1.  */
  57         and     r3,r3,r9
  58         cmpldi  cr7,r3,0      /* If r3 == 0, no BYTEs have been found.  */
  59         bne     cr7,L(done)
  60
  61         mtcrf   0x01,r8
  62         /* Are we now aligned to a quadword boundary?  If so, skip to
  63            the main loop.  Otherwise, go through the alignment code.  */
  64         bf      28,L(loop_setup)
  65
  66         /* Handle DWORD2 of pair.  */
  67 #ifdef __LITTLE_ENDIAN__
  68         ldx     r12,r8,r6
  69 #else
  70         ldbrx   r12,r8,r6
  71 #endif
  72         addi    r8,r8,-8
  73         cmpb    r3,r12,r4
  74         cmpldi  cr7,r3,0
  75         bne     cr7,L(done)
  76
  77 L(loop_setup):
  78         /* The last dword we want to read in the loop below is the one
  79            containing the first byte of the string, ie. the dword at
  80            s & ~7, or r0.  The first dword read is at r8 - 8, we
  81            read 2 * cnt dwords, so the last dword read will be at
  82            r8 - 8 - 16 * cnt + 8.  Solving for cnt gives
  83            cnt = (r8 - r0) / 16  */
  84         sub     r5,r8,r0
  85         addi    r8,r8,-8
  86         srdi    r9,r5,4       /* Number of loop iterations.  */
  87         mtctr   r9            /* Setup the counter.  */
  88
  89         /* Main loop to look for BYTE backwards in the string.
  90            FIXME: Investigate whether 32 byte align helps with this
  91            9 instruction loop.  */
  92         .align  5
  93 L(loop):
  94         /* Load two doublewords, compare and merge in a
  95            single register for speed.  This is an attempt
  96            to speed up the byte-checking process for bigger strings.  */
  97
  98 #ifdef __LITTLE_ENDIAN__
  99         ldx     r12,0,r8
 100         ldx     r11,r8,r6
 101 #else
 102         ldbrx   r12,0,r8
 103         ldbrx   r11,r8,r6
 104 #endif
 105         cmpb    r3,r12,r4
 106         cmpb    r9,r11,r4
 107         or      r5,r9,r3      /* Merge everything in one doubleword.  */
 108         cmpldi  cr7,r5,0
 109         bne     cr7,L(found)
 110         addi    r8,r8,-16
 111         bdnz    L(loop)
 112
 113         /* We may have one more word to read.  */
 114         cmpld   r8,r0
 115         bnelr
 116
 117 #ifdef __LITTLE_ENDIAN__
 118         ldx     r12,0,r8
 119 #else
 120         ldbrx   r12,0,r8
 121 #endif
 122         cmpb    r3,r12,r4
 123         cmpldi  cr7,r3,0
 124         bne     cr7,L(done)
 125         blr
 126
 127         .align  4
 128 L(found):
 129         /* OK, one (or both) of the dwords contains BYTE.  Check
 130            the first dword.  */
 131         cmpldi  cr6,r3,0
 132         bne     cr6,L(done)
 133
 134         /* BYTE must be in the second word.  Adjust the address
 135            again and move the result of cmpb to r3 so we can calculate the
 136            pointer.  */
 137
 138         mr      r3,r9
 139         addi    r8,r8,-8
 140
 141         /* r3 has the output of the cmpb instruction, that is, it contains
 142            0xff in the same position as BYTE in the original
 143            word from the string.  Use that to calculate the pointer.
 144            We need to make sure BYTE is *before* the end of the
 145            range.  */
 146 L(done):
 147         cntlzd  r9,r3         /* Count leading zeros before the match.  */
 148         cmpld   r8,r0         /* Are we on the last word?  */
 149         srdi    r6,r9,3       /* Convert leading zeros to bytes.  */
 150         addi    r0,r6,-7
 151         sub     r3,r8,r0
 152         cmpld   cr7,r3,r10
 153         bnelr
 154         bgelr   cr7
 155         li      r3,0
 156         blr
 157
 158         .align  4
 159 L(null):
 160         li      r3,0
 161         blr
 162
 163 /* Deals with size <= 32.  */
 164         .align  4
 165 L(small_range):
 166         cmpldi  r5,0
 167         beq     L(null)
 168
 169 #ifdef __LITTLE_ENDIAN__
 170         ldx     r12,0,r8
 171 #else
 172         ldbrx   r12,0,r8      /* Load reversed doubleword from memory.  */
 173 #endif
 174         cmpb    r3,r12,r4     /* Check for BYTE in DWORD1.  */
 175         and     r3,r3,r9
 176         cmpldi  cr7,r3,0
 177         bne     cr7,L(done)
 178
 179         /* Are we done already?  */
 180         cmpld   r8,r0
 181         addi    r8,r8,-8
 182         beqlr
 183
 184         .align  5
 185 L(loop_small):
 186 #ifdef __LITTLE_ENDIAN__
 187         ldx     r12,0,r8
 188 #else
 189         ldbrx   r12,0,r8
 190 #endif
 191         cmpb    r3,r12,r4
 192         cmpld   r8,r0
 193         cmpldi  cr7,r3,0
 194         bne     cr7,L(done)
 195         addi    r8,r8,-8
 196         bne     L(loop_small)
 197         blr
 198
 199 END (MEMRCHR)
 200 weak_alias (__memrchr, memrchr)
 201 libc_hidden_builtin_def (memrchr)