sysdeps/powerpc/powerpc64/power7/memchr.S

   1 /* Optimized memchr implementation for PowerPC64/POWER7 using cmpb insn.
   2    Copyright (C) 2010-2018 Free Software Foundation, Inc.
   3    Contributed by Luis Machado <luisgpm@br.ibm.com>.
   4    This file is part of the GNU C Library.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, see
  18    <http://www.gnu.org/licenses/>.  */
  19
  20 #include <sysdep.h>
  21
  22 /* int [r3] memchr (char *s [r3], int byte [r4], int size [r5])  */
  23
  24 #ifndef MEMCHR
  25 # define MEMCHR __memchr
  26 #endif
  27         .machine  power7
  28 ENTRY_TOCLESS (MEMCHR)
  29         CALL_MCOUNT 3
  30         dcbt    0,r3
  31         clrrdi  r8,r3,3
  32         insrdi  r4,r4,8,48
  33
  34         /* Calculate the last acceptable address and check for possible
  35            addition overflow by using satured math:
  36            r7 = r3 + r5
  37            r7 |= -(r7 < x)  */
  38         add     r7,r3,r5
  39         subfc   r6,r3,r7
  40         subfe   r9,r9,r9
  41         extsw   r6,r9
  42         or      r7,r7,r6
  43
  44         insrdi  r4,r4,16,32
  45         cmpldi  r5,32
  46         li      r9, -1
  47         rlwinm  r6,r3,3,26,28 /* Calculate padding.  */
  48         insrdi  r4,r4,32,0
  49         addi    r7,r7,-1
  50 #ifdef __LITTLE_ENDIAN__
  51         sld     r9,r9,r6
  52 #else
  53         srd     r9,r9,r6
  54 #endif
  55         ble     L(small_range)
  56
  57         ld      r12,0(r8)     /* Load doubleword from memory.  */
  58         cmpb    r3,r12,r4     /* Check for BYTEs in DWORD1.  */
  59         and     r3,r3,r9
  60         clrldi  r5,r7,61      /* Byte count - 1 in last dword.  */
  61         clrrdi  r7,r7,3       /* Address of last doubleword.  */
  62         cmpldi  cr7,r3,0      /* Does r3 indicate we got a hit?  */
  63         bne     cr7,L(done)
  64
  65         mtcrf   0x01,r8
  66         /* Are we now aligned to a quadword boundary?  If so, skip to
  67            the main loop.  Otherwise, go through the alignment code.  */
  68         bt      28,L(loop_setup)
  69
  70         /* Handle DWORD2 of pair.  */
  71         ldu     r12,8(r8)
  72         cmpb    r3,r12,r4
  73         cmpldi  cr7,r3,0
  74         bne     cr7,L(done)
  75
  76 L(loop_setup):
  77         /* The last dword we want to read in the loop below is the one
  78            containing the last byte of the string, ie. the dword at
  79            (s + size - 1) & ~7, or r7.  The first dword read is at
  80            r8 + 8, we read 2 * cnt dwords, so the last dword read will
  81            be at r8 + 8 + 16 * cnt - 8.  Solving for cnt gives
  82            cnt = (r7 - r8) / 16  */
  83         sub     r6,r7,r8
  84         srdi    r6,r6,4       /* Number of loop iterations.  */
  85         mtctr   r6            /* Setup the counter.  */
  86
  87         /* Main loop to look for BYTE in the string.  Since
  88            it's a small loop (8 instructions), align it to 32-bytes.  */
  89         .align  5
  90 L(loop):
  91         /* Load two doublewords, compare and merge in a
  92            single register for speed.  This is an attempt
  93            to speed up the byte-checking process for bigger strings.  */
  94         ld      r12,8(r8)
  95         ldu     r11,16(r8)
  96         cmpb    r3,r12,r4
  97         cmpb    r9,r11,r4
  98         or      r6,r9,r3      /* Merge everything in one doubleword.  */
  99         cmpldi  cr7,r6,0
 100         bne     cr7,L(found)
 101         bdnz    L(loop)
 102
 103         /* We may have one more dword to read.  */
 104         cmpld   r8,r7
 105         beqlr
 106
 107         ldu     r12,8(r8)
 108         cmpb    r3,r12,r4
 109         cmpldi  cr6,r3,0
 110         bne     cr6,L(done)
 111         blr
 112
 113         .align  4
 114 L(found):
 115         /* OK, one (or both) of the doublewords contains BYTE.  Check
 116            the first doubleword and decrement the address in case the first
 117            doubleword really contains BYTE.  */
 118         cmpldi  cr6,r3,0
 119         addi    r8,r8,-8
 120         bne     cr6,L(done)
 121
 122         /* BYTE must be in the second doubleword.  Adjust the address
 123            again and move the result of cmpb to r3 so we can calculate the
 124            pointer.  */
 125
 126         mr      r3,r9
 127         addi    r8,r8,8
 128
 129         /* r3 has the output of the cmpb instruction, that is, it contains
 130            0xff in the same position as BYTE in the original
 131            doubleword from the string.  Use that to calculate the pointer.
 132            We need to make sure BYTE is *before* the end of the range.  */
 133 L(done):
 134 #ifdef __LITTLE_ENDIAN__
 135         addi    r0,r3,-1
 136         andc    r0,r0,r3
 137         popcntd r0,r0         /* Count trailing zeros.  */
 138 #else
 139         cntlzd  r0,r3         /* Count leading zeros before the match.  */
 140 #endif
 141         cmpld   r8,r7         /* Are we on the last dword?  */
 142         srdi    r0,r0,3       /* Convert leading/trailing zeros to bytes.  */
 143         add     r3,r8,r0
 144         cmpld   cr7,r0,r5     /* If on the last dword, check byte offset.  */
 145         bnelr
 146         blelr   cr7
 147         li      r3,0
 148         blr
 149
 150         .align  4
 151 L(null):
 152         li      r3,0
 153         blr
 154
 155 /* Deals with size <= 32.  */
 156         .align  4
 157 L(small_range):
 158         cmpldi  r5,0
 159         beq     L(null)
 160         ld      r12,0(r8)     /* Load word from memory.  */
 161         cmpb    r3,r12,r4     /* Check for BYTE in DWORD1.  */
 162         and     r3,r3,r9
 163         cmpldi  cr7,r3,0
 164         clrldi  r5,r7,61      /* Byte count - 1 in last dword.  */
 165         clrrdi  r7,r7,3       /* Address of last doubleword.  */
 166         cmpld   r8,r7         /* Are we done already?  */
 167         bne     cr7,L(done)
 168         beqlr
 169
 170         ldu     r12,8(r8)
 171         cmpb    r3,r12,r4
 172         cmpldi  cr6,r3,0
 173         cmpld   r8,r7
 174         bne     cr6,L(done)   /* Found something.  */
 175         beqlr                 /* Hit end of string (length).  */
 176
 177         ldu     r12,8(r8)
 178         cmpb    r3,r12,r4
 179         cmpldi  cr6,r3,0
 180         cmpld   r8,r7
 181         bne     cr6,L(done)
 182         beqlr
 183
 184         ldu     r12,8(r8)
 185         cmpb    r3,r12,r4
 186         cmpldi  cr6,r3,0
 187         cmpld   r8,r7
 188         bne     cr6,L(done)
 189         beqlr
 190
 191         ldu     r12,8(r8)
 192         cmpb    r3,r12,r4
 193         cmpldi  cr6,r3,0
 194         bne     cr6,L(done)
 195         blr
 196
 197 END (MEMCHR)
 198 weak_alias (__memchr, memchr)
 199 libc_hidden_builtin_def (memchr)