sysdeps/powerpc/powerpc32/power7/memchr.S

   1 /* Optimized memchr implementation for PowerPC32/POWER7 using cmpb insn.
   2    Copyright (C) 2010-2020 Free Software Foundation, Inc.
   3    Contributed by Luis Machado <luisgpm@br.ibm.com>.
   4    This file is part of the GNU C Library.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, see
  18    <https://www.gnu.org/licenses/>.  */
  19
  20 #include <sysdep.h>
  21
  22 /* int [r3] memchr (char *s [r3], int byte [r4], int size [r5])  */
  23         .machine  power7
  24 ENTRY (__memchr)
  25         CALL_MCOUNT
  26         dcbt    0,r3
  27         clrrwi  r8,r3,2
  28         insrwi  r4,r4,8,16    /* Replicate byte to word.  */
  29
  30         /* Calculate the last acceptable address and check for possible
  31            addition overflow by using satured math:
  32            r7 = r3 + r5
  33            r7 |= -(r7 < x)  */
  34         add     r7,r3,r5
  35         subfc   r6,r3,r7
  36         subfe   r9,r9,r9
  37         or      r7,r7,r9
  38
  39         insrwi  r4,r4,16,0
  40         cmplwi  r5,16
  41         li      r9, -1
  42         rlwinm  r6,r3,3,27,28 /* Calculate padding.  */
  43         addi    r7,r7,-1
  44 #ifdef __LITTLE_ENDIAN__
  45         slw     r9,r9,r6
  46 #else
  47         srw     r9,r9,r6
  48 #endif
  49         ble     L(small_range)
  50
  51         lwz     r12,0(r8)     /* Load word from memory.  */
  52         cmpb    r3,r12,r4     /* Check for BYTEs in WORD1.  */
  53         and     r3,r3,r9
  54         clrlwi  r5,r7,30      /* Byte count - 1 in last word.  */
  55         clrrwi  r7,r7,2       /* Address of last word.  */
  56         cmplwi  cr7,r3,0      /* If r3 == 0, no BYTEs have been found.  */
  57         bne     cr7,L(done)
  58
  59         mtcrf   0x01,r8
  60         /* Are we now aligned to a doubleword boundary?  If so, skip to
  61            the main loop.  Otherwise, go through the alignment code.  */
  62         bt      29,L(loop_setup)
  63
  64         /* Handle WORD2 of pair.  */
  65         lwzu    r12,4(r8)
  66         cmpb    r3,r12,r4
  67         cmplwi  cr7,r3,0
  68         bne     cr7,L(done)
  69
  70 L(loop_setup):
  71         /* The last word we want to read in the loop below is the one
  72            containing the last byte of the string, ie. the word at
  73            (s + size - 1) & ~3, or r7.  The first word read is at
  74            r8 + 4, we read 2 * cnt words, so the last word read will
  75            be at r8 + 4 + 8 * cnt - 4.  Solving for cnt gives
  76            cnt = (r7 - r8) / 8  */
  77         sub     r6,r7,r8
  78         srwi    r6,r6,3       /* Number of loop iterations.  */
  79         mtctr   r6            /* Setup the counter.  */
  80
  81         /* Main loop to look for BYTE in the string.  Since
  82            it's a small loop (8 instructions), align it to 32-bytes.  */
  83         .align  5
  84 L(loop):
  85         /* Load two words, compare and merge in a
  86            single register for speed.  This is an attempt
  87            to speed up the byte-checking process for bigger strings.  */
  88         lwz     r12,4(r8)
  89         lwzu    r11,8(r8)
  90         cmpb    r3,r12,r4
  91         cmpb    r9,r11,r4
  92         or      r6,r9,r3      /* Merge everything in one word.  */
  93         cmplwi  cr7,r6,0
  94         bne     cr7,L(found)
  95         bdnz    L(loop)
  96
  97         /* We may have one more dword to read.  */
  98         cmplw   r8,r7
  99         beqlr
 100
 101         lwzu    r12,4(r8)
 102         cmpb    r3,r12,r4
 103         cmplwi  cr6,r3,0
 104         bne     cr6,L(done)
 105         blr
 106
 107         .align  4
 108 L(found):
 109         /* OK, one (or both) of the words contains BYTE.  Check
 110            the first word and decrement the address in case the first
 111            word really contains BYTE.  */
 112         cmplwi  cr6,r3,0
 113         addi    r8,r8,-4
 114         bne     cr6,L(done)
 115
 116         /* BYTE must be in the second word.  Adjust the address
 117            again and move the result of cmpb to r3 so we can calculate the
 118            pointer.  */
 119
 120         mr      r3,r9
 121         addi    r8,r8,4
 122
 123         /* r3 has the output of the cmpb instruction, that is, it contains
 124            0xff in the same position as BYTE in the original
 125            word from the string.  Use that to calculate the pointer.
 126            We need to make sure BYTE is *before* the end of the range.  */
 127 L(done):
 128 #ifdef __LITTLE_ENDIAN__
 129         addi    r0,r3,-1
 130         andc    r0,r0,r3
 131         popcntw r0,r0         /* Count trailing zeros.  */
 132 #else
 133         cntlzw  r0,r3         /* Count leading zeros before the match.  */
 134 #endif
 135         cmplw   r8,r7         /* Are we on the last word?  */
 136         srwi    r0,r0,3       /* Convert leading/trailing zeros to bytes.  */
 137         add     r3,r8,r0
 138         cmplw   cr7,r0,r5     /* If on the last dword, check byte offset.  */
 139         bnelr
 140         blelr   cr7
 141         li      r3,0
 142         blr
 143
 144         .align  4
 145 L(null):
 146         li      r3,0
 147         blr
 148
 149 /* Deals with size <= 16.  */
 150         .align  4
 151 L(small_range):
 152         cmplwi  r5,0
 153         beq     L(null)
 154         lwz     r12,0(r8)     /* Load word from memory.  */
 155         cmpb    r3,r12,r4     /* Check for BYTE in DWORD1.  */
 156         and     r3,r3,r9
 157         cmplwi  cr7,r3,0
 158         clrlwi  r5,r7,30      /* Byte count - 1 in last word.  */
 159         clrrwi  r7,r7,2       /* Address of last word.  */
 160         cmplw   r8,r7         /* Are we done already?  */
 161         bne     cr7,L(done)
 162         beqlr
 163
 164         lwzu    r12,4(r8)
 165         cmpb    r3,r12,r4
 166         cmplwi  cr6,r3,0
 167         cmplw   r8,r7
 168         bne     cr6,L(done)
 169         beqlr
 170
 171         lwzu    r12,4(r8)
 172         cmpb    r3,r12,r4
 173         cmplwi  cr6,r3,0
 174         cmplw   r8,r7
 175         bne     cr6,L(done)
 176         beqlr
 177
 178         lwzu    r12,4(r8)
 179         cmpb    r3,r12,r4
 180         cmplwi  cr6,r3,0
 181         cmplw   r8,r7
 182         bne     cr6,L(done)
 183         beqlr
 184
 185         lwzu    r12,4(r8)
 186         cmpb    r3,r12,r4
 187         cmplwi  cr6,r3,0
 188         bne     cr6,L(done)
 189         blr
 190
 191 END (__memchr)
 192 weak_alias (__memchr, memchr)
 193 libc_hidden_builtin_def (memchr)