sysdeps/aarch64/strcmp.S

   1 /* Copyright (C) 2012-2020 Free Software Foundation, Inc.
   2
   3    This file is part of the GNU C Library.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library.  If not, see
  17    <https://www.gnu.org/licenses/>.  */
  18
  19 /* Assumptions:
  20  *
  21  * ARMv8-a, AArch64
  22  */
  23
  24 #include <sysdep.h>
  25
  26 #define REP8_01 0x0101010101010101
  27 #define REP8_7f 0x7f7f7f7f7f7f7f7f
  28 #define REP8_80 0x8080808080808080
  29
  30 /* Parameters and result.  */
  31 #define src1            x0
  32 #define src2            x1
  33 #define result          x0
  34
  35 /* Internal variables.  */
  36 #define data1           x2
  37 #define data1w          w2
  38 #define data2           x3
  39 #define data2w          w3
  40 #define has_nul         x4
  41 #define diff            x5
  42 #define syndrome        x6
  43 #define tmp1            x7
  44 #define tmp2            x8
  45 #define tmp3            x9
  46 #define zeroones        x10
  47 #define pos             x11
  48
  49         /* Start of performance-critical section  -- one 64B cache line.  */
  50 ENTRY_ALIGN(strcmp, 6)
  51
  52         DELOUSE (0)
  53         DELOUSE (1)
  54         eor     tmp1, src1, src2
  55         mov     zeroones, #REP8_01
  56         tst     tmp1, #7
  57         b.ne    L(misaligned8)
  58         ands    tmp1, src1, #7
  59         b.ne    L(mutual_align)
  60         /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
  61            (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
  62            can be done in parallel across the entire word.  */
  63 L(loop_aligned):
  64         ldr     data1, [src1], #8
  65         ldr     data2, [src2], #8
  66 L(start_realigned):
  67         sub     tmp1, data1, zeroones
  68         orr     tmp2, data1, #REP8_7f
  69         eor     diff, data1, data2      /* Non-zero if differences found.  */
  70         bic     has_nul, tmp1, tmp2     /* Non-zero if NUL terminator.  */
  71         orr     syndrome, diff, has_nul
  72         cbz     syndrome, L(loop_aligned)
  73         /* End of performance-critical section  -- one 64B cache line.  */
  74
  75 L(end):
  76 #ifndef __AARCH64EB__
  77         rev     syndrome, syndrome
  78         rev     data1, data1
  79         /* The MS-non-zero bit of the syndrome marks either the first bit
  80            that is different, or the top bit of the first zero byte.
  81            Shifting left now will bring the critical information into the
  82            top bits.  */
  83         clz     pos, syndrome
  84         rev     data2, data2
  85         lsl     data1, data1, pos
  86         lsl     data2, data2, pos
  87         /* But we need to zero-extend (char is unsigned) the value and then
  88            perform a signed 32-bit subtraction.  */
  89         lsr     data1, data1, #56
  90         sub     result, data1, data2, lsr #56
  91         RET
  92 #else
  93         /* For big-endian we cannot use the trick with the syndrome value
  94            as carry-propagation can corrupt the upper bits if the trailing
  95            bytes in the string contain 0x01.  */
  96         /* However, if there is no NUL byte in the dword, we can generate
  97            the result directly.  We can't just subtract the bytes as the
  98            MSB might be significant.  */
  99         cbnz    has_nul, 1f
 100         cmp     data1, data2
 101         cset    result, ne
 102         cneg    result, result, lo
 103         RET
 104 1:
 105         /* Re-compute the NUL-byte detection, using a byte-reversed value.  */
 106         rev     tmp3, data1
 107         sub     tmp1, tmp3, zeroones
 108         orr     tmp2, tmp3, #REP8_7f
 109         bic     has_nul, tmp1, tmp2
 110         rev     has_nul, has_nul
 111         orr     syndrome, diff, has_nul
 112         clz     pos, syndrome
 113         /* The MS-non-zero bit of the syndrome marks either the first bit
 114            that is different, or the top bit of the first zero byte.
 115            Shifting left now will bring the critical information into the
 116            top bits.  */
 117         lsl     data1, data1, pos
 118         lsl     data2, data2, pos
 119         /* But we need to zero-extend (char is unsigned) the value and then
 120            perform a signed 32-bit subtraction.  */
 121         lsr     data1, data1, #56
 122         sub     result, data1, data2, lsr #56
 123         RET
 124 #endif
 125
 126 L(mutual_align):
 127         /* Sources are mutually aligned, but are not currently at an
 128            alignment boundary.  Round down the addresses and then mask off
 129            the bytes that preceed the start point.  */
 130         bic     src1, src1, #7
 131         bic     src2, src2, #7
 132         lsl     tmp1, tmp1, #3          /* Bytes beyond alignment -> bits.  */
 133         ldr     data1, [src1], #8
 134         neg     tmp1, tmp1              /* Bits to alignment -64.  */
 135         ldr     data2, [src2], #8
 136         mov     tmp2, #~0
 137 #ifdef __AARCH64EB__
 138         /* Big-endian.  Early bytes are at MSB.  */
 139         lsl     tmp2, tmp2, tmp1        /* Shift (tmp1 & 63).  */
 140 #else
 141         /* Little-endian.  Early bytes are at LSB.  */
 142         lsr     tmp2, tmp2, tmp1        /* Shift (tmp1 & 63).  */
 143 #endif
 144         orr     data1, data1, tmp2
 145         orr     data2, data2, tmp2
 146         b       L(start_realigned)
 147
 148 L(misaligned8):
 149         /* Align SRC1 to 8 bytes and then compare 8 bytes at a time, always
 150            checking to make sure that we don't access beyond page boundary in
 151            SRC2.  */
 152         tst     src1, #7
 153         b.eq    L(loop_misaligned)
 154 L(do_misaligned):
 155         ldrb    data1w, [src1], #1
 156         ldrb    data2w, [src2], #1
 157         cmp     data1w, #1
 158         ccmp    data1w, data2w, #0, cs  /* NZCV = 0b0000.  */
 159         b.ne    L(done)
 160         tst     src1, #7
 161         b.ne    L(do_misaligned)
 162
 163 L(loop_misaligned):
 164         /* Test if we are within the last dword of the end of a 4K page.  If
 165            yes then jump back to the misaligned loop to copy a byte at a time.  */
 166         and     tmp1, src2, #0xff8
 167         eor     tmp1, tmp1, #0xff8
 168         cbz     tmp1, L(do_misaligned)
 169         ldr     data1, [src1], #8
 170         ldr     data2, [src2], #8
 171
 172         sub     tmp1, data1, zeroones
 173         orr     tmp2, data1, #REP8_7f
 174         eor     diff, data1, data2      /* Non-zero if differences found.  */
 175         bic     has_nul, tmp1, tmp2     /* Non-zero if NUL terminator.  */
 176         orr     syndrome, diff, has_nul
 177         cbz     syndrome, L(loop_misaligned)
 178         b       L(end)
 179
 180 L(done):
 181         sub     result, data1, data2
 182         RET
 183 END(strcmp)
 184 libc_hidden_builtin_def (strcmp)