sysdeps/aarch64/strnlen.S

   1 /* strnlen - calculate the length of a string with limit.
   2
   3    Copyright (C) 2013-2016 Free Software Foundation, Inc.
   4
   5    This file is part of the GNU C Library.
   6
   7    The GNU C Library is free software; you can redistribute it and/or
   8    modify it under the terms of the GNU Lesser General Public
   9    License as published by the Free Software Foundation; either
  10    version 2.1 of the License, or (at your option) any later version.
  11
  12    The GNU C Library is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15    Lesser General Public License for more details.
  16
  17    You should have received a copy of the GNU Lesser General Public
  18    License along with the GNU C Library.  If not, see
  19    <http://www.gnu.org/licenses/>.  */
  20
  21 #include <sysdep.h>
  22
  23 /* Assumptions:
  24  *
  25  * ARMv8-a, AArch64
  26  */
  27
  28 /* Arguments and results.  */
  29 #define srcin           x0
  30 #define len             x0
  31 #define limit           x1
  32
  33 /* Locals and temporaries.  */
  34 #define src             x2
  35 #define data1           x3
  36 #define data2           x4
  37 #define data2a          x5
  38 #define has_nul1        x6
  39 #define has_nul2        x7
  40 #define tmp1            x8
  41 #define tmp2            x9
  42 #define tmp3            x10
  43 #define tmp4            x11
  44 #define zeroones        x12
  45 #define pos             x13
  46 #define limit_wd        x14
  47
  48 #define REP8_01 0x0101010101010101
  49 #define REP8_7f 0x7f7f7f7f7f7f7f7f
  50 #define REP8_80 0x8080808080808080
  51
  52 ENTRY_ALIGN_AND_PAD (__strnlen, 6, 9)
  53         cbz     limit, L(hit_limit)
  54         mov     zeroones, #REP8_01
  55         bic     src, srcin, #15
  56         ands    tmp1, srcin, #15
  57         b.ne    L(misaligned)
  58         /* Calculate the number of full and partial words -1.  */
  59         sub     limit_wd, limit, #1     /* Limit != 0, so no underflow.  */
  60         lsr     limit_wd, limit_wd, #4  /* Convert to Qwords.  */
  61
  62         /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
  63            (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
  64            can be done in parallel across the entire word.  */
  65         /* The inner loop deals with two Dwords at a time.  This has a
  66            slightly higher start-up cost, but we should win quite quickly,
  67            especially on cores with a high number of issue slots per
  68            cycle, as we get much better parallelism out of the operations.  */
  69
  70         /* Start of critial section -- keep to one 64Byte cache line.  */
  71 L(loop):
  72         ldp     data1, data2, [src], #16
  73 L(realigned):
  74         sub     tmp1, data1, zeroones
  75         orr     tmp2, data1, #REP8_7f
  76         sub     tmp3, data2, zeroones
  77         orr     tmp4, data2, #REP8_7f
  78         bic     has_nul1, tmp1, tmp2
  79         bic     has_nul2, tmp3, tmp4
  80         subs    limit_wd, limit_wd, #1
  81         orr     tmp1, has_nul1, has_nul2
  82         ccmp    tmp1, #0, #0, pl        /* NZCV = 0000  */
  83         b.eq    L(loop)
  84         /* End of critical section -- keep to one 64Byte cache line.  */
  85
  86         orr     tmp1, has_nul1, has_nul2
  87         cbz     tmp1, L(hit_limit)      /* No null in final Qword.  */
  88
  89         /* We know there's a null in the final Qword.  The easiest thing
  90            to do now is work out the length of the string and return
  91            MIN (len, limit).  */
  92
  93         sub     len, src, srcin
  94         cbz     has_nul1, L(nul_in_data2)
  95 #ifdef __AARCH64EB__
  96         mov     data2, data1
  97 #endif
  98         sub     len, len, #8
  99         mov     has_nul2, has_nul1
 100 L(nul_in_data2):
 101 #ifdef __AARCH64EB__
 102         /* For big-endian, carry propagation (if the final byte in the
 103            string is 0x01) means we cannot use has_nul directly.  The
 104            easiest way to get the correct byte is to byte-swap the data
 105            and calculate the syndrome a second time.  */
 106         rev     data2, data2
 107         sub     tmp1, data2, zeroones
 108         orr     tmp2, data2, #REP8_7f
 109         bic     has_nul2, tmp1, tmp2
 110 #endif
 111         sub     len, len, #8
 112         rev     has_nul2, has_nul2
 113         clz     pos, has_nul2
 114         add     len, len, pos, lsr #3           /* Bits to bytes.  */
 115         cmp     len, limit
 116         csel    len, len, limit, ls             /* Return the lower value.  */
 117         RET
 118
 119 L(misaligned):
 120         /* Deal with a partial first word.
 121            We're doing two things in parallel here;
 122            1) Calculate the number of words (but avoiding overflow if
 123               limit is near ULONG_MAX) - to do this we need to work out
 124               limit + tmp1 - 1 as a 65-bit value before shifting it;
 125            2) Load and mask the initial data words - we force the bytes
 126               before the ones we are interested in to 0xff - this ensures
 127               early bytes will not hit any zero detection.  */
 128         sub     limit_wd, limit, #1
 129         neg     tmp4, tmp1
 130         cmp     tmp1, #8
 131
 132         and     tmp3, limit_wd, #15
 133         lsr     limit_wd, limit_wd, #4
 134         mov     tmp2, #~0
 135
 136         ldp     data1, data2, [src], #16
 137         lsl     tmp4, tmp4, #3          /* Bytes beyond alignment -> bits.  */
 138         add     tmp3, tmp3, tmp1
 139
 140 #ifdef __AARCH64EB__
 141         /* Big-endian.  Early bytes are at MSB.  */
 142         lsl     tmp2, tmp2, tmp4        /* Shift (tmp1 & 63).  */
 143 #else
 144         /* Little-endian.  Early bytes are at LSB.  */
 145         lsr     tmp2, tmp2, tmp4        /* Shift (tmp1 & 63).  */
 146 #endif
 147         add     limit_wd, limit_wd, tmp3, lsr #4
 148
 149         orr     data1, data1, tmp2
 150         orr     data2a, data2, tmp2
 151
 152         csinv   data1, data1, xzr, le
 153         csel    data2, data2, data2a, le
 154         b       L(realigned)
 155
 156 L(hit_limit):
 157         mov     len, limit
 158         RET
 159 END (__strnlen)
 160 libc_hidden_def (__strnlen)
 161 weak_alias (__strnlen, strnlen)
 162 libc_hidden_def (strnlen)