sysdeps/i386/i686/multiarch/memset-sse2.S

   1 /* memset with SSE2
   2    Copyright (C) 2010-2022 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <https://www.gnu.org/licenses/>.  */
  18
  19 #if IS_IN (libc)
  20
  21 #include <sysdep.h>
  22 #include "asm-syntax.h"
  23
  24 #define CFI_PUSH(REG)                                           \
  25   cfi_adjust_cfa_offset (4);                                    \
  26   cfi_rel_offset (REG, 0)
  27
  28 #define CFI_POP(REG)                                            \
  29   cfi_adjust_cfa_offset (-4);                                   \
  30   cfi_restore (REG)
  31
  32 #define PUSH(REG)       pushl REG; CFI_PUSH (REG)
  33 #define POP(REG)        popl REG; CFI_POP (REG)
  34
  35 #ifdef USE_AS_BZERO
  36 # define DEST           PARMS
  37 # define LEN            DEST+4
  38 # define SETRTNVAL
  39 #else
  40 # define DEST           PARMS
  41 # define CHR            DEST+4
  42 # define LEN            CHR+4
  43 # define SETRTNVAL      movl DEST(%esp), %eax
  44 #endif
  45
  46 #ifdef PIC
  47 # define ENTRANCE       PUSH (%ebx);
  48 # define RETURN_END     POP (%ebx); ret
  49 # define RETURN         RETURN_END; CFI_PUSH (%ebx)
  50 # define PARMS          8               /* Preserve EBX.  */
  51 # define JMPTBL(I, B)   I - B
  52
  53 /* Load an entry in a jump table into EBX and branch to it.  TABLE is a
  54    jump table with relative offsets.   */
  55 # define BRANCH_TO_JMPTBL_ENTRY(TABLE)                          \
  56     /* We first load PC into EBX.  */                           \
  57     SETUP_PIC_REG(bx);                                          \
  58     /* Get the address of the jump table.  */                   \
  59     add         $(TABLE - .), %ebx;                             \
  60     /* Get the entry and convert the relative offset to the     \
  61        absolute address.  */                                    \
  62     add         (%ebx,%ecx,4), %ebx;                            \
  63     add         %ecx, %edx;                                     \
  64     /* We loaded the jump table and adjusted EDX. Go.  */       \
  65     _CET_NOTRACK jmp    *%ebx
  66 #else
  67 # define ENTRANCE
  68 # define RETURN_END     ret
  69 # define RETURN         RETURN_END
  70 # define PARMS          4
  71 # define JMPTBL(I, B)   I
  72
  73 /* Branch to an entry in a jump table.  TABLE is a jump table with
  74    absolute offsets.  */
  75 # define BRANCH_TO_JMPTBL_ENTRY(TABLE)                          \
  76     add         %ecx, %edx;                                     \
  77     _CET_NOTRACK jmp *TABLE(,%ecx,4)
  78 #endif
  79
  80         .section .text.sse2,"ax",@progbits
  81 #if defined SHARED && IS_IN (libc) && !defined USE_AS_BZERO
  82 ENTRY (__memset_chk_sse2)
  83         movl    12(%esp), %eax
  84         cmpl    %eax, 16(%esp)
  85         jb      HIDDEN_JUMPTARGET (__chk_fail)
  86 END (__memset_chk_sse2)
  87 #endif
  88 ENTRY (__memset_sse2)
  89         ENTRANCE
  90
  91         movl    LEN(%esp), %ecx
  92 #ifdef USE_AS_BZERO
  93         xor     %eax, %eax
  94 #else
  95         movzbl  CHR(%esp), %eax
  96         movb    %al, %ah
  97         /* Fill the whole EAX with pattern.  */
  98         movl    %eax, %edx
  99         shl     $16, %eax
 100         or      %edx, %eax
 101 #endif
 102         movl    DEST(%esp), %edx
 103         cmp     $32, %ecx
 104         jae     L(32bytesormore)
 105
 106 L(write_less32bytes):
 107         BRANCH_TO_JMPTBL_ENTRY (L(table_less_32bytes))
 108
 109
 110         .pushsection .rodata.sse2,"a",@progbits
 111         ALIGN (2)
 112 L(table_less_32bytes):
 113         .int    JMPTBL (L(write_0bytes), L(table_less_32bytes))
 114         .int    JMPTBL (L(write_1bytes), L(table_less_32bytes))
 115         .int    JMPTBL (L(write_2bytes), L(table_less_32bytes))
 116         .int    JMPTBL (L(write_3bytes), L(table_less_32bytes))
 117         .int    JMPTBL (L(write_4bytes), L(table_less_32bytes))
 118         .int    JMPTBL (L(write_5bytes), L(table_less_32bytes))
 119         .int    JMPTBL (L(write_6bytes), L(table_less_32bytes))
 120         .int    JMPTBL (L(write_7bytes), L(table_less_32bytes))
 121         .int    JMPTBL (L(write_8bytes), L(table_less_32bytes))
 122         .int    JMPTBL (L(write_9bytes), L(table_less_32bytes))
 123         .int    JMPTBL (L(write_10bytes), L(table_less_32bytes))
 124         .int    JMPTBL (L(write_11bytes), L(table_less_32bytes))
 125         .int    JMPTBL (L(write_12bytes), L(table_less_32bytes))
 126         .int    JMPTBL (L(write_13bytes), L(table_less_32bytes))
 127         .int    JMPTBL (L(write_14bytes), L(table_less_32bytes))
 128         .int    JMPTBL (L(write_15bytes), L(table_less_32bytes))
 129         .int    JMPTBL (L(write_16bytes), L(table_less_32bytes))
 130         .int    JMPTBL (L(write_17bytes), L(table_less_32bytes))
 131         .int    JMPTBL (L(write_18bytes), L(table_less_32bytes))
 132         .int    JMPTBL (L(write_19bytes), L(table_less_32bytes))
 133         .int    JMPTBL (L(write_20bytes), L(table_less_32bytes))
 134         .int    JMPTBL (L(write_21bytes), L(table_less_32bytes))
 135         .int    JMPTBL (L(write_22bytes), L(table_less_32bytes))
 136         .int    JMPTBL (L(write_23bytes), L(table_less_32bytes))
 137         .int    JMPTBL (L(write_24bytes), L(table_less_32bytes))
 138         .int    JMPTBL (L(write_25bytes), L(table_less_32bytes))
 139         .int    JMPTBL (L(write_26bytes), L(table_less_32bytes))
 140         .int    JMPTBL (L(write_27bytes), L(table_less_32bytes))
 141         .int    JMPTBL (L(write_28bytes), L(table_less_32bytes))
 142         .int    JMPTBL (L(write_29bytes), L(table_less_32bytes))
 143         .int    JMPTBL (L(write_30bytes), L(table_less_32bytes))
 144         .int    JMPTBL (L(write_31bytes), L(table_less_32bytes))
 145         .popsection
 146
 147         ALIGN (4)
 148 L(write_28bytes):
 149         movl    %eax, -28(%edx)
 150 L(write_24bytes):
 151         movl    %eax, -24(%edx)
 152 L(write_20bytes):
 153         movl    %eax, -20(%edx)
 154 L(write_16bytes):
 155         movl    %eax, -16(%edx)
 156 L(write_12bytes):
 157         movl    %eax, -12(%edx)
 158 L(write_8bytes):
 159         movl    %eax, -8(%edx)
 160 L(write_4bytes):
 161         movl    %eax, -4(%edx)
 162 L(write_0bytes):
 163         SETRTNVAL
 164         RETURN
 165
 166         ALIGN (4)
 167 L(write_29bytes):
 168         movl    %eax, -29(%edx)
 169 L(write_25bytes):
 170         movl    %eax, -25(%edx)
 171 L(write_21bytes):
 172         movl    %eax, -21(%edx)
 173 L(write_17bytes):
 174         movl    %eax, -17(%edx)
 175 L(write_13bytes):
 176         movl    %eax, -13(%edx)
 177 L(write_9bytes):
 178         movl    %eax, -9(%edx)
 179 L(write_5bytes):
 180         movl    %eax, -5(%edx)
 181 L(write_1bytes):
 182         movb    %al, -1(%edx)
 183         SETRTNVAL
 184         RETURN
 185
 186         ALIGN (4)
 187 L(write_30bytes):
 188         movl    %eax, -30(%edx)
 189 L(write_26bytes):
 190         movl    %eax, -26(%edx)
 191 L(write_22bytes):
 192         movl    %eax, -22(%edx)
 193 L(write_18bytes):
 194         movl    %eax, -18(%edx)
 195 L(write_14bytes):
 196         movl    %eax, -14(%edx)
 197 L(write_10bytes):
 198         movl    %eax, -10(%edx)
 199 L(write_6bytes):
 200         movl    %eax, -6(%edx)
 201 L(write_2bytes):
 202         movw    %ax, -2(%edx)
 203         SETRTNVAL
 204         RETURN
 205
 206         ALIGN (4)
 207 L(write_31bytes):
 208         movl    %eax, -31(%edx)
 209 L(write_27bytes):
 210         movl    %eax, -27(%edx)
 211 L(write_23bytes):
 212         movl    %eax, -23(%edx)
 213 L(write_19bytes):
 214         movl    %eax, -19(%edx)
 215 L(write_15bytes):
 216         movl    %eax, -15(%edx)
 217 L(write_11bytes):
 218         movl    %eax, -11(%edx)
 219 L(write_7bytes):
 220         movl    %eax, -7(%edx)
 221 L(write_3bytes):
 222         movw    %ax, -3(%edx)
 223         movb    %al, -1(%edx)
 224         SETRTNVAL
 225         RETURN
 226
 227         ALIGN (4)
 228 /* ECX > 32 and EDX is 4 byte aligned.  */
 229 L(32bytesormore):
 230         /* Fill xmm0 with the pattern.  */
 231 #ifdef USE_AS_BZERO
 232         pxor    %xmm0, %xmm0
 233 #else
 234         movd    %eax, %xmm0
 235         pshufd  $0, %xmm0, %xmm0
 236 #endif
 237         testl   $0xf, %edx
 238         jz      L(aligned_16)
 239 /* ECX > 32 and EDX is not 16 byte aligned.  */
 240 L(not_aligned_16):
 241         movdqu  %xmm0, (%edx)
 242         movl    %edx, %eax
 243         and     $-16, %edx
 244         add     $16, %edx
 245         sub     %edx, %eax
 246         add     %eax, %ecx
 247         movd    %xmm0, %eax
 248
 249         ALIGN (4)
 250 L(aligned_16):
 251         cmp     $128, %ecx
 252         jae     L(128bytesormore)
 253
 254 L(aligned_16_less128bytes):
 255         BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes))
 256
 257         ALIGN (4)
 258 L(128bytesormore):
 259 #ifdef SHARED_CACHE_SIZE
 260         PUSH (%ebx)
 261         mov     $SHARED_CACHE_SIZE, %ebx
 262 #else
 263 # ifdef PIC
 264         SETUP_PIC_REG(bx)
 265         add     $_GLOBAL_OFFSET_TABLE_, %ebx
 266         mov     __x86_shared_cache_size@GOTOFF(%ebx), %ebx
 267 # else
 268         PUSH (%ebx)
 269         mov     __x86_shared_cache_size, %ebx
 270 # endif
 271 #endif
 272         cmp     %ebx, %ecx
 273         jae     L(128bytesormore_nt_start)
 274
 275
 276 #ifdef DATA_CACHE_SIZE
 277         POP (%ebx)
 278 # define RESTORE_EBX_STATE CFI_PUSH (%ebx)
 279         cmp     $DATA_CACHE_SIZE, %ecx
 280 #else
 281 # ifdef PIC
 282 #  define RESTORE_EBX_STATE
 283         SETUP_PIC_REG(bx)
 284         add     $_GLOBAL_OFFSET_TABLE_, %ebx
 285         cmp     __x86_data_cache_size@GOTOFF(%ebx), %ecx
 286 # else
 287         POP (%ebx)
 288 #  define RESTORE_EBX_STATE CFI_PUSH (%ebx)
 289         cmp     __x86_data_cache_size, %ecx
 290 # endif
 291 #endif
 292
 293         jae     L(128bytes_L2_normal)
 294         subl    $128, %ecx
 295 L(128bytesormore_normal):
 296         sub     $128, %ecx
 297         movdqa  %xmm0, (%edx)
 298         movdqa  %xmm0, 0x10(%edx)
 299         movdqa  %xmm0, 0x20(%edx)
 300         movdqa  %xmm0, 0x30(%edx)
 301         movdqa  %xmm0, 0x40(%edx)
 302         movdqa  %xmm0, 0x50(%edx)
 303         movdqa  %xmm0, 0x60(%edx)
 304         movdqa  %xmm0, 0x70(%edx)
 305         lea     128(%edx), %edx
 306         jb      L(128bytesless_normal)
 307
 308
 309         sub     $128, %ecx
 310         movdqa  %xmm0, (%edx)
 311         movdqa  %xmm0, 0x10(%edx)
 312         movdqa  %xmm0, 0x20(%edx)
 313         movdqa  %xmm0, 0x30(%edx)
 314         movdqa  %xmm0, 0x40(%edx)
 315         movdqa  %xmm0, 0x50(%edx)
 316         movdqa  %xmm0, 0x60(%edx)
 317         movdqa  %xmm0, 0x70(%edx)
 318         lea     128(%edx), %edx
 319         jae     L(128bytesormore_normal)
 320
 321 L(128bytesless_normal):
 322         add     $128, %ecx
 323         BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes))
 324
 325         ALIGN (4)
 326 L(128bytes_L2_normal):
 327         prefetcht0      0x380(%edx)
 328         prefetcht0      0x3c0(%edx)
 329         sub     $128, %ecx
 330         movdqa  %xmm0, (%edx)
 331         movaps  %xmm0, 0x10(%edx)
 332         movaps  %xmm0, 0x20(%edx)
 333         movaps  %xmm0, 0x30(%edx)
 334         movaps  %xmm0, 0x40(%edx)
 335         movaps  %xmm0, 0x50(%edx)
 336         movaps  %xmm0, 0x60(%edx)
 337         movaps  %xmm0, 0x70(%edx)
 338         add     $128, %edx
 339         cmp     $128, %ecx
 340         jae     L(128bytes_L2_normal)
 341
 342 L(128bytesless_L2_normal):
 343         BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes))
 344
 345         RESTORE_EBX_STATE
 346 L(128bytesormore_nt_start):
 347         sub     %ebx, %ecx
 348         ALIGN (4)
 349 L(128bytesormore_shared_cache_loop):
 350         prefetcht0      0x3c0(%edx)
 351         prefetcht0      0x380(%edx)
 352         sub     $0x80, %ebx
 353         movdqa  %xmm0, (%edx)
 354         movdqa  %xmm0, 0x10(%edx)
 355         movdqa  %xmm0, 0x20(%edx)
 356         movdqa  %xmm0, 0x30(%edx)
 357         movdqa  %xmm0, 0x40(%edx)
 358         movdqa  %xmm0, 0x50(%edx)
 359         movdqa  %xmm0, 0x60(%edx)
 360         movdqa  %xmm0, 0x70(%edx)
 361         add     $0x80, %edx
 362         cmp     $0x80, %ebx
 363         jae     L(128bytesormore_shared_cache_loop)
 364         cmp     $0x80, %ecx
 365         jb      L(shared_cache_loop_end)
 366         ALIGN (4)
 367 L(128bytesormore_nt):
 368         sub     $0x80, %ecx
 369         movntdq %xmm0, (%edx)
 370         movntdq %xmm0, 0x10(%edx)
 371         movntdq %xmm0, 0x20(%edx)
 372         movntdq %xmm0, 0x30(%edx)
 373         movntdq %xmm0, 0x40(%edx)
 374         movntdq %xmm0, 0x50(%edx)
 375         movntdq %xmm0, 0x60(%edx)
 376         movntdq %xmm0, 0x70(%edx)
 377         add     $0x80, %edx
 378         cmp     $0x80, %ecx
 379         jae     L(128bytesormore_nt)
 380         sfence
 381 L(shared_cache_loop_end):
 382 #if defined DATA_CACHE_SIZE || !defined PIC
 383         POP (%ebx)
 384 #endif
 385         BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes))
 386
 387
 388         .pushsection .rodata.sse2,"a",@progbits
 389         ALIGN (2)
 390 L(table_16_128bytes):
 391         .int    JMPTBL (L(aligned_16_0bytes), L(table_16_128bytes))
 392         .int    JMPTBL (L(aligned_16_1bytes), L(table_16_128bytes))
 393         .int    JMPTBL (L(aligned_16_2bytes), L(table_16_128bytes))
 394         .int    JMPTBL (L(aligned_16_3bytes), L(table_16_128bytes))
 395         .int    JMPTBL (L(aligned_16_4bytes), L(table_16_128bytes))
 396         .int    JMPTBL (L(aligned_16_5bytes), L(table_16_128bytes))
 397         .int    JMPTBL (L(aligned_16_6bytes), L(table_16_128bytes))
 398         .int    JMPTBL (L(aligned_16_7bytes), L(table_16_128bytes))
 399         .int    JMPTBL (L(aligned_16_8bytes), L(table_16_128bytes))
 400         .int    JMPTBL (L(aligned_16_9bytes), L(table_16_128bytes))
 401         .int    JMPTBL (L(aligned_16_10bytes), L(table_16_128bytes))
 402         .int    JMPTBL (L(aligned_16_11bytes), L(table_16_128bytes))
 403         .int    JMPTBL (L(aligned_16_12bytes), L(table_16_128bytes))
 404         .int    JMPTBL (L(aligned_16_13bytes), L(table_16_128bytes))
 405         .int    JMPTBL (L(aligned_16_14bytes), L(table_16_128bytes))
 406         .int    JMPTBL (L(aligned_16_15bytes), L(table_16_128bytes))
 407         .int    JMPTBL (L(aligned_16_16bytes), L(table_16_128bytes))
 408         .int    JMPTBL (L(aligned_16_17bytes), L(table_16_128bytes))
 409         .int    JMPTBL (L(aligned_16_18bytes), L(table_16_128bytes))
 410         .int    JMPTBL (L(aligned_16_19bytes), L(table_16_128bytes))
 411         .int    JMPTBL (L(aligned_16_20bytes), L(table_16_128bytes))
 412         .int    JMPTBL (L(aligned_16_21bytes), L(table_16_128bytes))
 413         .int    JMPTBL (L(aligned_16_22bytes), L(table_16_128bytes))
 414         .int    JMPTBL (L(aligned_16_23bytes), L(table_16_128bytes))
 415         .int    JMPTBL (L(aligned_16_24bytes), L(table_16_128bytes))
 416         .int    JMPTBL (L(aligned_16_25bytes), L(table_16_128bytes))
 417         .int    JMPTBL (L(aligned_16_26bytes), L(table_16_128bytes))
 418         .int    JMPTBL (L(aligned_16_27bytes), L(table_16_128bytes))
 419         .int    JMPTBL (L(aligned_16_28bytes), L(table_16_128bytes))
 420         .int    JMPTBL (L(aligned_16_29bytes), L(table_16_128bytes))
 421         .int    JMPTBL (L(aligned_16_30bytes), L(table_16_128bytes))
 422         .int    JMPTBL (L(aligned_16_31bytes), L(table_16_128bytes))
 423         .int    JMPTBL (L(aligned_16_32bytes), L(table_16_128bytes))
 424         .int    JMPTBL (L(aligned_16_33bytes), L(table_16_128bytes))
 425         .int    JMPTBL (L(aligned_16_34bytes), L(table_16_128bytes))
 426         .int    JMPTBL (L(aligned_16_35bytes), L(table_16_128bytes))
 427         .int    JMPTBL (L(aligned_16_36bytes), L(table_16_128bytes))
 428         .int    JMPTBL (L(aligned_16_37bytes), L(table_16_128bytes))
 429         .int    JMPTBL (L(aligned_16_38bytes), L(table_16_128bytes))
 430         .int    JMPTBL (L(aligned_16_39bytes), L(table_16_128bytes))
 431         .int    JMPTBL (L(aligned_16_40bytes), L(table_16_128bytes))
 432         .int    JMPTBL (L(aligned_16_41bytes), L(table_16_128bytes))
 433         .int    JMPTBL (L(aligned_16_42bytes), L(table_16_128bytes))
 434         .int    JMPTBL (L(aligned_16_43bytes), L(table_16_128bytes))
 435         .int    JMPTBL (L(aligned_16_44bytes), L(table_16_128bytes))
 436         .int    JMPTBL (L(aligned_16_45bytes), L(table_16_128bytes))
 437         .int    JMPTBL (L(aligned_16_46bytes), L(table_16_128bytes))
 438         .int    JMPTBL (L(aligned_16_47bytes), L(table_16_128bytes))
 439         .int    JMPTBL (L(aligned_16_48bytes), L(table_16_128bytes))
 440         .int    JMPTBL (L(aligned_16_49bytes), L(table_16_128bytes))
 441         .int    JMPTBL (L(aligned_16_50bytes), L(table_16_128bytes))
 442         .int    JMPTBL (L(aligned_16_51bytes), L(table_16_128bytes))
 443         .int    JMPTBL (L(aligned_16_52bytes), L(table_16_128bytes))
 444         .int    JMPTBL (L(aligned_16_53bytes), L(table_16_128bytes))
 445         .int    JMPTBL (L(aligned_16_54bytes), L(table_16_128bytes))
 446         .int    JMPTBL (L(aligned_16_55bytes), L(table_16_128bytes))
 447         .int    JMPTBL (L(aligned_16_56bytes), L(table_16_128bytes))
 448         .int    JMPTBL (L(aligned_16_57bytes), L(table_16_128bytes))
 449         .int    JMPTBL (L(aligned_16_58bytes), L(table_16_128bytes))
 450         .int    JMPTBL (L(aligned_16_59bytes), L(table_16_128bytes))
 451         .int    JMPTBL (L(aligned_16_60bytes), L(table_16_128bytes))
 452         .int    JMPTBL (L(aligned_16_61bytes), L(table_16_128bytes))
 453         .int    JMPTBL (L(aligned_16_62bytes), L(table_16_128bytes))
 454         .int    JMPTBL (L(aligned_16_63bytes), L(table_16_128bytes))
 455         .int    JMPTBL (L(aligned_16_64bytes), L(table_16_128bytes))
 456         .int    JMPTBL (L(aligned_16_65bytes), L(table_16_128bytes))
 457         .int    JMPTBL (L(aligned_16_66bytes), L(table_16_128bytes))
 458         .int    JMPTBL (L(aligned_16_67bytes), L(table_16_128bytes))
 459         .int    JMPTBL (L(aligned_16_68bytes), L(table_16_128bytes))
 460         .int    JMPTBL (L(aligned_16_69bytes), L(table_16_128bytes))
 461         .int    JMPTBL (L(aligned_16_70bytes), L(table_16_128bytes))
 462         .int    JMPTBL (L(aligned_16_71bytes), L(table_16_128bytes))
 463         .int    JMPTBL (L(aligned_16_72bytes), L(table_16_128bytes))
 464         .int    JMPTBL (L(aligned_16_73bytes), L(table_16_128bytes))
 465         .int    JMPTBL (L(aligned_16_74bytes), L(table_16_128bytes))
 466         .int    JMPTBL (L(aligned_16_75bytes), L(table_16_128bytes))
 467         .int    JMPTBL (L(aligned_16_76bytes), L(table_16_128bytes))
 468         .int    JMPTBL (L(aligned_16_77bytes), L(table_16_128bytes))
 469         .int    JMPTBL (L(aligned_16_78bytes), L(table_16_128bytes))
 470         .int    JMPTBL (L(aligned_16_79bytes), L(table_16_128bytes))
 471         .int    JMPTBL (L(aligned_16_80bytes), L(table_16_128bytes))
 472         .int    JMPTBL (L(aligned_16_81bytes), L(table_16_128bytes))
 473         .int    JMPTBL (L(aligned_16_82bytes), L(table_16_128bytes))
 474         .int    JMPTBL (L(aligned_16_83bytes), L(table_16_128bytes))
 475         .int    JMPTBL (L(aligned_16_84bytes), L(table_16_128bytes))
 476         .int    JMPTBL (L(aligned_16_85bytes), L(table_16_128bytes))
 477         .int    JMPTBL (L(aligned_16_86bytes), L(table_16_128bytes))
 478         .int    JMPTBL (L(aligned_16_87bytes), L(table_16_128bytes))
 479         .int    JMPTBL (L(aligned_16_88bytes), L(table_16_128bytes))
 480         .int    JMPTBL (L(aligned_16_89bytes), L(table_16_128bytes))
 481         .int    JMPTBL (L(aligned_16_90bytes), L(table_16_128bytes))
 482         .int    JMPTBL (L(aligned_16_91bytes), L(table_16_128bytes))
 483         .int    JMPTBL (L(aligned_16_92bytes), L(table_16_128bytes))
 484         .int    JMPTBL (L(aligned_16_93bytes), L(table_16_128bytes))
 485         .int    JMPTBL (L(aligned_16_94bytes), L(table_16_128bytes))
 486         .int    JMPTBL (L(aligned_16_95bytes), L(table_16_128bytes))
 487         .int    JMPTBL (L(aligned_16_96bytes), L(table_16_128bytes))
 488         .int    JMPTBL (L(aligned_16_97bytes), L(table_16_128bytes))
 489         .int    JMPTBL (L(aligned_16_98bytes), L(table_16_128bytes))
 490         .int    JMPTBL (L(aligned_16_99bytes), L(table_16_128bytes))
 491         .int    JMPTBL (L(aligned_16_100bytes), L(table_16_128bytes))
 492         .int    JMPTBL (L(aligned_16_101bytes), L(table_16_128bytes))
 493         .int    JMPTBL (L(aligned_16_102bytes), L(table_16_128bytes))
 494         .int    JMPTBL (L(aligned_16_103bytes), L(table_16_128bytes))
 495         .int    JMPTBL (L(aligned_16_104bytes), L(table_16_128bytes))
 496         .int    JMPTBL (L(aligned_16_105bytes), L(table_16_128bytes))
 497         .int    JMPTBL (L(aligned_16_106bytes), L(table_16_128bytes))
 498         .int    JMPTBL (L(aligned_16_107bytes), L(table_16_128bytes))
 499         .int    JMPTBL (L(aligned_16_108bytes), L(table_16_128bytes))
 500         .int    JMPTBL (L(aligned_16_109bytes), L(table_16_128bytes))
 501         .int    JMPTBL (L(aligned_16_110bytes), L(table_16_128bytes))
 502         .int    JMPTBL (L(aligned_16_111bytes), L(table_16_128bytes))
 503         .int    JMPTBL (L(aligned_16_112bytes), L(table_16_128bytes))
 504         .int    JMPTBL (L(aligned_16_113bytes), L(table_16_128bytes))
 505         .int    JMPTBL (L(aligned_16_114bytes), L(table_16_128bytes))
 506         .int    JMPTBL (L(aligned_16_115bytes), L(table_16_128bytes))
 507         .int    JMPTBL (L(aligned_16_116bytes), L(table_16_128bytes))
 508         .int    JMPTBL (L(aligned_16_117bytes), L(table_16_128bytes))
 509         .int    JMPTBL (L(aligned_16_118bytes), L(table_16_128bytes))
 510         .int    JMPTBL (L(aligned_16_119bytes), L(table_16_128bytes))
 511         .int    JMPTBL (L(aligned_16_120bytes), L(table_16_128bytes))
 512         .int    JMPTBL (L(aligned_16_121bytes), L(table_16_128bytes))
 513         .int    JMPTBL (L(aligned_16_122bytes), L(table_16_128bytes))
 514         .int    JMPTBL (L(aligned_16_123bytes), L(table_16_128bytes))
 515         .int    JMPTBL (L(aligned_16_124bytes), L(table_16_128bytes))
 516         .int    JMPTBL (L(aligned_16_125bytes), L(table_16_128bytes))
 517         .int    JMPTBL (L(aligned_16_126bytes), L(table_16_128bytes))
 518         .int    JMPTBL (L(aligned_16_127bytes), L(table_16_128bytes))
 519         .popsection
 520
 521         ALIGN (4)
 522 L(aligned_16_112bytes):
 523         movdqa  %xmm0, -112(%edx)
 524 L(aligned_16_96bytes):
 525         movdqa  %xmm0, -96(%edx)
 526 L(aligned_16_80bytes):
 527         movdqa  %xmm0, -80(%edx)
 528 L(aligned_16_64bytes):
 529         movdqa  %xmm0, -64(%edx)
 530 L(aligned_16_48bytes):
 531         movdqa  %xmm0, -48(%edx)
 532 L(aligned_16_32bytes):
 533         movdqa  %xmm0, -32(%edx)
 534 L(aligned_16_16bytes):
 535         movdqa  %xmm0, -16(%edx)
 536 L(aligned_16_0bytes):
 537         SETRTNVAL
 538         RETURN
 539
 540         ALIGN (4)
 541 L(aligned_16_113bytes):
 542         movdqa  %xmm0, -113(%edx)
 543 L(aligned_16_97bytes):
 544         movdqa  %xmm0, -97(%edx)
 545 L(aligned_16_81bytes):
 546         movdqa  %xmm0, -81(%edx)
 547 L(aligned_16_65bytes):
 548         movdqa  %xmm0, -65(%edx)
 549 L(aligned_16_49bytes):
 550         movdqa  %xmm0, -49(%edx)
 551 L(aligned_16_33bytes):
 552         movdqa  %xmm0, -33(%edx)
 553 L(aligned_16_17bytes):
 554         movdqa  %xmm0, -17(%edx)
 555 L(aligned_16_1bytes):
 556         movb    %al, -1(%edx)
 557         SETRTNVAL
 558         RETURN
 559
 560         ALIGN (4)
 561 L(aligned_16_114bytes):
 562         movdqa  %xmm0, -114(%edx)
 563 L(aligned_16_98bytes):
 564         movdqa  %xmm0, -98(%edx)
 565 L(aligned_16_82bytes):
 566         movdqa  %xmm0, -82(%edx)
 567 L(aligned_16_66bytes):
 568         movdqa  %xmm0, -66(%edx)
 569 L(aligned_16_50bytes):
 570         movdqa  %xmm0, -50(%edx)
 571 L(aligned_16_34bytes):
 572         movdqa  %xmm0, -34(%edx)
 573 L(aligned_16_18bytes):
 574         movdqa  %xmm0, -18(%edx)
 575 L(aligned_16_2bytes):
 576         movw    %ax, -2(%edx)
 577         SETRTNVAL
 578         RETURN
 579
 580         ALIGN (4)
 581 L(aligned_16_115bytes):
 582         movdqa  %xmm0, -115(%edx)
 583 L(aligned_16_99bytes):
 584         movdqa  %xmm0, -99(%edx)
 585 L(aligned_16_83bytes):
 586         movdqa  %xmm0, -83(%edx)
 587 L(aligned_16_67bytes):
 588         movdqa  %xmm0, -67(%edx)
 589 L(aligned_16_51bytes):
 590         movdqa  %xmm0, -51(%edx)
 591 L(aligned_16_35bytes):
 592         movdqa  %xmm0, -35(%edx)
 593 L(aligned_16_19bytes):
 594         movdqa  %xmm0, -19(%edx)
 595 L(aligned_16_3bytes):
 596         movw    %ax, -3(%edx)
 597         movb    %al, -1(%edx)
 598         SETRTNVAL
 599         RETURN
 600
 601         ALIGN (4)
 602 L(aligned_16_116bytes):
 603         movdqa  %xmm0, -116(%edx)
 604 L(aligned_16_100bytes):
 605         movdqa  %xmm0, -100(%edx)
 606 L(aligned_16_84bytes):
 607         movdqa  %xmm0, -84(%edx)
 608 L(aligned_16_68bytes):
 609         movdqa  %xmm0, -68(%edx)
 610 L(aligned_16_52bytes):
 611         movdqa  %xmm0, -52(%edx)
 612 L(aligned_16_36bytes):
 613         movdqa  %xmm0, -36(%edx)
 614 L(aligned_16_20bytes):
 615         movdqa  %xmm0, -20(%edx)
 616 L(aligned_16_4bytes):
 617         movl    %eax, -4(%edx)
 618         SETRTNVAL
 619         RETURN
 620
 621         ALIGN (4)
 622 L(aligned_16_117bytes):
 623         movdqa  %xmm0, -117(%edx)
 624 L(aligned_16_101bytes):
 625         movdqa  %xmm0, -101(%edx)
 626 L(aligned_16_85bytes):
 627         movdqa  %xmm0, -85(%edx)
 628 L(aligned_16_69bytes):
 629         movdqa  %xmm0, -69(%edx)
 630 L(aligned_16_53bytes):
 631         movdqa  %xmm0, -53(%edx)
 632 L(aligned_16_37bytes):
 633         movdqa  %xmm0, -37(%edx)
 634 L(aligned_16_21bytes):
 635         movdqa  %xmm0, -21(%edx)
 636 L(aligned_16_5bytes):
 637         movl    %eax, -5(%edx)
 638         movb    %al, -1(%edx)
 639         SETRTNVAL
 640         RETURN
 641
 642         ALIGN (4)
 643 L(aligned_16_118bytes):
 644         movdqa  %xmm0, -118(%edx)
 645 L(aligned_16_102bytes):
 646         movdqa  %xmm0, -102(%edx)
 647 L(aligned_16_86bytes):
 648         movdqa  %xmm0, -86(%edx)
 649 L(aligned_16_70bytes):
 650         movdqa  %xmm0, -70(%edx)
 651 L(aligned_16_54bytes):
 652         movdqa  %xmm0, -54(%edx)
 653 L(aligned_16_38bytes):
 654         movdqa  %xmm0, -38(%edx)
 655 L(aligned_16_22bytes):
 656         movdqa  %xmm0, -22(%edx)
 657 L(aligned_16_6bytes):
 658         movl    %eax, -6(%edx)
 659         movw    %ax, -2(%edx)
 660         SETRTNVAL
 661         RETURN
 662
 663         ALIGN (4)
 664 L(aligned_16_119bytes):
 665         movdqa  %xmm0, -119(%edx)
 666 L(aligned_16_103bytes):
 667         movdqa  %xmm0, -103(%edx)
 668 L(aligned_16_87bytes):
 669         movdqa  %xmm0, -87(%edx)
 670 L(aligned_16_71bytes):
 671         movdqa  %xmm0, -71(%edx)
 672 L(aligned_16_55bytes):
 673         movdqa  %xmm0, -55(%edx)
 674 L(aligned_16_39bytes):
 675         movdqa  %xmm0, -39(%edx)
 676 L(aligned_16_23bytes):
 677         movdqa  %xmm0, -23(%edx)
 678 L(aligned_16_7bytes):
 679         movl    %eax, -7(%edx)
 680         movw    %ax, -3(%edx)
 681         movb    %al, -1(%edx)
 682         SETRTNVAL
 683         RETURN
 684
 685         ALIGN (4)
 686 L(aligned_16_120bytes):
 687         movdqa  %xmm0, -120(%edx)
 688 L(aligned_16_104bytes):
 689         movdqa  %xmm0, -104(%edx)
 690 L(aligned_16_88bytes):
 691         movdqa  %xmm0, -88(%edx)
 692 L(aligned_16_72bytes):
 693         movdqa  %xmm0, -72(%edx)
 694 L(aligned_16_56bytes):
 695         movdqa  %xmm0, -56(%edx)
 696 L(aligned_16_40bytes):
 697         movdqa  %xmm0, -40(%edx)
 698 L(aligned_16_24bytes):
 699         movdqa  %xmm0, -24(%edx)
 700 L(aligned_16_8bytes):
 701         movq    %xmm0, -8(%edx)
 702         SETRTNVAL
 703         RETURN
 704
 705         ALIGN (4)
 706 L(aligned_16_121bytes):
 707         movdqa  %xmm0, -121(%edx)
 708 L(aligned_16_105bytes):
 709         movdqa  %xmm0, -105(%edx)
 710 L(aligned_16_89bytes):
 711         movdqa  %xmm0, -89(%edx)
 712 L(aligned_16_73bytes):
 713         movdqa  %xmm0, -73(%edx)
 714 L(aligned_16_57bytes):
 715         movdqa  %xmm0, -57(%edx)
 716 L(aligned_16_41bytes):
 717         movdqa  %xmm0, -41(%edx)
 718 L(aligned_16_25bytes):
 719         movdqa  %xmm0, -25(%edx)
 720 L(aligned_16_9bytes):
 721         movq    %xmm0, -9(%edx)
 722         movb    %al, -1(%edx)
 723         SETRTNVAL
 724         RETURN
 725
 726         ALIGN (4)
 727 L(aligned_16_122bytes):
 728         movdqa  %xmm0, -122(%edx)
 729 L(aligned_16_106bytes):
 730         movdqa  %xmm0, -106(%edx)
 731 L(aligned_16_90bytes):
 732         movdqa  %xmm0, -90(%edx)
 733 L(aligned_16_74bytes):
 734         movdqa  %xmm0, -74(%edx)
 735 L(aligned_16_58bytes):
 736         movdqa  %xmm0, -58(%edx)
 737 L(aligned_16_42bytes):
 738         movdqa  %xmm0, -42(%edx)
 739 L(aligned_16_26bytes):
 740         movdqa  %xmm0, -26(%edx)
 741 L(aligned_16_10bytes):
 742         movq    %xmm0, -10(%edx)
 743         movw    %ax, -2(%edx)
 744         SETRTNVAL
 745         RETURN
 746
 747         ALIGN (4)
 748 L(aligned_16_123bytes):
 749         movdqa  %xmm0, -123(%edx)
 750 L(aligned_16_107bytes):
 751         movdqa  %xmm0, -107(%edx)
 752 L(aligned_16_91bytes):
 753         movdqa  %xmm0, -91(%edx)
 754 L(aligned_16_75bytes):
 755         movdqa  %xmm0, -75(%edx)
 756 L(aligned_16_59bytes):
 757         movdqa  %xmm0, -59(%edx)
 758 L(aligned_16_43bytes):
 759         movdqa  %xmm0, -43(%edx)
 760 L(aligned_16_27bytes):
 761         movdqa  %xmm0, -27(%edx)
 762 L(aligned_16_11bytes):
 763         movq    %xmm0, -11(%edx)
 764         movw    %ax, -3(%edx)
 765         movb    %al, -1(%edx)
 766         SETRTNVAL
 767         RETURN
 768
 769         ALIGN (4)
 770 L(aligned_16_124bytes):
 771         movdqa  %xmm0, -124(%edx)
 772 L(aligned_16_108bytes):
 773         movdqa  %xmm0, -108(%edx)
 774 L(aligned_16_92bytes):
 775         movdqa  %xmm0, -92(%edx)
 776 L(aligned_16_76bytes):
 777         movdqa  %xmm0, -76(%edx)
 778 L(aligned_16_60bytes):
 779         movdqa  %xmm0, -60(%edx)
 780 L(aligned_16_44bytes):
 781         movdqa  %xmm0, -44(%edx)
 782 L(aligned_16_28bytes):
 783         movdqa  %xmm0, -28(%edx)
 784 L(aligned_16_12bytes):
 785         movq    %xmm0, -12(%edx)
 786         movl    %eax, -4(%edx)
 787         SETRTNVAL
 788         RETURN
 789
 790         ALIGN (4)
 791 L(aligned_16_125bytes):
 792         movdqa  %xmm0, -125(%edx)
 793 L(aligned_16_109bytes):
 794         movdqa  %xmm0, -109(%edx)
 795 L(aligned_16_93bytes):
 796         movdqa  %xmm0, -93(%edx)
 797 L(aligned_16_77bytes):
 798         movdqa  %xmm0, -77(%edx)
 799 L(aligned_16_61bytes):
 800         movdqa  %xmm0, -61(%edx)
 801 L(aligned_16_45bytes):
 802         movdqa  %xmm0, -45(%edx)
 803 L(aligned_16_29bytes):
 804         movdqa  %xmm0, -29(%edx)
 805 L(aligned_16_13bytes):
 806         movq    %xmm0, -13(%edx)
 807         movl    %eax, -5(%edx)
 808         movb    %al, -1(%edx)
 809         SETRTNVAL
 810         RETURN
 811
 812         ALIGN (4)
 813 L(aligned_16_126bytes):
 814         movdqa  %xmm0, -126(%edx)
 815 L(aligned_16_110bytes):
 816         movdqa  %xmm0, -110(%edx)
 817 L(aligned_16_94bytes):
 818         movdqa  %xmm0, -94(%edx)
 819 L(aligned_16_78bytes):
 820         movdqa  %xmm0, -78(%edx)
 821 L(aligned_16_62bytes):
 822         movdqa  %xmm0, -62(%edx)
 823 L(aligned_16_46bytes):
 824         movdqa  %xmm0, -46(%edx)
 825 L(aligned_16_30bytes):
 826         movdqa  %xmm0, -30(%edx)
 827 L(aligned_16_14bytes):
 828         movq    %xmm0, -14(%edx)
 829         movl    %eax, -6(%edx)
 830         movw    %ax, -2(%edx)
 831         SETRTNVAL
 832         RETURN
 833
 834         ALIGN (4)
 835 L(aligned_16_127bytes):
 836         movdqa  %xmm0, -127(%edx)
 837 L(aligned_16_111bytes):
 838         movdqa  %xmm0, -111(%edx)
 839 L(aligned_16_95bytes):
 840         movdqa  %xmm0, -95(%edx)
 841 L(aligned_16_79bytes):
 842         movdqa  %xmm0, -79(%edx)
 843 L(aligned_16_63bytes):
 844         movdqa  %xmm0, -63(%edx)
 845 L(aligned_16_47bytes):
 846         movdqa  %xmm0, -47(%edx)
 847 L(aligned_16_31bytes):
 848         movdqa  %xmm0, -31(%edx)
 849 L(aligned_16_15bytes):
 850         movq    %xmm0, -15(%edx)
 851         movl    %eax, -7(%edx)
 852         movw    %ax, -3(%edx)
 853         movb    %al, -1(%edx)
 854         SETRTNVAL
 855         RETURN_END
 856
 857 END (__memset_sse2)
 858
 859 #endif