sysdeps/i386/i686/multiarch/memset-sse2-rep.S

   1 /* memset with SSE2 and REP string.
   2    Copyright (C) 2010-2014 Free Software Foundation, Inc.
   3    Contributed by Intel Corporation.
   4    This file is part of the GNU C Library.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, see
  18    <http://www.gnu.org/licenses/>.  */
  19
  20 #ifndef NOT_IN_libc
  21
  22 #include <sysdep.h>
  23 #include "asm-syntax.h"
  24
  25 #define CFI_PUSH(REG)                                           \
  26   cfi_adjust_cfa_offset (4);                                    \
  27   cfi_rel_offset (REG, 0)
  28
  29 #define CFI_POP(REG)                                            \
  30   cfi_adjust_cfa_offset (-4);                                   \
  31   cfi_restore (REG)
  32
  33 #define PUSH(REG)       pushl REG; CFI_PUSH (REG)
  34 #define POP(REG)        popl REG; CFI_POP (REG)
  35
  36 #ifdef USE_AS_BZERO
  37 # define DEST           PARMS
  38 # define LEN            DEST+4
  39 # define SETRTNVAL
  40 #else
  41 # define DEST           PARMS
  42 # define CHR            DEST+4
  43 # define LEN            CHR+4
  44 # define SETRTNVAL      movl DEST(%esp), %eax
  45 #endif
  46
  47 #ifdef SHARED
  48 # define ENTRANCE       PUSH (%ebx);
  49 # define RETURN_END     POP (%ebx); ret
  50 # define RETURN         RETURN_END; CFI_PUSH (%ebx)
  51 # define PARMS          8               /* Preserve EBX.  */
  52 # define JMPTBL(I, B)   I - B
  53
  54 /* Load an entry in a jump table into EBX and branch to it.  TABLE is a
  55    jump table with relative offsets.   */
  56 # define BRANCH_TO_JMPTBL_ENTRY(TABLE)                          \
  57     /* We first load PC into EBX.  */                           \
  58     SETUP_PIC_REG(bx);                                          \
  59     /* Get the address of the jump table.  */                   \
  60     add         $(TABLE - .), %ebx;                             \
  61     /* Get the entry and convert the relative offset to the     \
  62        absolute address.  */                                    \
  63     add         (%ebx,%ecx,4), %ebx;                            \
  64     add         %ecx, %edx;                                     \
  65     /* We loaded the jump table and adjusted EDX. Go.  */       \
  66     jmp         *%ebx
  67 #else
  68 # define ENTRANCE
  69 # define RETURN_END     ret
  70 # define RETURN         RETURN_END
  71 # define PARMS          4
  72 # define JMPTBL(I, B)   I
  73
  74 /* Branch to an entry in a jump table.  TABLE is a jump table with
  75    absolute offsets.  */
  76 # define BRANCH_TO_JMPTBL_ENTRY(TABLE)                          \
  77     add         %ecx, %edx;                                     \
  78     jmp         *TABLE(,%ecx,4)
  79 #endif
  80
  81         .section .text.sse2,"ax",@progbits
  82 #if defined SHARED && !defined NOT_IN_libc && !defined USE_AS_BZERO
  83 ENTRY (__memset_chk_sse2_rep)
  84         movl    12(%esp), %eax
  85         cmpl    %eax, 16(%esp)
  86         jb      HIDDEN_JUMPTARGET (__chk_fail)
  87 END (__memset_chk_sse2_rep)
  88 #endif
  89 ENTRY (__memset_sse2_rep)
  90         ENTRANCE
  91
  92         movl    LEN(%esp), %ecx
  93 #ifdef USE_AS_BZERO
  94         xor     %eax, %eax
  95 #else
  96         movzbl  CHR(%esp), %eax
  97         movb    %al, %ah
  98         /* Fill the whole EAX with pattern.  */
  99         movl    %eax, %edx
 100         shl     $16, %eax
 101         or      %edx, %eax
 102 #endif
 103         movl    DEST(%esp), %edx
 104         cmp     $32, %ecx
 105         jae     L(32bytesormore)
 106
 107 L(write_less32bytes):
 108         BRANCH_TO_JMPTBL_ENTRY (L(table_less_32bytes))
 109
 110
 111         .pushsection .rodata.sse2,"a",@progbits
 112         ALIGN (2)
 113 L(table_less_32bytes):
 114         .int    JMPTBL (L(write_0bytes), L(table_less_32bytes))
 115         .int    JMPTBL (L(write_1bytes), L(table_less_32bytes))
 116         .int    JMPTBL (L(write_2bytes), L(table_less_32bytes))
 117         .int    JMPTBL (L(write_3bytes), L(table_less_32bytes))
 118         .int    JMPTBL (L(write_4bytes), L(table_less_32bytes))
 119         .int    JMPTBL (L(write_5bytes), L(table_less_32bytes))
 120         .int    JMPTBL (L(write_6bytes), L(table_less_32bytes))
 121         .int    JMPTBL (L(write_7bytes), L(table_less_32bytes))
 122         .int    JMPTBL (L(write_8bytes), L(table_less_32bytes))
 123         .int    JMPTBL (L(write_9bytes), L(table_less_32bytes))
 124         .int    JMPTBL (L(write_10bytes), L(table_less_32bytes))
 125         .int    JMPTBL (L(write_11bytes), L(table_less_32bytes))
 126         .int    JMPTBL (L(write_12bytes), L(table_less_32bytes))
 127         .int    JMPTBL (L(write_13bytes), L(table_less_32bytes))
 128         .int    JMPTBL (L(write_14bytes), L(table_less_32bytes))
 129         .int    JMPTBL (L(write_15bytes), L(table_less_32bytes))
 130         .int    JMPTBL (L(write_16bytes), L(table_less_32bytes))
 131         .int    JMPTBL (L(write_17bytes), L(table_less_32bytes))
 132         .int    JMPTBL (L(write_18bytes), L(table_less_32bytes))
 133         .int    JMPTBL (L(write_19bytes), L(table_less_32bytes))
 134         .int    JMPTBL (L(write_20bytes), L(table_less_32bytes))
 135         .int    JMPTBL (L(write_21bytes), L(table_less_32bytes))
 136         .int    JMPTBL (L(write_22bytes), L(table_less_32bytes))
 137         .int    JMPTBL (L(write_23bytes), L(table_less_32bytes))
 138         .int    JMPTBL (L(write_24bytes), L(table_less_32bytes))
 139         .int    JMPTBL (L(write_25bytes), L(table_less_32bytes))
 140         .int    JMPTBL (L(write_26bytes), L(table_less_32bytes))
 141         .int    JMPTBL (L(write_27bytes), L(table_less_32bytes))
 142         .int    JMPTBL (L(write_28bytes), L(table_less_32bytes))
 143         .int    JMPTBL (L(write_29bytes), L(table_less_32bytes))
 144         .int    JMPTBL (L(write_30bytes), L(table_less_32bytes))
 145         .int    JMPTBL (L(write_31bytes), L(table_less_32bytes))
 146         .popsection
 147
 148         ALIGN (4)
 149 L(write_28bytes):
 150         movl    %eax, -28(%edx)
 151 L(write_24bytes):
 152         movl    %eax, -24(%edx)
 153 L(write_20bytes):
 154         movl    %eax, -20(%edx)
 155 L(write_16bytes):
 156         movl    %eax, -16(%edx)
 157 L(write_12bytes):
 158         movl    %eax, -12(%edx)
 159 L(write_8bytes):
 160         movl    %eax, -8(%edx)
 161 L(write_4bytes):
 162         movl    %eax, -4(%edx)
 163 L(write_0bytes):
 164         SETRTNVAL
 165         RETURN
 166
 167         ALIGN (4)
 168 L(write_29bytes):
 169         movl    %eax, -29(%edx)
 170 L(write_25bytes):
 171         movl    %eax, -25(%edx)
 172 L(write_21bytes):
 173         movl    %eax, -21(%edx)
 174 L(write_17bytes):
 175         movl    %eax, -17(%edx)
 176 L(write_13bytes):
 177         movl    %eax, -13(%edx)
 178 L(write_9bytes):
 179         movl    %eax, -9(%edx)
 180 L(write_5bytes):
 181         movl    %eax, -5(%edx)
 182 L(write_1bytes):
 183         movb    %al, -1(%edx)
 184         SETRTNVAL
 185         RETURN
 186
 187         ALIGN (4)
 188 L(write_30bytes):
 189         movl    %eax, -30(%edx)
 190 L(write_26bytes):
 191         movl    %eax, -26(%edx)
 192 L(write_22bytes):
 193         movl    %eax, -22(%edx)
 194 L(write_18bytes):
 195         movl    %eax, -18(%edx)
 196 L(write_14bytes):
 197         movl    %eax, -14(%edx)
 198 L(write_10bytes):
 199         movl    %eax, -10(%edx)
 200 L(write_6bytes):
 201         movl    %eax, -6(%edx)
 202 L(write_2bytes):
 203         movw    %ax, -2(%edx)
 204         SETRTNVAL
 205         RETURN
 206
 207         ALIGN (4)
 208 L(write_31bytes):
 209         movl    %eax, -31(%edx)
 210 L(write_27bytes):
 211         movl    %eax, -27(%edx)
 212 L(write_23bytes):
 213         movl    %eax, -23(%edx)
 214 L(write_19bytes):
 215         movl    %eax, -19(%edx)
 216 L(write_15bytes):
 217         movl    %eax, -15(%edx)
 218 L(write_11bytes):
 219         movl    %eax, -11(%edx)
 220 L(write_7bytes):
 221         movl    %eax, -7(%edx)
 222 L(write_3bytes):
 223         movw    %ax, -3(%edx)
 224         movb    %al, -1(%edx)
 225         SETRTNVAL
 226         RETURN
 227
 228         ALIGN (4)
 229 /* ECX > 32 and EDX is 4 byte aligned.  */
 230 L(32bytesormore):
 231         /* Fill xmm0 with the pattern.  */
 232 #ifdef USE_AS_BZERO
 233         pxor    %xmm0, %xmm0
 234 #else
 235         movd    %eax, %xmm0
 236         pshufd  $0, %xmm0, %xmm0
 237 #endif
 238         testl   $0xf, %edx
 239         jz      L(aligned_16)
 240 /* ECX > 32 and EDX is not 16 byte aligned.  */
 241 L(not_aligned_16):
 242         movdqu  %xmm0, (%edx)
 243         movl    %edx, %eax
 244         and     $-16, %edx
 245         add     $16, %edx
 246         sub     %edx, %eax
 247         add     %eax, %ecx
 248         movd    %xmm0, %eax
 249
 250         ALIGN (4)
 251 L(aligned_16):
 252         cmp     $128, %ecx
 253         jae     L(128bytesormore)
 254
 255 L(aligned_16_less128bytes):
 256         BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes))
 257
 258         ALIGN (4)
 259 L(128bytesormore):
 260         PUSH (%edi)
 261 #ifdef DATA_CACHE_SIZE
 262         PUSH (%ebx)
 263         mov     $DATA_CACHE_SIZE, %ebx
 264 #else
 265 # ifdef SHARED
 266         SETUP_PIC_REG(bx)
 267         add     $_GLOBAL_OFFSET_TABLE_, %ebx
 268         mov     __x86_data_cache_size@GOTOFF(%ebx), %ebx
 269 # else
 270         PUSH (%ebx)
 271         mov     __x86_data_cache_size, %ebx
 272 # endif
 273 #endif
 274         mov     %ebx, %edi
 275         shr     $4, %ebx
 276         sub     %ebx, %edi
 277 #if defined DATA_CACHE_SIZE || !defined SHARED
 278         POP (%ebx)
 279 #endif
 280 /*
 281  * When data size approximate the end of L1 cache,
 282  * fast string will prefetch and combine data efficiently.
 283  */
 284         cmp     %edi, %ecx
 285         jae     L(128bytesormore_endof_L1)
 286         subl    $128, %ecx
 287 L(128bytesormore_normal):
 288         sub     $128, %ecx
 289         movdqa  %xmm0, (%edx)
 290         movdqa  %xmm0, 0x10(%edx)
 291         movdqa  %xmm0, 0x20(%edx)
 292         movdqa  %xmm0, 0x30(%edx)
 293         movdqa  %xmm0, 0x40(%edx)
 294         movdqa  %xmm0, 0x50(%edx)
 295         movdqa  %xmm0, 0x60(%edx)
 296         movdqa  %xmm0, 0x70(%edx)
 297         lea     128(%edx), %edx
 298         jb      L(128bytesless_normal)
 299
 300
 301         sub     $128, %ecx
 302         movdqa  %xmm0, (%edx)
 303         movdqa  %xmm0, 0x10(%edx)
 304         movdqa  %xmm0, 0x20(%edx)
 305         movdqa  %xmm0, 0x30(%edx)
 306         movdqa  %xmm0, 0x40(%edx)
 307         movdqa  %xmm0, 0x50(%edx)
 308         movdqa  %xmm0, 0x60(%edx)
 309         movdqa  %xmm0, 0x70(%edx)
 310         lea     128(%edx), %edx
 311         jae     L(128bytesormore_normal)
 312
 313 L(128bytesless_normal):
 314         POP (%edi)
 315         add     $128, %ecx
 316         BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes))
 317
 318         CFI_PUSH (%edi)
 319         ALIGN (4)
 320 L(128bytesormore_endof_L1):
 321         mov     %edx, %edi
 322         mov     %ecx, %edx
 323         shr     $2, %ecx
 324         and     $3, %edx
 325         rep stosl
 326         jz      L(copy_page_by_rep_exit)
 327         cmp     $2, %edx
 328         jb      L(copy_page_by_rep_left_1)
 329         movw    %ax, (%edi)
 330         add     $2, %edi
 331         sub     $2, %edx
 332         jz      L(copy_page_by_rep_exit)
 333 L(copy_page_by_rep_left_1):
 334         movb    %al, (%edi)
 335 L(copy_page_by_rep_exit):
 336         POP (%edi)
 337         SETRTNVAL
 338         RETURN
 339
 340         .pushsection .rodata.sse2,"a",@progbits
 341         ALIGN (2)
 342 L(table_16_128bytes):
 343         .int    JMPTBL (L(aligned_16_0bytes), L(table_16_128bytes))
 344         .int    JMPTBL (L(aligned_16_1bytes), L(table_16_128bytes))
 345         .int    JMPTBL (L(aligned_16_2bytes), L(table_16_128bytes))
 346         .int    JMPTBL (L(aligned_16_3bytes), L(table_16_128bytes))
 347         .int    JMPTBL (L(aligned_16_4bytes), L(table_16_128bytes))
 348         .int    JMPTBL (L(aligned_16_5bytes), L(table_16_128bytes))
 349         .int    JMPTBL (L(aligned_16_6bytes), L(table_16_128bytes))
 350         .int    JMPTBL (L(aligned_16_7bytes), L(table_16_128bytes))
 351         .int    JMPTBL (L(aligned_16_8bytes), L(table_16_128bytes))
 352         .int    JMPTBL (L(aligned_16_9bytes), L(table_16_128bytes))
 353         .int    JMPTBL (L(aligned_16_10bytes), L(table_16_128bytes))
 354         .int    JMPTBL (L(aligned_16_11bytes), L(table_16_128bytes))
 355         .int    JMPTBL (L(aligned_16_12bytes), L(table_16_128bytes))
 356         .int    JMPTBL (L(aligned_16_13bytes), L(table_16_128bytes))
 357         .int    JMPTBL (L(aligned_16_14bytes), L(table_16_128bytes))
 358         .int    JMPTBL (L(aligned_16_15bytes), L(table_16_128bytes))
 359         .int    JMPTBL (L(aligned_16_16bytes), L(table_16_128bytes))
 360         .int    JMPTBL (L(aligned_16_17bytes), L(table_16_128bytes))
 361         .int    JMPTBL (L(aligned_16_18bytes), L(table_16_128bytes))
 362         .int    JMPTBL (L(aligned_16_19bytes), L(table_16_128bytes))
 363         .int    JMPTBL (L(aligned_16_20bytes), L(table_16_128bytes))
 364         .int    JMPTBL (L(aligned_16_21bytes), L(table_16_128bytes))
 365         .int    JMPTBL (L(aligned_16_22bytes), L(table_16_128bytes))
 366         .int    JMPTBL (L(aligned_16_23bytes), L(table_16_128bytes))
 367         .int    JMPTBL (L(aligned_16_24bytes), L(table_16_128bytes))
 368         .int    JMPTBL (L(aligned_16_25bytes), L(table_16_128bytes))
 369         .int    JMPTBL (L(aligned_16_26bytes), L(table_16_128bytes))
 370         .int    JMPTBL (L(aligned_16_27bytes), L(table_16_128bytes))
 371         .int    JMPTBL (L(aligned_16_28bytes), L(table_16_128bytes))
 372         .int    JMPTBL (L(aligned_16_29bytes), L(table_16_128bytes))
 373         .int    JMPTBL (L(aligned_16_30bytes), L(table_16_128bytes))
 374         .int    JMPTBL (L(aligned_16_31bytes), L(table_16_128bytes))
 375         .int    JMPTBL (L(aligned_16_32bytes), L(table_16_128bytes))
 376         .int    JMPTBL (L(aligned_16_33bytes), L(table_16_128bytes))
 377         .int    JMPTBL (L(aligned_16_34bytes), L(table_16_128bytes))
 378         .int    JMPTBL (L(aligned_16_35bytes), L(table_16_128bytes))
 379         .int    JMPTBL (L(aligned_16_36bytes), L(table_16_128bytes))
 380         .int    JMPTBL (L(aligned_16_37bytes), L(table_16_128bytes))
 381         .int    JMPTBL (L(aligned_16_38bytes), L(table_16_128bytes))
 382         .int    JMPTBL (L(aligned_16_39bytes), L(table_16_128bytes))
 383         .int    JMPTBL (L(aligned_16_40bytes), L(table_16_128bytes))
 384         .int    JMPTBL (L(aligned_16_41bytes), L(table_16_128bytes))
 385         .int    JMPTBL (L(aligned_16_42bytes), L(table_16_128bytes))
 386         .int    JMPTBL (L(aligned_16_43bytes), L(table_16_128bytes))
 387         .int    JMPTBL (L(aligned_16_44bytes), L(table_16_128bytes))
 388         .int    JMPTBL (L(aligned_16_45bytes), L(table_16_128bytes))
 389         .int    JMPTBL (L(aligned_16_46bytes), L(table_16_128bytes))
 390         .int    JMPTBL (L(aligned_16_47bytes), L(table_16_128bytes))
 391         .int    JMPTBL (L(aligned_16_48bytes), L(table_16_128bytes))
 392         .int    JMPTBL (L(aligned_16_49bytes), L(table_16_128bytes))
 393         .int    JMPTBL (L(aligned_16_50bytes), L(table_16_128bytes))
 394         .int    JMPTBL (L(aligned_16_51bytes), L(table_16_128bytes))
 395         .int    JMPTBL (L(aligned_16_52bytes), L(table_16_128bytes))
 396         .int    JMPTBL (L(aligned_16_53bytes), L(table_16_128bytes))
 397         .int    JMPTBL (L(aligned_16_54bytes), L(table_16_128bytes))
 398         .int    JMPTBL (L(aligned_16_55bytes), L(table_16_128bytes))
 399         .int    JMPTBL (L(aligned_16_56bytes), L(table_16_128bytes))
 400         .int    JMPTBL (L(aligned_16_57bytes), L(table_16_128bytes))
 401         .int    JMPTBL (L(aligned_16_58bytes), L(table_16_128bytes))
 402         .int    JMPTBL (L(aligned_16_59bytes), L(table_16_128bytes))
 403         .int    JMPTBL (L(aligned_16_60bytes), L(table_16_128bytes))
 404         .int    JMPTBL (L(aligned_16_61bytes), L(table_16_128bytes))
 405         .int    JMPTBL (L(aligned_16_62bytes), L(table_16_128bytes))
 406         .int    JMPTBL (L(aligned_16_63bytes), L(table_16_128bytes))
 407         .int    JMPTBL (L(aligned_16_64bytes), L(table_16_128bytes))
 408         .int    JMPTBL (L(aligned_16_65bytes), L(table_16_128bytes))
 409         .int    JMPTBL (L(aligned_16_66bytes), L(table_16_128bytes))
 410         .int    JMPTBL (L(aligned_16_67bytes), L(table_16_128bytes))
 411         .int    JMPTBL (L(aligned_16_68bytes), L(table_16_128bytes))
 412         .int    JMPTBL (L(aligned_16_69bytes), L(table_16_128bytes))
 413         .int    JMPTBL (L(aligned_16_70bytes), L(table_16_128bytes))
 414         .int    JMPTBL (L(aligned_16_71bytes), L(table_16_128bytes))
 415         .int    JMPTBL (L(aligned_16_72bytes), L(table_16_128bytes))
 416         .int    JMPTBL (L(aligned_16_73bytes), L(table_16_128bytes))
 417         .int    JMPTBL (L(aligned_16_74bytes), L(table_16_128bytes))
 418         .int    JMPTBL (L(aligned_16_75bytes), L(table_16_128bytes))
 419         .int    JMPTBL (L(aligned_16_76bytes), L(table_16_128bytes))
 420         .int    JMPTBL (L(aligned_16_77bytes), L(table_16_128bytes))
 421         .int    JMPTBL (L(aligned_16_78bytes), L(table_16_128bytes))
 422         .int    JMPTBL (L(aligned_16_79bytes), L(table_16_128bytes))
 423         .int    JMPTBL (L(aligned_16_80bytes), L(table_16_128bytes))
 424         .int    JMPTBL (L(aligned_16_81bytes), L(table_16_128bytes))
 425         .int    JMPTBL (L(aligned_16_82bytes), L(table_16_128bytes))
 426         .int    JMPTBL (L(aligned_16_83bytes), L(table_16_128bytes))
 427         .int    JMPTBL (L(aligned_16_84bytes), L(table_16_128bytes))
 428         .int    JMPTBL (L(aligned_16_85bytes), L(table_16_128bytes))
 429         .int    JMPTBL (L(aligned_16_86bytes), L(table_16_128bytes))
 430         .int    JMPTBL (L(aligned_16_87bytes), L(table_16_128bytes))
 431         .int    JMPTBL (L(aligned_16_88bytes), L(table_16_128bytes))
 432         .int    JMPTBL (L(aligned_16_89bytes), L(table_16_128bytes))
 433         .int    JMPTBL (L(aligned_16_90bytes), L(table_16_128bytes))
 434         .int    JMPTBL (L(aligned_16_91bytes), L(table_16_128bytes))
 435         .int    JMPTBL (L(aligned_16_92bytes), L(table_16_128bytes))
 436         .int    JMPTBL (L(aligned_16_93bytes), L(table_16_128bytes))
 437         .int    JMPTBL (L(aligned_16_94bytes), L(table_16_128bytes))
 438         .int    JMPTBL (L(aligned_16_95bytes), L(table_16_128bytes))
 439         .int    JMPTBL (L(aligned_16_96bytes), L(table_16_128bytes))
 440         .int    JMPTBL (L(aligned_16_97bytes), L(table_16_128bytes))
 441         .int    JMPTBL (L(aligned_16_98bytes), L(table_16_128bytes))
 442         .int    JMPTBL (L(aligned_16_99bytes), L(table_16_128bytes))
 443         .int    JMPTBL (L(aligned_16_100bytes), L(table_16_128bytes))
 444         .int    JMPTBL (L(aligned_16_101bytes), L(table_16_128bytes))
 445         .int    JMPTBL (L(aligned_16_102bytes), L(table_16_128bytes))
 446         .int    JMPTBL (L(aligned_16_103bytes), L(table_16_128bytes))
 447         .int    JMPTBL (L(aligned_16_104bytes), L(table_16_128bytes))
 448         .int    JMPTBL (L(aligned_16_105bytes), L(table_16_128bytes))
 449         .int    JMPTBL (L(aligned_16_106bytes), L(table_16_128bytes))
 450         .int    JMPTBL (L(aligned_16_107bytes), L(table_16_128bytes))
 451         .int    JMPTBL (L(aligned_16_108bytes), L(table_16_128bytes))
 452         .int    JMPTBL (L(aligned_16_109bytes), L(table_16_128bytes))
 453         .int    JMPTBL (L(aligned_16_110bytes), L(table_16_128bytes))
 454         .int    JMPTBL (L(aligned_16_111bytes), L(table_16_128bytes))
 455         .int    JMPTBL (L(aligned_16_112bytes), L(table_16_128bytes))
 456         .int    JMPTBL (L(aligned_16_113bytes), L(table_16_128bytes))
 457         .int    JMPTBL (L(aligned_16_114bytes), L(table_16_128bytes))
 458         .int    JMPTBL (L(aligned_16_115bytes), L(table_16_128bytes))
 459         .int    JMPTBL (L(aligned_16_116bytes), L(table_16_128bytes))
 460         .int    JMPTBL (L(aligned_16_117bytes), L(table_16_128bytes))
 461         .int    JMPTBL (L(aligned_16_118bytes), L(table_16_128bytes))
 462         .int    JMPTBL (L(aligned_16_119bytes), L(table_16_128bytes))
 463         .int    JMPTBL (L(aligned_16_120bytes), L(table_16_128bytes))
 464         .int    JMPTBL (L(aligned_16_121bytes), L(table_16_128bytes))
 465         .int    JMPTBL (L(aligned_16_122bytes), L(table_16_128bytes))
 466         .int    JMPTBL (L(aligned_16_123bytes), L(table_16_128bytes))
 467         .int    JMPTBL (L(aligned_16_124bytes), L(table_16_128bytes))
 468         .int    JMPTBL (L(aligned_16_125bytes), L(table_16_128bytes))
 469         .int    JMPTBL (L(aligned_16_126bytes), L(table_16_128bytes))
 470         .int    JMPTBL (L(aligned_16_127bytes), L(table_16_128bytes))
 471         .popsection
 472
 473         ALIGN (4)
 474 L(aligned_16_112bytes):
 475         movdqa  %xmm0, -112(%edx)
 476 L(aligned_16_96bytes):
 477         movdqa  %xmm0, -96(%edx)
 478 L(aligned_16_80bytes):
 479         movdqa  %xmm0, -80(%edx)
 480 L(aligned_16_64bytes):
 481         movdqa  %xmm0, -64(%edx)
 482 L(aligned_16_48bytes):
 483         movdqa  %xmm0, -48(%edx)
 484 L(aligned_16_32bytes):
 485         movdqa  %xmm0, -32(%edx)
 486 L(aligned_16_16bytes):
 487         movdqa  %xmm0, -16(%edx)
 488 L(aligned_16_0bytes):
 489         SETRTNVAL
 490         RETURN
 491
 492         ALIGN (4)
 493 L(aligned_16_113bytes):
 494         movdqa  %xmm0, -113(%edx)
 495 L(aligned_16_97bytes):
 496         movdqa  %xmm0, -97(%edx)
 497 L(aligned_16_81bytes):
 498         movdqa  %xmm0, -81(%edx)
 499 L(aligned_16_65bytes):
 500         movdqa  %xmm0, -65(%edx)
 501 L(aligned_16_49bytes):
 502         movdqa  %xmm0, -49(%edx)
 503 L(aligned_16_33bytes):
 504         movdqa  %xmm0, -33(%edx)
 505 L(aligned_16_17bytes):
 506         movdqa  %xmm0, -17(%edx)
 507 L(aligned_16_1bytes):
 508         movb    %al, -1(%edx)
 509         SETRTNVAL
 510         RETURN
 511
 512         ALIGN (4)
 513 L(aligned_16_114bytes):
 514         movdqa  %xmm0, -114(%edx)
 515 L(aligned_16_98bytes):
 516         movdqa  %xmm0, -98(%edx)
 517 L(aligned_16_82bytes):
 518         movdqa  %xmm0, -82(%edx)
 519 L(aligned_16_66bytes):
 520         movdqa  %xmm0, -66(%edx)
 521 L(aligned_16_50bytes):
 522         movdqa  %xmm0, -50(%edx)
 523 L(aligned_16_34bytes):
 524         movdqa  %xmm0, -34(%edx)
 525 L(aligned_16_18bytes):
 526         movdqa  %xmm0, -18(%edx)
 527 L(aligned_16_2bytes):
 528         movw    %ax, -2(%edx)
 529         SETRTNVAL
 530         RETURN
 531
 532         ALIGN (4)
 533 L(aligned_16_115bytes):
 534         movdqa  %xmm0, -115(%edx)
 535 L(aligned_16_99bytes):
 536         movdqa  %xmm0, -99(%edx)
 537 L(aligned_16_83bytes):
 538         movdqa  %xmm0, -83(%edx)
 539 L(aligned_16_67bytes):
 540         movdqa  %xmm0, -67(%edx)
 541 L(aligned_16_51bytes):
 542         movdqa  %xmm0, -51(%edx)
 543 L(aligned_16_35bytes):
 544         movdqa  %xmm0, -35(%edx)
 545 L(aligned_16_19bytes):
 546         movdqa  %xmm0, -19(%edx)
 547 L(aligned_16_3bytes):
 548         movw    %ax, -3(%edx)
 549         movb    %al, -1(%edx)
 550         SETRTNVAL
 551         RETURN
 552
 553         ALIGN (4)
 554 L(aligned_16_116bytes):
 555         movdqa  %xmm0, -116(%edx)
 556 L(aligned_16_100bytes):
 557         movdqa  %xmm0, -100(%edx)
 558 L(aligned_16_84bytes):
 559         movdqa  %xmm0, -84(%edx)
 560 L(aligned_16_68bytes):
 561         movdqa  %xmm0, -68(%edx)
 562 L(aligned_16_52bytes):
 563         movdqa  %xmm0, -52(%edx)
 564 L(aligned_16_36bytes):
 565         movdqa  %xmm0, -36(%edx)
 566 L(aligned_16_20bytes):
 567         movdqa  %xmm0, -20(%edx)
 568 L(aligned_16_4bytes):
 569         movl    %eax, -4(%edx)
 570         SETRTNVAL
 571         RETURN
 572
 573         ALIGN (4)
 574 L(aligned_16_117bytes):
 575         movdqa  %xmm0, -117(%edx)
 576 L(aligned_16_101bytes):
 577         movdqa  %xmm0, -101(%edx)
 578 L(aligned_16_85bytes):
 579         movdqa  %xmm0, -85(%edx)
 580 L(aligned_16_69bytes):
 581         movdqa  %xmm0, -69(%edx)
 582 L(aligned_16_53bytes):
 583         movdqa  %xmm0, -53(%edx)
 584 L(aligned_16_37bytes):
 585         movdqa  %xmm0, -37(%edx)
 586 L(aligned_16_21bytes):
 587         movdqa  %xmm0, -21(%edx)
 588 L(aligned_16_5bytes):
 589         movl    %eax, -5(%edx)
 590         movb    %al, -1(%edx)
 591         SETRTNVAL
 592         RETURN
 593
 594         ALIGN (4)
 595 L(aligned_16_118bytes):
 596         movdqa  %xmm0, -118(%edx)
 597 L(aligned_16_102bytes):
 598         movdqa  %xmm0, -102(%edx)
 599 L(aligned_16_86bytes):
 600         movdqa  %xmm0, -86(%edx)
 601 L(aligned_16_70bytes):
 602         movdqa  %xmm0, -70(%edx)
 603 L(aligned_16_54bytes):
 604         movdqa  %xmm0, -54(%edx)
 605 L(aligned_16_38bytes):
 606         movdqa  %xmm0, -38(%edx)
 607 L(aligned_16_22bytes):
 608         movdqa  %xmm0, -22(%edx)
 609 L(aligned_16_6bytes):
 610         movl    %eax, -6(%edx)
 611         movw    %ax, -2(%edx)
 612         SETRTNVAL
 613         RETURN
 614
 615         ALIGN (4)
 616 L(aligned_16_119bytes):
 617         movdqa  %xmm0, -119(%edx)
 618 L(aligned_16_103bytes):
 619         movdqa  %xmm0, -103(%edx)
 620 L(aligned_16_87bytes):
 621         movdqa  %xmm0, -87(%edx)
 622 L(aligned_16_71bytes):
 623         movdqa  %xmm0, -71(%edx)
 624 L(aligned_16_55bytes):
 625         movdqa  %xmm0, -55(%edx)
 626 L(aligned_16_39bytes):
 627         movdqa  %xmm0, -39(%edx)
 628 L(aligned_16_23bytes):
 629         movdqa  %xmm0, -23(%edx)
 630 L(aligned_16_7bytes):
 631         movl    %eax, -7(%edx)
 632         movw    %ax, -3(%edx)
 633         movb    %al, -1(%edx)
 634         SETRTNVAL
 635         RETURN
 636
 637         ALIGN (4)
 638 L(aligned_16_120bytes):
 639         movdqa  %xmm0, -120(%edx)
 640 L(aligned_16_104bytes):
 641         movdqa  %xmm0, -104(%edx)
 642 L(aligned_16_88bytes):
 643         movdqa  %xmm0, -88(%edx)
 644 L(aligned_16_72bytes):
 645         movdqa  %xmm0, -72(%edx)
 646 L(aligned_16_56bytes):
 647         movdqa  %xmm0, -56(%edx)
 648 L(aligned_16_40bytes):
 649         movdqa  %xmm0, -40(%edx)
 650 L(aligned_16_24bytes):
 651         movdqa  %xmm0, -24(%edx)
 652 L(aligned_16_8bytes):
 653         movq    %xmm0, -8(%edx)
 654         SETRTNVAL
 655         RETURN
 656
 657         ALIGN (4)
 658 L(aligned_16_121bytes):
 659         movdqa  %xmm0, -121(%edx)
 660 L(aligned_16_105bytes):
 661         movdqa  %xmm0, -105(%edx)
 662 L(aligned_16_89bytes):
 663         movdqa  %xmm0, -89(%edx)
 664 L(aligned_16_73bytes):
 665         movdqa  %xmm0, -73(%edx)
 666 L(aligned_16_57bytes):
 667         movdqa  %xmm0, -57(%edx)
 668 L(aligned_16_41bytes):
 669         movdqa  %xmm0, -41(%edx)
 670 L(aligned_16_25bytes):
 671         movdqa  %xmm0, -25(%edx)
 672 L(aligned_16_9bytes):
 673         movq    %xmm0, -9(%edx)
 674         movb    %al, -1(%edx)
 675         SETRTNVAL
 676         RETURN
 677
 678         ALIGN (4)
 679 L(aligned_16_122bytes):
 680         movdqa  %xmm0, -122(%edx)
 681 L(aligned_16_106bytes):
 682         movdqa  %xmm0, -106(%edx)
 683 L(aligned_16_90bytes):
 684         movdqa  %xmm0, -90(%edx)
 685 L(aligned_16_74bytes):
 686         movdqa  %xmm0, -74(%edx)
 687 L(aligned_16_58bytes):
 688         movdqa  %xmm0, -58(%edx)
 689 L(aligned_16_42bytes):
 690         movdqa  %xmm0, -42(%edx)
 691 L(aligned_16_26bytes):
 692         movdqa  %xmm0, -26(%edx)
 693 L(aligned_16_10bytes):
 694         movq    %xmm0, -10(%edx)
 695         movw    %ax, -2(%edx)
 696         SETRTNVAL
 697         RETURN
 698
 699         ALIGN (4)
 700 L(aligned_16_123bytes):
 701         movdqa  %xmm0, -123(%edx)
 702 L(aligned_16_107bytes):
 703         movdqa  %xmm0, -107(%edx)
 704 L(aligned_16_91bytes):
 705         movdqa  %xmm0, -91(%edx)
 706 L(aligned_16_75bytes):
 707         movdqa  %xmm0, -75(%edx)
 708 L(aligned_16_59bytes):
 709         movdqa  %xmm0, -59(%edx)
 710 L(aligned_16_43bytes):
 711         movdqa  %xmm0, -43(%edx)
 712 L(aligned_16_27bytes):
 713         movdqa  %xmm0, -27(%edx)
 714 L(aligned_16_11bytes):
 715         movq    %xmm0, -11(%edx)
 716         movw    %ax, -3(%edx)
 717         movb    %al, -1(%edx)
 718         SETRTNVAL
 719         RETURN
 720
 721         ALIGN (4)
 722 L(aligned_16_124bytes):
 723         movdqa  %xmm0, -124(%edx)
 724 L(aligned_16_108bytes):
 725         movdqa  %xmm0, -108(%edx)
 726 L(aligned_16_92bytes):
 727         movdqa  %xmm0, -92(%edx)
 728 L(aligned_16_76bytes):
 729         movdqa  %xmm0, -76(%edx)
 730 L(aligned_16_60bytes):
 731         movdqa  %xmm0, -60(%edx)
 732 L(aligned_16_44bytes):
 733         movdqa  %xmm0, -44(%edx)
 734 L(aligned_16_28bytes):
 735         movdqa  %xmm0, -28(%edx)
 736 L(aligned_16_12bytes):
 737         movq    %xmm0, -12(%edx)
 738         movl    %eax, -4(%edx)
 739         SETRTNVAL
 740         RETURN
 741
 742         ALIGN (4)
 743 L(aligned_16_125bytes):
 744         movdqa  %xmm0, -125(%edx)
 745 L(aligned_16_109bytes):
 746         movdqa  %xmm0, -109(%edx)
 747 L(aligned_16_93bytes):
 748         movdqa  %xmm0, -93(%edx)
 749 L(aligned_16_77bytes):
 750         movdqa  %xmm0, -77(%edx)
 751 L(aligned_16_61bytes):
 752         movdqa  %xmm0, -61(%edx)
 753 L(aligned_16_45bytes):
 754         movdqa  %xmm0, -45(%edx)
 755 L(aligned_16_29bytes):
 756         movdqa  %xmm0, -29(%edx)
 757 L(aligned_16_13bytes):
 758         movq    %xmm0, -13(%edx)
 759         movl    %eax, -5(%edx)
 760         movb    %al, -1(%edx)
 761         SETRTNVAL
 762         RETURN
 763
 764         ALIGN (4)
 765 L(aligned_16_126bytes):
 766         movdqa  %xmm0, -126(%edx)
 767 L(aligned_16_110bytes):
 768         movdqa  %xmm0, -110(%edx)
 769 L(aligned_16_94bytes):
 770         movdqa  %xmm0, -94(%edx)
 771 L(aligned_16_78bytes):
 772         movdqa  %xmm0, -78(%edx)
 773 L(aligned_16_62bytes):
 774         movdqa  %xmm0, -62(%edx)
 775 L(aligned_16_46bytes):
 776         movdqa  %xmm0, -46(%edx)
 777 L(aligned_16_30bytes):
 778         movdqa  %xmm0, -30(%edx)
 779 L(aligned_16_14bytes):
 780         movq    %xmm0, -14(%edx)
 781         movl    %eax, -6(%edx)
 782         movw    %ax, -2(%edx)
 783         SETRTNVAL
 784         RETURN
 785
 786         ALIGN (4)
 787 L(aligned_16_127bytes):
 788         movdqa  %xmm0, -127(%edx)
 789 L(aligned_16_111bytes):
 790         movdqa  %xmm0, -111(%edx)
 791 L(aligned_16_95bytes):
 792         movdqa  %xmm0, -95(%edx)
 793 L(aligned_16_79bytes):
 794         movdqa  %xmm0, -79(%edx)
 795 L(aligned_16_63bytes):
 796         movdqa  %xmm0, -63(%edx)
 797 L(aligned_16_47bytes):
 798         movdqa  %xmm0, -47(%edx)
 799 L(aligned_16_31bytes):
 800         movdqa  %xmm0, -31(%edx)
 801 L(aligned_16_15bytes):
 802         movq    %xmm0, -15(%edx)
 803         movl    %eax, -7(%edx)
 804         movw    %ax, -3(%edx)
 805         movb    %al, -1(%edx)
 806         SETRTNVAL
 807         RETURN_END
 808
 809 END (__memset_sse2_rep)
 810
 811 #endif