sysdeps/x86_64/multiarch/strcat-ssse3.S

   1 /* strcat with SSSE3
   2    Copyright (C) 2011-2013 Free Software Foundation, Inc.
   3    Contributed by Intel Corporation.
   4    This file is part of the GNU C Library.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, see
  18    <http://www.gnu.org/licenses/>.  */
  19
  20 #ifndef NOT_IN_libc
  21
  22 # include <sysdep.h>
  23
  24 # ifndef STRCAT
  25 #  define STRCAT  __strcat_ssse3
  26 # endif
  27
  28 # define USE_AS_STRCAT
  29
  30 .text
  31 ENTRY (STRCAT)
  32 # ifdef USE_AS_STRNCAT
  33         mov     %rdx, %r8
  34 # endif
  35
  36 # define RETURN  jmp L(StartStrcpyPart)
  37 # include "strlen-sse2-no-bsf.S"
  38
  39 # undef RETURN
  40
  41 L(StartStrcpyPart):
  42         mov     %rsi, %rcx
  43         lea     (%rdi, %rax), %rdx
  44 # ifdef USE_AS_STRNCAT
  45         test    %r8, %r8
  46         jz      L(StrncatExit0)
  47         cmp     $8, %r8
  48         jbe     L(StrncatExit8Bytes)
  49 # endif
  50         cmpb    $0, (%rcx)
  51         jz      L(Exit1)
  52         cmpb    $0, 1(%rcx)
  53         jz      L(Exit2)
  54         cmpb    $0, 2(%rcx)
  55         jz      L(Exit3)
  56         cmpb    $0, 3(%rcx)
  57         jz      L(Exit4)
  58         cmpb    $0, 4(%rcx)
  59         jz      L(Exit5)
  60         cmpb    $0, 5(%rcx)
  61         jz      L(Exit6)
  62         cmpb    $0, 6(%rcx)
  63         jz      L(Exit7)
  64         cmpb    $0, 7(%rcx)
  65         jz      L(Exit8)
  66         cmpb    $0, 8(%rcx)
  67         jz      L(Exit9)
  68 # ifdef USE_AS_STRNCAT
  69         cmp     $16, %r8
  70         jb      L(StrncatExit15Bytes)
  71 # endif
  72         cmpb    $0, 9(%rcx)
  73         jz      L(Exit10)
  74         cmpb    $0, 10(%rcx)
  75         jz      L(Exit11)
  76         cmpb    $0, 11(%rcx)
  77         jz      L(Exit12)
  78         cmpb    $0, 12(%rcx)
  79         jz      L(Exit13)
  80         cmpb    $0, 13(%rcx)
  81         jz      L(Exit14)
  82         cmpb    $0, 14(%rcx)
  83         jz      L(Exit15)
  84         cmpb    $0, 15(%rcx)
  85         jz      L(Exit16)
  86 # ifdef USE_AS_STRNCAT
  87         cmp     $16, %r8
  88         je      L(StrncatExit16)
  89 #  define USE_AS_STRNCPY
  90 # endif
  91
  92 # include "strcpy-ssse3.S"
  93
  94         .p2align 4
  95 L(CopyFrom1To16Bytes):
  96         add     %rsi, %rdx
  97         add     %rsi, %rcx
  98
  99         test    %al, %al
 100         jz      L(ExitHigh)
 101         test    $0x01, %al
 102         jnz     L(Exit1)
 103         test    $0x02, %al
 104         jnz     L(Exit2)
 105         test    $0x04, %al
 106         jnz     L(Exit3)
 107         test    $0x08, %al
 108         jnz     L(Exit4)
 109         test    $0x10, %al
 110         jnz     L(Exit5)
 111         test    $0x20, %al
 112         jnz     L(Exit6)
 113         test    $0x40, %al
 114         jnz     L(Exit7)
 115         movlpd  (%rcx), %xmm0
 116         movlpd  %xmm0, (%rdx)
 117         mov     %rdi, %rax
 118         ret
 119
 120         .p2align 4
 121 L(ExitHigh):
 122         test    $0x01, %ah
 123         jnz     L(Exit9)
 124         test    $0x02, %ah
 125         jnz     L(Exit10)
 126         test    $0x04, %ah
 127         jnz     L(Exit11)
 128         test    $0x08, %ah
 129         jnz     L(Exit12)
 130         test    $0x10, %ah
 131         jnz     L(Exit13)
 132         test    $0x20, %ah
 133         jnz     L(Exit14)
 134         test    $0x40, %ah
 135         jnz     L(Exit15)
 136         movlpd  (%rcx), %xmm0
 137         movlpd  8(%rcx), %xmm1
 138         movlpd  %xmm0, (%rdx)
 139         movlpd  %xmm1, 8(%rdx)
 140         mov     %rdi, %rax
 141         ret
 142
 143         .p2align 4
 144 L(StrncatExit1):
 145         xor     %ah, %ah
 146         movb    %ah, 1(%rdx)
 147 L(Exit1):
 148         movb    (%rcx), %al
 149         movb    %al, (%rdx)
 150         mov     %rdi, %rax
 151         ret
 152
 153         .p2align 4
 154 L(StrncatExit2):
 155         xor     %ah, %ah
 156         movb    %ah, 2(%rdx)
 157 L(Exit2):
 158         movw    (%rcx), %ax
 159         movw    %ax, (%rdx)
 160         mov     %rdi, %rax
 161         ret
 162
 163         .p2align 4
 164 L(StrncatExit3):
 165         xor     %ah, %ah
 166         movb    %ah, 3(%rdx)
 167 L(Exit3):
 168         movw    (%rcx), %ax
 169         movw    %ax, (%rdx)
 170         movb    2(%rcx), %al
 171         movb    %al, 2(%rdx)
 172         mov     %rdi, %rax
 173         ret
 174
 175         .p2align 4
 176 L(StrncatExit4):
 177         xor     %ah, %ah
 178         movb    %ah, 4(%rdx)
 179 L(Exit4):
 180         mov     (%rcx), %eax
 181         mov     %eax, (%rdx)
 182         mov     %rdi, %rax
 183         ret
 184
 185         .p2align 4
 186 L(StrncatExit5):
 187         xor     %ah, %ah
 188         movb    %ah, 5(%rdx)
 189 L(Exit5):
 190         mov     (%rcx), %eax
 191         mov     %eax, (%rdx)
 192         movb    4(%rcx), %al
 193         movb    %al, 4(%rdx)
 194         mov     %rdi, %rax
 195         ret
 196
 197         .p2align 4
 198 L(StrncatExit6):
 199         xor     %ah, %ah
 200         movb    %ah, 6(%rdx)
 201 L(Exit6):
 202         mov     (%rcx), %eax
 203         mov     %eax, (%rdx)
 204         movw    4(%rcx), %ax
 205         movw    %ax, 4(%rdx)
 206         mov     %rdi, %rax
 207         ret
 208
 209         .p2align 4
 210 L(StrncatExit7):
 211         xor     %ah, %ah
 212         movb    %ah, 7(%rdx)
 213 L(Exit7):
 214         mov     (%rcx), %eax
 215         mov     %eax, (%rdx)
 216         mov     3(%rcx), %eax
 217         mov     %eax, 3(%rdx)
 218         mov     %rdi, %rax
 219         ret
 220
 221         .p2align 4
 222 L(StrncatExit8):
 223         xor     %ah, %ah
 224         movb    %ah, 8(%rdx)
 225 L(Exit8):
 226         movlpd  (%rcx), %xmm0
 227         movlpd  %xmm0, (%rdx)
 228         mov     %rdi, %rax
 229         ret
 230
 231         .p2align 4
 232 L(StrncatExit9):
 233         xor     %ah, %ah
 234         movb    %ah, 9(%rdx)
 235 L(Exit9):
 236         movlpd  (%rcx), %xmm0
 237         movlpd  %xmm0, (%rdx)
 238         movb    8(%rcx), %al
 239         movb    %al, 8(%rdx)
 240         mov     %rdi, %rax
 241         ret
 242
 243         .p2align 4
 244 L(StrncatExit10):
 245         xor     %ah, %ah
 246         movb    %ah, 10(%rdx)
 247 L(Exit10):
 248         movlpd  (%rcx), %xmm0
 249         movlpd  %xmm0, (%rdx)
 250         movw    8(%rcx), %ax
 251         movw    %ax, 8(%rdx)
 252         mov     %rdi, %rax
 253         ret
 254
 255         .p2align 4
 256 L(StrncatExit11):
 257         xor     %ah, %ah
 258         movb    %ah, 11(%rdx)
 259 L(Exit11):
 260         movlpd  (%rcx), %xmm0
 261         movlpd  %xmm0, (%rdx)
 262         mov     7(%rcx), %eax
 263         mov     %eax, 7(%rdx)
 264         mov     %rdi, %rax
 265         ret
 266
 267         .p2align 4
 268 L(StrncatExit12):
 269         xor     %ah, %ah
 270         movb    %ah, 12(%rdx)
 271 L(Exit12):
 272         movlpd  (%rcx), %xmm0
 273         movlpd  %xmm0, (%rdx)
 274         mov     8(%rcx), %eax
 275         mov     %eax, 8(%rdx)
 276         mov     %rdi, %rax
 277         ret
 278
 279         .p2align 4
 280 L(StrncatExit13):
 281         xor     %ah, %ah
 282         movb    %ah, 13(%rdx)
 283 L(Exit13):
 284         movlpd  (%rcx), %xmm0
 285         movlpd  %xmm0, (%rdx)
 286         movlpd  5(%rcx), %xmm1
 287         movlpd  %xmm1, 5(%rdx)
 288         mov     %rdi, %rax
 289         ret
 290
 291         .p2align 4
 292 L(StrncatExit14):
 293         xor     %ah, %ah
 294         movb    %ah, 14(%rdx)
 295 L(Exit14):
 296         movlpd  (%rcx), %xmm0
 297         movlpd  %xmm0, (%rdx)
 298         movlpd  6(%rcx), %xmm1
 299         movlpd  %xmm1, 6(%rdx)
 300         mov     %rdi, %rax
 301         ret
 302
 303         .p2align 4
 304 L(StrncatExit15):
 305         xor     %ah, %ah
 306         movb    %ah, 15(%rdx)
 307 L(Exit15):
 308         movlpd  (%rcx), %xmm0
 309         movlpd  %xmm0, (%rdx)
 310         movlpd  7(%rcx), %xmm1
 311         movlpd  %xmm1, 7(%rdx)
 312         mov     %rdi, %rax
 313         ret
 314
 315         .p2align 4
 316 L(StrncatExit16):
 317         xor     %ah, %ah
 318         movb    %ah, 16(%rdx)
 319 L(Exit16):
 320         movlpd  (%rcx), %xmm0
 321         movlpd  8(%rcx), %xmm1
 322         movlpd  %xmm0, (%rdx)
 323         movlpd  %xmm1, 8(%rdx)
 324         mov     %rdi, %rax
 325         ret
 326
 327 # ifdef USE_AS_STRNCPY
 328
 329         .p2align 4
 330 L(CopyFrom1To16BytesCase2):
 331         add     $16, %r8
 332         add     %rsi, %rcx
 333         lea     (%rsi, %rdx), %rsi
 334         lea     -9(%r8), %rdx
 335         and     $1<<7, %dh
 336         or      %al, %dh
 337         test    %dh, %dh
 338         lea     (%rsi), %rdx
 339         jz      L(ExitHighCase2)
 340
 341         test    $0x01, %al
 342         jnz     L(Exit1)
 343         cmp     $1, %r8
 344         je      L(StrncatExit1)
 345         test    $0x02, %al
 346         jnz     L(Exit2)
 347         cmp     $2, %r8
 348         je      L(StrncatExit2)
 349         test    $0x04, %al
 350         jnz     L(Exit3)
 351         cmp     $3, %r8
 352         je      L(StrncatExit3)
 353         test    $0x08, %al
 354         jnz     L(Exit4)
 355         cmp     $4, %r8
 356         je      L(StrncatExit4)
 357         test    $0x10, %al
 358         jnz     L(Exit5)
 359         cmp     $5, %r8
 360         je      L(StrncatExit5)
 361         test    $0x20, %al
 362         jnz     L(Exit6)
 363         cmp     $6, %r8
 364         je      L(StrncatExit6)
 365         test    $0x40, %al
 366         jnz     L(Exit7)
 367         cmp     $7, %r8
 368         je      L(StrncatExit7)
 369         movlpd  (%rcx), %xmm0
 370         movlpd  %xmm0, (%rdx)
 371         lea     7(%rdx), %rax
 372         cmpb    $1, (%rax)
 373         sbb     $-1, %rax
 374         xor     %cl, %cl
 375         movb    %cl, (%rax)
 376         mov     %rdi, %rax
 377         ret
 378
 379         .p2align 4
 380 L(ExitHighCase2):
 381         test    $0x01, %ah
 382         jnz     L(Exit9)
 383         cmp     $9, %r8
 384         je      L(StrncatExit9)
 385         test    $0x02, %ah
 386         jnz     L(Exit10)
 387         cmp     $10, %r8
 388         je      L(StrncatExit10)
 389         test    $0x04, %ah
 390         jnz     L(Exit11)
 391         cmp     $11, %r8
 392         je      L(StrncatExit11)
 393         test    $0x8, %ah
 394         jnz     L(Exit12)
 395         cmp     $12, %r8
 396         je      L(StrncatExit12)
 397         test    $0x10, %ah
 398         jnz     L(Exit13)
 399         cmp     $13, %r8
 400         je      L(StrncatExit13)
 401         test    $0x20, %ah
 402         jnz     L(Exit14)
 403         cmp     $14, %r8
 404         je      L(StrncatExit14)
 405         test    $0x40, %ah
 406         jnz     L(Exit15)
 407         cmp     $15, %r8
 408         je      L(StrncatExit15)
 409         movlpd  (%rcx), %xmm0
 410         movlpd  %xmm0, (%rdx)
 411         movlpd  8(%rcx), %xmm1
 412         movlpd  %xmm1, 8(%rdx)
 413         mov     %rdi, %rax
 414         ret
 415
 416 L(CopyFrom1To16BytesCase2OrCase3):
 417         test    %rax, %rax
 418         jnz     L(CopyFrom1To16BytesCase2)
 419
 420         .p2align 4
 421 L(CopyFrom1To16BytesCase3):
 422         add     $16, %r8
 423         add     %rsi, %rdx
 424         add     %rsi, %rcx
 425
 426         cmp     $8, %r8
 427         ja      L(ExitHighCase3)
 428         cmp     $1, %r8
 429         je      L(StrncatExit1)
 430         cmp     $2, %r8
 431         je      L(StrncatExit2)
 432         cmp     $3, %r8
 433         je      L(StrncatExit3)
 434         cmp     $4, %r8
 435         je      L(StrncatExit4)
 436         cmp     $5, %r8
 437         je      L(StrncatExit5)
 438         cmp     $6, %r8
 439         je      L(StrncatExit6)
 440         cmp     $7, %r8
 441         je      L(StrncatExit7)
 442         movlpd  (%rcx), %xmm0
 443         movlpd  %xmm0, (%rdx)
 444         xor     %ah, %ah
 445         movb    %ah, 8(%rdx)
 446         mov     %rdi, %rax
 447         ret
 448
 449         .p2align 4
 450 L(ExitHighCase3):
 451         cmp     $9, %r8
 452         je      L(StrncatExit9)
 453         cmp     $10, %r8
 454         je      L(StrncatExit10)
 455         cmp     $11, %r8
 456         je      L(StrncatExit11)
 457         cmp     $12, %r8
 458         je      L(StrncatExit12)
 459         cmp     $13, %r8
 460         je      L(StrncatExit13)
 461         cmp     $14, %r8
 462         je      L(StrncatExit14)
 463         cmp     $15, %r8
 464         je      L(StrncatExit15)
 465         movlpd  (%rcx), %xmm0
 466         movlpd  %xmm0, (%rdx)
 467         movlpd  8(%rcx), %xmm1
 468         movlpd  %xmm1, 8(%rdx)
 469         xor     %ah, %ah
 470         movb    %ah, 16(%rdx)
 471         mov     %rdi, %rax
 472         ret
 473
 474         .p2align 4
 475 L(StrncatExit0):
 476         mov     %rdi, %rax
 477         ret
 478
 479         .p2align 4
 480 L(StrncatExit15Bytes):
 481         cmp     $9, %r8
 482         je      L(StrncatExit9)
 483         cmpb    $0, 9(%rcx)
 484         jz      L(Exit10)
 485         cmp     $10, %r8
 486         je      L(StrncatExit10)
 487         cmpb    $0, 10(%rcx)
 488         jz      L(Exit11)
 489         cmp     $11, %r8
 490         je      L(StrncatExit11)
 491         cmpb    $0, 11(%rcx)
 492         jz      L(Exit12)
 493         cmp     $12, %r8
 494         je      L(StrncatExit12)
 495         cmpb    $0, 12(%rcx)
 496         jz      L(Exit13)
 497         cmp     $13, %r8
 498         je      L(StrncatExit13)
 499         cmpb    $0, 13(%rcx)
 500         jz      L(Exit14)
 501         cmp     $14, %r8
 502         je      L(StrncatExit14)
 503         movlpd  (%rcx), %xmm0
 504         movlpd  %xmm0, (%rdx)
 505         movlpd  7(%rcx), %xmm1
 506         movlpd  %xmm1, 7(%rdx)
 507         lea     14(%rdx), %rax
 508         cmpb    $1, (%rax)
 509         sbb     $-1, %rax
 510         xor     %cl, %cl
 511         movb    %cl, (%rax)
 512         mov     %rdi, %rax
 513         ret
 514
 515         .p2align 4
 516 L(StrncatExit8Bytes):
 517         cmpb    $0, (%rcx)
 518         jz      L(Exit1)
 519         cmp     $1, %r8
 520         je      L(StrncatExit1)
 521         cmpb    $0, 1(%rcx)
 522         jz      L(Exit2)
 523         cmp     $2, %r8
 524         je      L(StrncatExit2)
 525         cmpb    $0, 2(%rcx)
 526         jz      L(Exit3)
 527         cmp     $3, %r8
 528         je      L(StrncatExit3)
 529         cmpb    $0, 3(%rcx)
 530         jz      L(Exit4)
 531         cmp     $4, %r8
 532         je      L(StrncatExit4)
 533         cmpb    $0, 4(%rcx)
 534         jz      L(Exit5)
 535         cmp     $5, %r8
 536         je      L(StrncatExit5)
 537         cmpb    $0, 5(%rcx)
 538         jz      L(Exit6)
 539         cmp     $6, %r8
 540         je      L(StrncatExit6)
 541         cmpb    $0, 6(%rcx)
 542         jz      L(Exit7)
 543         cmp     $7, %r8
 544         je      L(StrncatExit7)
 545         movlpd  (%rcx), %xmm0
 546         movlpd  %xmm0, (%rdx)
 547         lea     7(%rdx), %rax
 548         cmpb    $1, (%rax)
 549         sbb     $-1, %rax
 550         xor     %cl, %cl
 551         movb    %cl, (%rax)
 552         mov     %rdi, %rax
 553         ret
 554
 555 # endif
 556 END (STRCAT)
 557 #endif