From f26f92b534f9d68371322071f309ef3e0e95f38c Mon Sep 17 00:00:00 2001 From: Tamar Christina Date: Tue, 21 Nov 2023 13:17:17 +0000 Subject: [PATCH] AArch64: Add pattern for unsigned widenings (uxtl) to zip{1,2} This changes unpack instructions to use zip{1,2} when doing a zero-extending widening operation. Permutes generally have a higher throughput than the widening operations. Zeros are shuffled into the top half of the registers. The testcase void d2 (unsigned * restrict a, unsigned short *b, int n) { for (int i = 0; i < (n & -8); i++) a[i] = b[i]; } now generates: movi v1.4s, 0 .L3: ldr q0, [x1], 16 zip1 v2.8h, v0.8h, v1.8h zip2 v0.8h, v0.8h, v1.8h stp q2, q0, [x0] add x0, x0, 32 cmp x1, x2 bne .L3 instead of: .L3: ldr q0, [x1], 16 uxtl v1.4s, v0.4h uxtl2 v0.4s, v0.8h stp q1, q0, [x0] add x0, x0, 32 cmp x1, x2 bne .L3 Since we need the extra 0 register we do this only for the vectorizer's lo/hi pairs when we know the 0 will be floated outside of the loop. This gives an 8% speed-up in Imagick in SPECCPU 2017 on Neoverse V2. gcc/ChangeLog: * config/aarch64/aarch64-simd.md (vec_unpack_lo__lo___zip): New. (aarch64_uaddw__zip): New. * config/aarch64/iterators.md (PERM_EXTEND, perm_index): New. (perm_hilo): Add UNSPEC_ZIP1, UNSPEC_ZIP2. gcc/testsuite/ChangeLog: * gcc.target/aarch64/simd/vmovl_high_1.c: Update codegen. * gcc.target/aarch64/uxtl-combine-1.c: New test. * gcc.target/aarch64/uxtl-combine-2.c: New test. * gcc.target/aarch64/uxtl-combine-3.c: New test. * gcc.target/aarch64/uxtl-combine-4.c: New test. * gcc.target/aarch64/uxtl-combine-5.c: New test. * gcc.target/aarch64/uxtl-combine-6.c: New test. --- gcc/config/aarch64/aarch64-simd.md | 78 +++++++++++++++++-- gcc/config/aarch64/iterators.md | 8 +- .../gcc.target/aarch64/simd/vmovl_high_1.c | 6 +- .../gcc.target/aarch64/uxtl-combine-1.c | 20 +++++ .../gcc.target/aarch64/uxtl-combine-2.c | 20 +++++ .../gcc.target/aarch64/uxtl-combine-3.c | 20 +++++ .../gcc.target/aarch64/uxtl-combine-4.c | 20 +++++ .../gcc.target/aarch64/uxtl-combine-5.c | 20 +++++ .../gcc.target/aarch64/uxtl-combine-6.c | 20 +++++ 9 files changed, 200 insertions(+), 12 deletions(-) create mode 100755 gcc/testsuite/gcc.target/aarch64/uxtl-combine-1.c create mode 100755 gcc/testsuite/gcc.target/aarch64/uxtl-combine-2.c create mode 100755 gcc/testsuite/gcc.target/aarch64/uxtl-combine-3.c create mode 100755 gcc/testsuite/gcc.target/aarch64/uxtl-combine-4.c create mode 100755 gcc/testsuite/gcc.target/aarch64/uxtl-combine-5.c create mode 100755 gcc/testsuite/gcc.target/aarch64/uxtl-combine-6.c diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md index c6f2d5828373..75ee65987108 100644 --- a/gcc/config/aarch64/aarch64-simd.md +++ b/gcc/config/aarch64/aarch64-simd.md @@ -2029,26 +2029,60 @@ [(set_attr "type" "neon_shift_imm_long")] ) -(define_expand "vec_unpack_hi_" +(define_expand "vec_unpacku_hi_" [(match_operand: 0 "register_operand") - (ANY_EXTEND: (match_operand:VQW 1 "register_operand"))] + (match_operand:VQW 1 "register_operand")] + "TARGET_SIMD" + { + rtx res = gen_reg_rtx (mode); + rtx tmp = aarch64_gen_shareable_zero (mode); + if (BYTES_BIG_ENDIAN) + emit_insn (gen_aarch64_zip2 (res, tmp, operands[1])); + else + emit_insn (gen_aarch64_zip2 (res, operands[1], tmp)); + emit_move_insn (operands[0], + simplify_gen_subreg (mode, res, mode, 0)); + DONE; + } +) + +(define_expand "vec_unpacks_hi_" + [(match_operand: 0 "register_operand") + (match_operand:VQW 1 "register_operand")] "TARGET_SIMD" { rtx p = aarch64_simd_vect_par_cnst_half (mode, , true); - emit_insn (gen_aarch64_simd_vec_unpack_hi_ (operands[0], - operands[1], p)); + emit_insn (gen_aarch64_simd_vec_unpacks_hi_ (operands[0], + operands[1], p)); + DONE; + } +) + +(define_expand "vec_unpacku_lo_" + [(match_operand: 0 "register_operand") + (match_operand:VQW 1 "register_operand")] + "TARGET_SIMD" + { + rtx res = gen_reg_rtx (mode); + rtx tmp = aarch64_gen_shareable_zero (mode); + if (BYTES_BIG_ENDIAN) + emit_insn (gen_aarch64_zip1 (res, tmp, operands[1])); + else + emit_insn (gen_aarch64_zip1 (res, operands[1], tmp)); + emit_move_insn (operands[0], + simplify_gen_subreg (mode, res, mode, 0)); DONE; } ) -(define_expand "vec_unpack_lo_" +(define_expand "vec_unpacks_lo_" [(match_operand: 0 "register_operand") - (ANY_EXTEND: (match_operand:VQW 1 "register_operand"))] + (match_operand:VQW 1 "register_operand")] "TARGET_SIMD" { rtx p = aarch64_simd_vect_par_cnst_half (mode, , false); - emit_insn (gen_aarch64_simd_vec_unpack_lo_ (operands[0], - operands[1], p)); + emit_insn (gen_aarch64_simd_vec_unpacks_lo_ (operands[0], + operands[1], p)); DONE; } ) @@ -4776,6 +4810,34 @@ [(set_attr "type" "neon_sub_widen")] ) +(define_insn "aarch64_usubw__zip" + [(set (match_operand: 0 "register_operand" "=w") + (minus: + (match_operand: 1 "register_operand" "w") + (subreg: + (unspec: [ + (match_operand:VQW 2 "register_operand" "w") + (match_operand:VQW 3 "aarch64_simd_imm_zero") + ] PERM_EXTEND) 0)))] + "TARGET_SIMD" + "usubw\\t%0., %1., %2." + [(set_attr "type" "neon_sub_widen")] +) + +(define_insn "aarch64_uaddw__zip" + [(set (match_operand: 0 "register_operand" "=w") + (plus: + (subreg: + (unspec: [ + (match_operand:VQW 2 "register_operand" "w") + (match_operand:VQW 3 "aarch64_simd_imm_zero") + ] PERM_EXTEND) 0) + (match_operand: 1 "register_operand" "w")))] + "TARGET_SIMD" + "uaddw\\t%0., %1., %2." + [(set_attr "type" "neon_add_widen")] +) + (define_insn "aarch64_addw" [(set (match_operand: 0 "register_operand" "=w") (plus: diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md index a920de99ffca..2354315d7d24 100644 --- a/gcc/config/aarch64/iterators.md +++ b/gcc/config/aarch64/iterators.md @@ -2645,6 +2645,9 @@ (define_int_iterator OPTAB_PERMUTE [UNSPEC_ZIP1 UNSPEC_ZIP2 UNSPEC_UZP1 UNSPEC_UZP2]) +;; Permutes for zero extends +(define_int_iterator PERM_EXTEND [UNSPEC_ZIP1 UNSPEC_ZIP2]) + (define_int_iterator REVERSE [UNSPEC_REV64 UNSPEC_REV32 UNSPEC_REV16]) (define_int_iterator FRINT [UNSPEC_FRINTZ UNSPEC_FRINTP UNSPEC_FRINTM @@ -3467,7 +3470,10 @@ (UNSPEC_REV16 "16")]) (define_int_attr perm_hilo [(UNSPEC_UNPACKSHI "hi") (UNSPEC_UNPACKUHI "hi") - (UNSPEC_UNPACKSLO "lo") (UNSPEC_UNPACKULO "lo")]) + (UNSPEC_UNPACKSLO "lo") (UNSPEC_UNPACKULO "lo") + (UNSPEC_ZIP2 "hi") (UNSPEC_ZIP1 "lo")]) + +(define_int_attr perm_index [(UNSPEC_ZIP2 "2") (UNSPEC_ZIP1 "")]) ;; Return true if the associated optab refers to the high-numbered lanes, ;; false if it refers to the low-numbered lanes. The convention is for diff --git a/gcc/testsuite/gcc.target/aarch64/simd/vmovl_high_1.c b/gcc/testsuite/gcc.target/aarch64/simd/vmovl_high_1.c index d45bb83e3503..a2d09eaee0de 100644 --- a/gcc/testsuite/gcc.target/aarch64/simd/vmovl_high_1.c +++ b/gcc/testsuite/gcc.target/aarch64/simd/vmovl_high_1.c @@ -22,11 +22,11 @@ FUNC (int32x4_t, int64x2_t, s32) /* { dg-final { scan-assembler-times {sxtl2\tv0\.2d, v0\.4s} 1} } */ FUNC (uint8x16_t, uint16x8_t, u8) -/* { dg-final { scan-assembler-times {uxtl2\tv0\.8h, v0\.16b} 1} } */ +/* { dg-final { scan-assembler-times {zip2\tv0\.16b, v0\.16b} 1} } */ FUNC (uint16x8_t, uint32x4_t, u16) -/* { dg-final { scan-assembler-times {uxtl2\tv0\.4s, v0\.8h} 1} } */ +/* { dg-final { scan-assembler-times {zip2\tv0\.8h, v0\.8h} 1} } */ FUNC (uint32x4_t, uint64x2_t, u32) -/* { dg-final { scan-assembler-times {uxtl2\tv0\.2d, v0\.4s} 1} } */ +/* { dg-final { scan-assembler-times {zip2\tv0\.4s, v0\.4s} 1} } */ diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-1.c b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-1.c new file mode 100755 index 000000000000..68fa9a09fe55 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-1.c @@ -0,0 +1,20 @@ +/* { dg-do assemble } */ +/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */ + +#pragma GCC target "+nosve" + +#define SIGN unsigned +#define TYPE1 char +#define TYPE2 short + +void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n) +{ + for (int i = 0; i < (n & -8); i++) + a[i] = b[i]; +} + +/* { dg-final { scan-assembler-times {\tzip1\t} 1 } } */ +/* { dg-final { scan-assembler-times {\tzip2\t} 1 } } */ +/* { dg-final { scan-assembler-not {\tuxtl\t} } } */ +/* { dg-final { scan-assembler-not {\tuxtl2\t} } } */ + diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-2.c b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-2.c new file mode 100755 index 000000000000..af8a89085cfc --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-2.c @@ -0,0 +1,20 @@ +/* { dg-do assemble } */ +/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */ + +#pragma GCC target "+nosve" + +#define SIGN unsigned +#define TYPE1 short +#define TYPE2 int + +void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n) +{ + for (int i = 0; i < (n & -8); i++) + a[i] = b[i]; +} + +/* { dg-final { scan-assembler-times {\tzip1\t} 1 } } */ +/* { dg-final { scan-assembler-times {\tzip2\t} 1 } } */ +/* { dg-final { scan-assembler-not {\tuxtl\t} } } */ +/* { dg-final { scan-assembler-not {\tuxtl2\t} } } */ + diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-3.c b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-3.c new file mode 100755 index 000000000000..cdae6d09529b --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-3.c @@ -0,0 +1,20 @@ +/* { dg-do assemble } */ +/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */ + +#pragma GCC target "+nosve" + +#define SIGN unsigned +#define TYPE1 int +#define TYPE2 long long + +void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n) +{ + for (int i = 0; i < (n & -8); i++) + a[i] = b[i]; +} + +/* { dg-final { scan-assembler-times {\tzip1\t} 1 } } */ +/* { dg-final { scan-assembler-times {\tzip2\t} 1 } } */ +/* { dg-final { scan-assembler-not {\tuxtl\t} } } */ +/* { dg-final { scan-assembler-not {\tuxtl2\t} } } */ + diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-4.c b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-4.c new file mode 100755 index 000000000000..e1a9c4f5661a --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-4.c @@ -0,0 +1,20 @@ +/* { dg-do assemble } */ +/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */ + +#pragma GCC target "+nosve" + +#define SIGN signed +#define TYPE1 char +#define TYPE2 short + +void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n) +{ + for (int i = 0; i < (n & -8); i++) + a[i] = b[i]; +} + +/* { dg-final { scan-assembler-not {\tzip1\t} } } */ +/* { dg-final { scan-assembler-not {\tzip2\t} } } */ +/* { dg-final { scan-assembler-times {\tsxtl\t} 1 } } */ +/* { dg-final { scan-assembler-time {\tsxtl2\t} 1 } } */ + diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-5.c b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-5.c new file mode 100755 index 000000000000..92b09ba4abba --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-5.c @@ -0,0 +1,20 @@ +/* { dg-do assemble } */ +/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */ + +#pragma GCC target "+nosve" + +#define SIGN signed +#define TYPE1 short +#define TYPE2 int + +void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n) +{ + for (int i = 0; i < (n & -8); i++) + a[i] = b[i]; +} + +/* { dg-final { scan-assembler-not {\tzip1\t} } } */ +/* { dg-final { scan-assembler-not {\tzip2\t} } } */ +/* { dg-final { scan-assembler-times {\tsxtl\t} 1 } } */ +/* { dg-final { scan-assembler-time {\tsxtl2\t} 1 } } */ + diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-6.c b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-6.c new file mode 100755 index 000000000000..5c6e635f29d1 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-6.c @@ -0,0 +1,20 @@ +/* { dg-do assemble } */ +/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */ + +#pragma GCC target "+nosve" + +#define SIGN signed +#define TYPE1 int +#define TYPE2 long long + +void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n) +{ + for (int i = 0; i < (n & -8); i++) + a[i] = b[i]; +} + +/* { dg-final { scan-assembler-not {\tzip1\t} } } */ +/* { dg-final { scan-assembler-not {\tzip2\t} } } */ +/* { dg-final { scan-assembler-times {\tsxtl\t} 1 } } */ +/* { dg-final { scan-assembler-time {\tsxtl2\t} 1 } } */ + -- 2.47.2