sve: combine nested if predicates

The following example

void f5(float * restrict z0, float * restrict z1, float *restrict x,
	float * restrict y, float c, int n)
{
    for (int i = 0; i < n; i++) {
        float a = x[i];
        float b = y[i];
        if (a > b) {
            z0[i] = a + b;
            if (a > c) {
                z1[i] = a - b;
            }
        }
    }
}

generates currently:

        ptrue   p3.b, all
        ld1w    z1.s, p1/z, [x2, x5, lsl 2]
        ld1w    z2.s, p1/z, [x3, x5, lsl 2]
        fcmgt   p0.s, p3/z, z1.s, z0.s
        fcmgt   p2.s, p1/z, z1.s, z2.s
        fcmgt   p0.s, p0/z, z1.s, z2.s
        and     p0.b, p0/z, p1.b, p1.b

The conditions for a > b and a > c become separate comparisons.

After this patch we generate:

        ld1w    z1.s, p0/z, [x2, x5, lsl 2]
        ld1w    z2.s, p0/z, [x3, x5, lsl 2]
        fcmgt   p1.s, p0/z, z1.s, z2.s
        fcmgt   p1.s, p1/z, z1.s, z0.s

Where the condition a > b && a > c are folded by using the predicate result of
the previous compare and thus allows the removal of one of the compares.

When never a mask is being generated from an BIT_AND we mask the operands of
the and instead and then just AND the result.

This allows us to be able to CSE the masks and generate the right combination.
However because re-assoc will try to re-order the masks in the & we have to now
perform a small local CSE on the vectorized loop is vectorization is successful.

Note: This patch series is working incrementally towards generating the most
      efficient code for this and other loops in small steps.

gcc/ChangeLog:

	* tree-vect-stmts.c (prepare_load_store_mask): Rename to...
	(prepare_vec_mask): ...This and record operations that have already been
	masked.
	(vectorizable_call): Use it.
	(vectorizable_operation): Likewise.
	(vectorizable_store): Likewise.
	(vectorizable_load): Likewise.
	* tree-vectorizer.h (class _loop_vec_info): Add vec_cond_masked_set.
	(vec_cond_masked_set_type, tree_cond_mask_hash): New.

gcc/testsuite/ChangeLog:

	* gcc.target/aarch64/sve/pred-combine-and.c: New test.

diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pred-combine-and.c b/gcc/testsuite/gcc.target/aarch64/sve/pred-combine-and.c
new file mode 100644 (file)
index 0000000..ee92734
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pred-combine-and.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O3" } */
+
+void f5(float * restrict z0, float * restrict z1, float *restrict x, float * restrict y, float c, int n)
+{
+    for (int i = 0; i < n; i++) {
+        float a = x[i];
+        float b = y[i];
+        if (a > b) {
+            z0[i] = a + b;
+            if (a > c) {
+                z1[i] = a - b;
+            }
+        }
+    }
+}
+
+/* { dg-final { scan-assembler-times {\tfcmgt\tp[0-9]+\.s, p[0-9]+/z, z[0-9]+\.s, z[0-9]+\.s} 2 } } */

diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index e63bc2afdaafd95e6b3715c317deb0f1ed465f14..84c6d9777dbd5147014baa56adf0096e4703eae5 100644 (file)
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -1795,23 +1795,29 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
 /* Return the mask input to a masked load or store.  VEC_MASK is the vectorized
    form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
    that needs to be applied to all loads and stores in a vectorized loop.
-   Return VEC_MASK if LOOP_MASK is null, otherwise return VEC_MASK & LOOP_MASK.
+   Return VEC_MASK if LOOP_MASK is null or if VEC_MASK is already masked,
+   otherwise return VEC_MASK & LOOP_MASK.
 
    MASK_TYPE is the type of both masks.  If new statements are needed,
    insert them before GSI.  */
 
 static tree
-prepare_load_store_mask (tree mask_type, tree loop_mask, tree vec_mask,
-                        gimple_stmt_iterator *gsi)
+prepare_vec_mask (loop_vec_info loop_vinfo, tree mask_type, tree loop_mask,
+                 tree vec_mask, gimple_stmt_iterator *gsi)
 {
   gcc_assert (useless_type_conversion_p (mask_type, TREE_TYPE (vec_mask)));
   if (!loop_mask)
     return vec_mask;
 
   gcc_assert (TREE_TYPE (loop_mask) == mask_type);
+
+  if (loop_vinfo->vec_cond_masked_set.contains ({ vec_mask, loop_mask }))
+    return vec_mask;
+
   tree and_res = make_temp_ssa_name (mask_type, NULL, "vec_mask_and");
   gimple *and_stmt = gimple_build_assign (and_res, BIT_AND_EXPR,
                                          vec_mask, loop_mask);
+
   gsi_insert_before (gsi, and_stmt, GSI_SAME_STMT);
   return and_res;
 }
@@ -3568,8 +3574,9 @@ vectorizable_call (vec_info *vinfo,
                          gcc_assert (ncopies == 1);
                          tree mask = vect_get_loop_mask (gsi, masks, vec_num,
                                                          vectype_out, i);
-                         vargs[mask_opno] = prepare_load_store_mask
-                           (TREE_TYPE (mask), mask, vargs[mask_opno], gsi);
+                         vargs[mask_opno] = prepare_vec_mask
+                           (loop_vinfo, TREE_TYPE (mask), mask,
+                            vargs[mask_opno], gsi);
                        }
 
                      gcall *call;
@@ -3612,8 +3619,8 @@ vectorizable_call (vec_info *vinfo,
              tree mask = vect_get_loop_mask (gsi, masks, ncopies,
                                              vectype_out, j);
              vargs[mask_opno]
-               = prepare_load_store_mask (TREE_TYPE (mask), mask,
-                                          vargs[mask_opno], gsi);
+               = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
+                                   vargs[mask_opno], gsi);
            }
 
          gimple *new_stmt;
@@ -6350,10 +6357,43 @@ vectorizable_operation (vec_info *vinfo,
        }
       else
        {
+         tree mask = NULL_TREE;
+         /* When combining two masks check if either of them is elsewhere
+            combined with a loop mask, if that's the case we can mark that the
+            new combined mask doesn't need to be combined with a loop mask.  */
+         if (masked_loop_p && code == BIT_AND_EXPR)
+           {
+             if (loop_vinfo->scalar_cond_masked_set.contains ({ op0,
+                                                                ncopies}))
+               {
+                 mask = vect_get_loop_mask (gsi, masks, vec_num * ncopies,
+                                            vectype, i);
+
+                 vop0 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
+                                          vop0, gsi);
+               }
+
+             if (loop_vinfo->scalar_cond_masked_set.contains ({ op1,
+                                                                ncopies }))
+               {
+                 mask = vect_get_loop_mask (gsi, masks, vec_num * ncopies,
+                                            vectype, i);
+
+                 vop1 = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
+                                          vop1, gsi);
+               }
+           }
+
          new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1, vop2);
          new_temp = make_ssa_name (vec_dest, new_stmt);
          gimple_assign_set_lhs (new_stmt, new_temp);
          vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+
+         /* Enter the combined value into the vector cond hash so we don't
+            AND it with a loop mask again.  */
+         if (mask)
+           loop_vinfo->vec_cond_masked_set.add ({ new_temp, mask });
+
          if (vec_cvt_dest)
            {
              new_temp = build1 (VIEW_CONVERT_EXPR, vectype_out, new_temp);
@@ -8214,8 +8254,8 @@ vectorizable_store (vec_info *vinfo,
            final_mask = vect_get_loop_mask (gsi, loop_masks, ncopies,
                                             vectype, j);
          if (vec_mask)
-           final_mask = prepare_load_store_mask (mask_vectype, final_mask,
-                                                 vec_mask, gsi);
+           final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
+                                          final_mask, vec_mask, gsi);
 
          gcall *call;
          if (final_mask)
@@ -8269,8 +8309,8 @@ vectorizable_store (vec_info *vinfo,
                                                 vec_num * ncopies,
                                                 vectype, vec_num * j + i);
              if (vec_mask)
-               final_mask = prepare_load_store_mask (mask_vectype, final_mask,
-                                                     vec_mask, gsi);
+               final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
+                                              final_mask, vec_mask, gsi);
 
              if (memory_access_type == VMAT_GATHER_SCATTER)
                {
@@ -9497,8 +9537,8 @@ vectorizable_load (vec_info *vinfo,
            final_mask = vect_get_loop_mask (gsi, loop_masks, ncopies,
                                             vectype, j);
          if (vec_mask)
-           final_mask = prepare_load_store_mask (mask_vectype, final_mask,
-                                                 vec_mask, gsi);
+           final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
+                                          final_mask, vec_mask, gsi);
 
          gcall *call;
          if (final_mask)
@@ -9549,8 +9589,8 @@ vectorizable_load (vec_info *vinfo,
                                                 vec_num * ncopies,
                                                 vectype, vec_num * j + i);
              if (vec_mask)
-               final_mask = prepare_load_store_mask (mask_vectype, final_mask,
-                                                     vec_mask, gsi);
+               final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
+                                              final_mask, vec_mask, gsi);
 
              if (i > 0)
                dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,

diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 76e81ea546ad4ac75572c1311a8284f96249b497..2f6e1e268fb07e9de065ff9c45af87546e565d66 100644 (file)
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -328,6 +328,12 @@ struct default_hash_traits<scalar_cond_masked_key>
 
 typedef hash_set<scalar_cond_masked_key> scalar_cond_masked_set_type;
 
+/* Key and map that records association between vector conditions and
+   corresponding loop mask, and is populated by prepare_vec_mask.  */
+
+typedef pair_hash<tree_operand_hash, tree_operand_hash> tree_cond_mask_hash;
+typedef hash_set<tree_cond_mask_hash> vec_cond_masked_set_type;
+
 /* Describes two objects whose addresses must be unequal for the vectorized
    loop to be valid.  */
 typedef std::pair<tree, tree> vec_object_pair;
@@ -647,6 +653,9 @@ public:
   /* Set of scalar conditions that have loop mask applied.  */
   scalar_cond_masked_set_type scalar_cond_masked_set;
 
+  /* Set of vector conditions that have loop mask applied.  */
+  vec_cond_masked_set_type vec_cond_masked_set;
+
   /* If we are using a loop mask to align memory addresses, this variable
      contains the number of vector elements that we should skip in the
      first iteration of the vector loop (i.e. the number of leading