tree-optimization/103489 - fix ICE when bool pattern recog fails

bool pattern recog currently does not handle cycles correctly
and when it fails we can ICE later vectorizing PHIs with
mismatched bool and non-bool vector types.  The following avoids
blindly trusting bool pattern recog here and verifies things
more thoroughly in vectorizable_phi.  A bool pattern recog fix
is for GCC 13.

2021-11-30  Richard Biener  <rguenther@suse.de>

	PR tree-optimization/103489
	* tree-vect-loop.c (vectorizable_phi): Verify argument
	vector type compatibility to mitigate bool pattern recog
	bug.

	* gcc.dg/torture/pr103489.c: New testcase.

diff --git a/gcc/testsuite/gcc.dg/torture/pr103489.c b/gcc/testsuite/gcc.dg/torture/pr103489.c
new file mode 100644 (file)
index 0000000..cd62623
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr103489.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-ftree-vectorize" } */
+
+_Bool a[80];
+short b, f;
+void g(short h[][8][16])
+{
+  for (_Bool c = 0; c < b;)
+    for (_Bool d = 0; d < (_Bool)f; d = 1)
+      for (short e = 0; e < 16; e++)
+        a[e] = h[b][1][e];
+}

diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
index 841da78f1fd10f40abc72eef97cd3ad100529294..7f544ba1fd5198dd32cda05e62382ab2e1e9bb50 100644 (file)
--- a/gcc/tree-vect-loop.c
+++ b/gcc/tree-vect-loop.c
@@ -7846,6 +7846,24 @@ vectorizable_phi (vec_info *,
                               "incompatible vector types for invariants\n");
            return false;
          }
+       else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def
+                && !useless_type_conversion_p (vectype,
+                                               SLP_TREE_VECTYPE (child)))
+         {
+           /* With bools we can have mask and non-mask precision vectors,
+              while pattern recog is supposed to guarantee consistency here
+              bugs in it can cause mismatches (PR103489 for example).
+              Deal with them here instead of ICEing later.  */
+           if (dump_enabled_p ())
+             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+                              "incompatible vector type setup from "
+                              "bool pattern detection\n");
+           gcc_checking_assert
+             (VECTOR_BOOLEAN_TYPE_P (SLP_TREE_VECTYPE (child))
+              != VECTOR_BOOLEAN_TYPE_P (vectype));
+           return false;
+         }
+
       /* For single-argument PHIs assume coalescing which means zero cost
         for the scalar and the vector PHIs.  This avoids artificially
         favoring the vector path (but may pessimize it in some cases).  */