isel: Fix ICE on out of bounds vector elt access [PR126446]

The isel pass has a check for out of bounds constant index before
optimizing into .VEC_SET, but it does it using
      // if index is a constant, then check the bounds
      poly_uint64 idx_poly;
      if (poly_int_tree_p (idx, &idx_poly))
        {
          poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (view_op0));
          if (known_gt (idx_poly, nelts))
            return false;
        }
In the testcase below, idx is INTEGER_CST with long long type and
negative value, that doesn't fit into poly_uint64, so we happily convert
it into .VEC_SET.

Furthermore, the known_gt check looks wrong, already idx_poly known_eq
to nelts is too large and out of bounds for .VEC_SET.

This patch fixes that by punting if !poly_int_tree_p (idx, &idx_poly)
and poly_int_tree_p (idx), so when it is INTEGER_CST or POLY_INT_CST
which doesn't fit into poly_uint64 (so likely negative), and
uses known_ge instead of known_gt.

2026-07-29  Jakub Jelinek  <jakub@redhat.com>

	PR target/126446
	* gimple-isel.cc (gimple_expand_vec_set_extract_expr): Punt if
	idx doesn't fit into poly_uint64 but is poly_int_tree_p.  Use
	known_ge rather than known_gt for out of bounds check.  Formatting
	fixes.

Reviewed-by: Richard Biener <rguenth@suse.de>

diff --git a/gcc/gimple-isel.cc b/gcc/gimple-isel.cc
index 25c11d645405e09c8de714010c12b9242c231fa4..1d346ee124ffdfdccf658ea79cc72cbf80240f81 100644 (file)
--- a/gcc/gimple-isel.cc
+++ b/gcc/gimple-isel.cc
@@ -96,10 +96,11 @@ gimple_expand_vec_set_extract_expr (struct function *fun,
     return false;
 
   tree op0 = TREE_OPERAND (ref, 0);
-  if (TREE_CODE (op0) == VIEW_CONVERT_EXPR && DECL_P (TREE_OPERAND (op0, 0))
+  if (TREE_CODE (op0) == VIEW_CONVERT_EXPR
+      && DECL_P (TREE_OPERAND (op0, 0))
       && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (op0, 0)))
-      && TYPE_MODE (TREE_TYPE (ref))
-          == TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0)))))
+      && (TYPE_MODE (TREE_TYPE (ref))
+         == TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0))))))
     {
       tree pos = TREE_OPERAND (ref, 1);
 
@@ -111,9 +112,13 @@ gimple_expand_vec_set_extract_expr (struct function *fun,
       if (poly_int_tree_p (idx, &idx_poly))
        {
          poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (view_op0));
-         if (known_gt (idx_poly, nelts))
+         if (known_ge (idx_poly, nelts))
            return false;
        }
+      else if (poly_int_tree_p (idx))
+       // if idx doesn't fit into poly_uint64, but is constant, it
+       // must be out of bounds
+       return false;
       machine_mode outermode = TYPE_MODE (TREE_TYPE (view_op0));
       machine_mode extract_mode = TYPE_MODE (TREE_TYPE (ref));