bitintlower: Improve __builtin_{clz,ctz,clrsb,ffs,parity,popcount,bswap,bitreverse...

I've noticed that we emit terrible code for the following testcase (all
functions).  We copy the _BitInt values from memory to a temporary and
perform the bit query or bswap/bitreverse operations on the temporary when
it could be done on the value in memory directly, and for bswap/bitreverse
we can also merge it with a following store to (non-bitfield) memory.
The only exception is that we need some temporary for the
x = __builtin_{bswap,bitreverse}g (x);
cases, when they load from the same memory as they store to.  Because the
implementation loops over all the limbs and swaps them, we need one
temporary in that case.  This is similar to e.g. multiplication/division.
The bit-query ifns return a small integer scalar, so they don't need any
merging with a store.

The following patch implements this.

2026-06-05  Jakub Jelinek  <jakub@redhat.com>

	* gimple-lower-bitint.cc (bitint_large_huge::lower_bswap_bitreverse):
	Add OBJ argument, use it instead of m_vars[part] if non-NULL.  Don't
	gsi_remove stmt.
	(bitint_large_huge::lower_call): Adjust caller.
	(bitint_large_huge::lower_stmt): Handle store of
	IFN_BSWAP/IFN_BITREVERSE result.
	(stmt_needs_operand_addr): Return true also for
	IFN_BSWAP/IFN_BITREVERSE.
	(build_bitint_stmt_ssa_conflicts): Formatting fix.
	(gimple_lower_bitint): Allow merging IFN_BSWAP/IFN_BITREVERSE with
	subsequent non-bitfield store.  Allow merging load of
	IFN_{CLZ,CTZ,CLRSB,FFS,PARITY,POPCOUNT,BSWAP,BITREVERSE} argument
	with the call.

	* gcc.dg/bitint-137.c: New test.

Reviewed-by: Richard Biener <rguenth@suse.de>

diff --git a/gcc/gimple-lower-bitint.cc b/gcc/gimple-lower-bitint.cc
index dc443a4a72f6714144dd31c7abeca98b4fd6ce1b..19e39f4d7efbd41aa530ceb75f6bed810abd2df2 100644 (file)
--- a/gcc/gimple-lower-bitint.cc
+++ b/gcc/gimple-lower-bitint.cc
@@ -474,7 +474,7 @@ struct bitint_large_huge
   void lower_cplxpart_stmt (tree, gimple *);
   void lower_complexexpr_stmt (gimple *);
   void lower_bit_query (gimple *);
-  void lower_bswap_bitreverse (gimple *);
+  void lower_bswap_bitreverse (tree, gimple *);
   void lower_call (tree, gimple *);
   void lower_asm (gimple *);
   void lower_stmt (gimple *);
@@ -6110,7 +6110,7 @@ bitint_large_huge::lower_bit_query (gimple *stmt)
 /* Lower a .{BSWAP,BITREVERSE} call with one large/huge _BitInt argument.  */
 
 void
-bitint_large_huge::lower_bswap_bitreverse (gimple *stmt)
+bitint_large_huge::lower_bswap_bitreverse (tree obj, gimple *stmt)
 {
   tree arg = gimple_call_arg (stmt, 0);
   tree lhs = gimple_call_lhs (stmt);
@@ -6126,9 +6126,12 @@ bitint_large_huge::lower_bswap_bitreverse (gimple *stmt)
   gcc_assert (BITINT_TYPE_P (type));
   bitint_prec_kind kind = bitint_precision_kind (type);
   gcc_assert (kind >= bitint_prec_large);
-  int part = var_to_partition (m_map, lhs);
-  gcc_assert (m_vars[part] != NULL_TREE);
-  tree obj = m_vars[part];
+  if (!obj)
+    {
+      int part = var_to_partition (m_map, lhs);
+      gcc_assert (m_vars[part] != NULL_TREE);
+      obj = m_vars[part];
+    }
   enum internal_fn ifn = gimple_call_internal_fn (stmt);
   enum built_in_function bcode = END_BUILTINS;
   switch (limb_prec)
@@ -6279,7 +6282,6 @@ bitint_large_huge::lower_bswap_bitreverse (gimple *stmt)
       g = gimple_build_assign (l, build_zero_cst (m_limb_type));
       insert_before (g);
     }
-  gsi_remove (&gsi, true);
 }
 
 /* Lower a call statement with one or more large/huge _BitInt
@@ -6313,7 +6315,7 @@ bitint_large_huge::lower_call (tree obj, gimple *stmt)
        return;
       case IFN_BSWAP:
       case IFN_BITREVERSE:
-       lower_bswap_bitreverse (stmt);
+       lower_bswap_bitreverse (obj, stmt);
        return;
       default:
        break;
@@ -6626,6 +6628,17 @@ bitint_large_huge::lower_stmt (gimple *stmt)
              lower_call (lhs, g);
              goto handled;
            }
+         else if (is_gimple_call (g) && gimple_call_internal_p (g))
+           switch (gimple_call_internal_fn (g))
+             {
+             case IFN_BSWAP:
+             case IFN_BITREVERSE:
+               lower_call (lhs, g);
+               goto handled;
+             default:
+               break;
+             }
+
          m_loc = gimple_location (stmt);
        }
     }
@@ -6780,9 +6793,20 @@ stmt_needs_operand_addr (gimple *stmt)
       default:
        break;
       }
-  else if (gimple_call_internal_p (stmt, IFN_MUL_OVERFLOW)
-          || gimple_call_internal_p (stmt, IFN_UBSAN_CHECK_MUL))
-    return true;
+  else if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
+    switch (gimple_call_internal_fn (stmt))
+      {
+      case IFN_MUL_OVERFLOW:
+      case IFN_UBSAN_CHECK_MUL:
+        return true;
+      /* These two actually don't take address, but reshuffle
+        all bytes or bits, so need similar treatment.  */
+      case IFN_BSWAP:
+      case IFN_BITREVERSE:
+       return true;
+      default:
+       break;
+      }
   return false;
 }
 
@@ -6995,8 +7019,7 @@ build_bitint_stmt_ssa_conflicts (gimple *stmt, live_track *live,
            }
        }
     }
-  else if (is_gimple_call (stmt)
-          && gimple_call_internal_p (stmt))
+  else if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
     switch (gimple_call_internal_fn (stmt))
       {
       case IFN_ADD_OVERFLOW:
@@ -7811,6 +7834,27 @@ gimple_lower_bitint (void)
                  default:
                    break;
                  }
+             else if (is_gimple_call (SSA_NAME_DEF_STMT (s))
+                      && gimple_call_internal_p (SSA_NAME_DEF_STMT (s)))
+               switch (gimple_call_internal_fn (SSA_NAME_DEF_STMT (s)))
+                 {
+                 case IFN_BSWAP:
+                 case IFN_BITREVERSE:
+                   if (gimple_store_p (use_stmt)
+                       && is_gimple_assign (use_stmt)
+                       && !gimple_has_volatile_ops (use_stmt)
+                       && !stmt_ends_bb_p (use_stmt))
+                     {
+                       tree lhs = gimple_assign_lhs (use_stmt);
+                       if (TREE_CODE (lhs) == COMPONENT_REF
+                           && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
+                         break;
+                       continue;
+                     }
+                   break;
+                 default:
+                   break;
+                 }
            }
 
          /* Also ignore uninitialized uses.  */
@@ -7866,6 +7910,22 @@ gimple_lower_bitint (void)
                  gimple *use_stmt = USE_STMT (use_p);
                  if (is_gimple_debug (use_stmt))
                    continue;
+                 if (is_gimple_call (use_stmt)
+                     && gimple_call_internal_p (use_stmt))
+                   switch (gimple_call_internal_fn (use_stmt))
+                     {
+                     case IFN_CLZ:
+                     case IFN_CTZ:
+                     case IFN_CLRSB:
+                     case IFN_FFS:
+                     case IFN_PARITY:
+                     case IFN_POPCOUNT:
+                     case IFN_BSWAP:
+                     case IFN_BITREVERSE:
+                       continue;
+                     default:
+                       break;
+                     }
                  if (gimple_code (use_stmt) == GIMPLE_PHI
                      || is_gimple_call (use_stmt)
                      || gimple_code (use_stmt) == GIMPLE_ASM

diff --git a/gcc/testsuite/gcc.dg/bitint-137.c b/gcc/testsuite/gcc.dg/bitint-137.c
new file mode 100644 (file)
index 0000000..458ce8b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/bitint-137.c
@@ -0,0 +1,66 @@
+/* { dg-do compile { target bitint575 } } */
+/* { dg-options "-O2" } */
+
+unsigned _BitInt(575) x, y;
+signed _BitInt(575) z;
+unsigned _BitInt(568) v, w;
+
+int
+foo ()
+{
+  return __builtin_clzg (x, 575);
+}
+
+int
+bar ()
+{
+  return __builtin_ctzg (x, 575);
+}
+
+int
+baz ()
+{
+  return __builtin_clrsbg (z);
+}
+
+int
+qux ()
+{
+  return __builtin_ffsg (z);
+}
+
+int
+corge ()
+{
+  return __builtin_parityg (x);
+}
+
+int
+garply ()
+{
+  return __builtin_popcountg (x);
+}
+
+void
+fred ()
+{
+  v = __builtin_bswapg (v);
+}
+
+void
+grault ()
+{
+  w = __builtin_bswapg (v);
+}
+
+void
+waldo ()
+{
+  x = __builtin_bitreverseg (x);
+}
+
+void
+xyzzy ()
+{
+  y = __builtin_bitreverseg (x);
+}