sched-deps: release reg_last lists outside reg_last_in_use

reg_last_in_use does not record every reg_last entry that contains a list.

The debug-insn path adds uses and then clears reg_pending_uses before the
normal update of reg_last_in_use.  The control-use path also adds lists
without updating reg_last_in_use.  free_deps only walks reg_last_in_use, so
these INSN_LIST nodes are not returned to the recycler.

Add a teardown-only reg_last_dirty regset for these entries.  Merge it into
reg_last_in_use in free_deps, then release all lists in the existing loop.
Do not use reg_last_dirty for dependence generation.  Adding these entries
to reg_last_in_use during analysis would make barrier handling create new
anti-dependences for debug uses and control dependences for control uses.

Add a selftest that puts a use list and a control-use list in a dirty-only
entry.  It calls free_deps and verifies that the INSN_LIST recycler returns
both nodes.  Ignoring the dirty-only entry makes this selftest fail.

Bootstrapped and regtested on aarch64-none-linux-gnu and x86_64-linux.

gcc/ChangeLog:

	* sched-int.h (struct deps_desc): Add reg_last_dirty.
	* sched-deps.cc: Include selftest.h.
	(sched_analyze_insn): Record debug-insn uses and control uses in
	reg_last_dirty.
	(init_deps): Initialize reg_last_dirty.
	(free_deps): Merge reg_last_dirty into reg_last_in_use for teardown,
	then clear it.
	(selftest::test_dirty_reg_last_release): New.
	(selftest::sched_deps_cc_tests): New.  Provide an empty definition
	when INSN_SCHEDULING is not defined.
	* selftest-run-tests.cc (selftest::run_tests): Call
	sched_deps_cc_tests.
	* selftest.h (selftest::sched_deps_cc_tests): Declare.

Signed-off-by: Kyrylo Tkachov <ktkachov@nvidia.com>

diff --git a/gcc/sched-deps.cc b/gcc/sched-deps.cc
index 23f3545b54c9dd657782b12dedb31c8637b25458..3a170d87b7e13d5179c896bb1ea97c8e5e835a28 100644 (file)
--- a/gcc/sched-deps.cc
+++ b/gcc/sched-deps.cc
@@ -38,6 +38,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "sched-int.h"
 #include "cselib.h"
 #include "function-abi.h"
+#include "selftest.h"
 
 #ifdef INSN_SCHEDULING
 
@@ -3115,6 +3116,8 @@ sched_analyze_insn (class deps_desc *deps, rtx x, rtx_insn *insn)
          if (!deps->readonly)
            reg_last->uses = alloc_INSN_LIST (insn, reg_last->uses);
        }
+      if (!deps->readonly)
+       IOR_REG_SET (&deps->reg_last_dirty, reg_pending_uses);
       CLEAR_REG_SET (reg_pending_uses);
 
       /* Quite often, a debug insn will refer to stuff in the
@@ -3306,6 +3309,7 @@ sched_analyze_insn (class deps_desc *deps, rtx x, rtx_insn *insn)
              reg_last->control_uses
                = alloc_INSN_LIST (insn, reg_last->control_uses);
            }
+         IOR_REG_SET (&deps->reg_last_dirty, reg_pending_control_uses);
        }
     }
 
@@ -3941,6 +3945,7 @@ init_deps (class deps_desc *deps, bool lazy_reg_last)
   else
     deps->reg_last = XCNEWVEC (struct deps_reg, max_reg);
   INIT_REG_SET (&deps->reg_last_in_use);
+  INIT_REG_SET (&deps->reg_last_dirty);
 
   deps->pending_read_insns = 0;
   deps->pending_read_mems = 0;
@@ -3999,6 +4004,11 @@ free_deps (class deps_desc *deps)
   free_EXPR_LIST_list (&deps->pending_write_mems);
   free_INSN_LIST_list (&deps->last_pending_memory_flush);
 
+  /* Teardown only: fold the entries recorded solely in reg_last_dirty into the
+     live set, so that one loop releases everything.  free_deps creates no
+     dependences, so this merge cannot add one.  */
+  IOR_REG_SET (&deps->reg_last_in_use, &deps->reg_last_dirty);
+
   /* Without the EXECUTE_IF_SET, this loop is executed max_reg * nr_regions
      times.  For a testcase with 42000 regs and 8000 small basic blocks,
      this loop accounted for nearly 60% (84 sec) of the total -O2 runtime.  */
@@ -4017,6 +4027,7 @@ free_deps (class deps_desc *deps)
        free_INSN_LIST_list (&reg_last->clobbers);
     }
   CLEAR_REG_SET (&deps->reg_last_in_use);
+  CLEAR_REG_SET (&deps->reg_last_dirty);
 
   /* As we initialize reg_last lazily, it is possible that we didn't allocate
      it at all.  */
@@ -5023,4 +5034,74 @@ find_modifiable_mems (rtx_insn *head, rtx_insn *tail)
             success_in_block);
 }
 
+#if CHECKING_P
+
+namespace selftest {
+
+/* Verify that free_deps releases entries recorded only in reg_last_dirty.  */
+
+static void
+test_dirty_reg_last_release ()
+{
+  bitmap_obstack test_obstack;
+  bitmap_obstack_initialize (&test_obstack);
+
+  deps_desc deps = {};
+  deps.max_reg = 2;
+  deps.reg_last = XCNEWVEC (deps_reg, deps.max_reg);
+  bitmap_initialize (&deps.reg_last_in_use, &test_obstack);
+  bitmap_initialize (&deps.reg_last_dirty, &test_obstack);
+
+  rtx_insn_list *uses = alloc_INSN_LIST (NULL_RTX, NULL_RTX);
+  rtx_insn_list *control_uses = alloc_INSN_LIST (NULL_RTX, NULL_RTX);
+  deps.reg_last[1].uses = uses;
+  deps.reg_last[1].control_uses = control_uses;
+  SET_REGNO_REG_SET (&deps.reg_last_dirty, 1);
+
+  common_sched_info_def sched_info = {};
+  sched_info.sched_pass_id = SCHED_RGN_PASS;
+  common_sched_info_def *saved_common_sched_info = common_sched_info;
+  common_sched_info = &sched_info;
+  free_deps (&deps);
+  common_sched_info = saved_common_sched_info;
+
+  ASSERT_EQ (0, deps.max_reg);
+  ASSERT_EQ (NULL, deps.reg_last);
+
+  rtx_insn_list *first = alloc_INSN_LIST (NULL_RTX, NULL_RTX);
+  rtx_insn_list *second = alloc_INSN_LIST (NULL_RTX, NULL_RTX);
+  ASSERT_TRUE ((first == control_uses && second == uses)
+              || (first == uses && second == control_uses));
+  free_INSN_LIST_list (&first);
+  free_INSN_LIST_list (&second);
+
+  sched_deps_finish ();
+  bitmap_obstack_release (&test_obstack);
+}
+
+/* Run the sched-deps.cc selftests.  */
+
+void
+sched_deps_cc_tests ()
+{
+  test_dirty_reg_last_release ();
+}
+
+} // namespace selftest
+
+#endif
+
 #endif /* INSN_SCHEDULING */
+
+#if CHECKING_P && !defined (INSN_SCHEDULING)
+
+namespace selftest {
+
+void
+sched_deps_cc_tests ()
+{
+}
+
+} // namespace selftest
+
+#endif

diff --git a/gcc/sched-int.h b/gcc/sched-int.h
index 4e7553329df9d8319888af2efead20103fc3cf1b..0c7c05abcf27c743c82c44391d8952e69c9cd1e3 100644 (file)
--- a/gcc/sched-int.h
+++ b/gcc/sched-int.h
@@ -567,6 +567,12 @@ public:
      in reg_last[N].{uses,sets,clobbers}.  */
   regset_head reg_last_in_use;
 
+  /* Element N is set for each register whose reg_last[N] was written on a
+     path that does not record it in reg_last_in_use, namely debug insn uses
+     and control uses.  Used only to release those lists in free_deps; it
+     takes no part in dependence generation.  */
+  regset_head reg_last_dirty;
+
   /* Shows the last value of reg_pending_barrier associated with the insn.  */
   enum reg_pending_barrier_mode last_reg_pending_barrier;
 

diff --git a/gcc/selftest-run-tests.cc b/gcc/selftest-run-tests.cc
index e39a94f8688aaf96d4bcdeced1b0afb02a4aa9cf..1d21b8751b4a75b4afb64f596c0f9c850827f35b 100644 (file)
--- a/gcc/selftest-run-tests.cc
+++ b/gcc/selftest-run-tests.cc
@@ -114,6 +114,7 @@ selftest::run_tests ()
 
   /* This one relies on most of the above.  */
   function_tests_cc_tests ();
+  sched_deps_cc_tests ();
 
   /* Run any target-specific selftests.  */
   if (targetm.run_target_selftests)

diff --git a/gcc/selftest.h b/gcc/selftest.h
index 8891d0b7b6f79a4b3f06a8810cc0976fe47355ff..f2e2b49fc20d21cbaa40baabf5eb742b0715dbdf 100644 (file)
--- a/gcc/selftest.h
+++ b/gcc/selftest.h
@@ -255,6 +255,7 @@ extern void read_rtl_function_cc_tests ();
 extern void relation_tests ();
 extern void rtl_tests_cc_tests ();
 extern void sbitmap_cc_tests ();
+extern void sched_deps_cc_tests ();
 extern void selftest_cc_tests ();
 extern void simple_diagnostic_path_cc_tests ();
 extern void simplify_rtx_cc_tests ();