1 /* Passes for transactional memory support.
2 Copyright (C) 2008-2015 Free Software Foundation, Inc.
3 Contributed by Richard Henderson <rth@redhat.com>
4 and Aldy Hernandez <aldyh@redhat.com>.
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
31 #include "tree-pass.h"
35 #include "gimple-pretty-print.h"
36 #include "diagnostic-core.h"
38 #include "fold-const.h"
39 #include "internal-fn.h"
43 #include "gimple-iterator.h"
44 #include "gimplify-me.h"
45 #include "gimple-walk.h"
47 #include "tree-into-ssa.h"
48 #include "tree-inline.h"
51 #include "trans-mem.h"
53 #include "langhooks.h"
55 #include "tree-ssa-address.h"
58 #define A_RUNINSTRUMENTEDCODE 0x0001
59 #define A_RUNUNINSTRUMENTEDCODE 0x0002
60 #define A_SAVELIVEVARIABLES 0x0004
61 #define A_RESTORELIVEVARIABLES 0x0008
62 #define A_ABORTTRANSACTION 0x0010
64 #define AR_USERABORT 0x0001
65 #define AR_USERRETRY 0x0002
66 #define AR_TMCONFLICT 0x0004
67 #define AR_EXCEPTIONBLOCKABORT 0x0008
68 #define AR_OUTERABORT 0x0010
70 #define MODE_SERIALIRREVOCABLE 0x0000
73 /* The representation of a transaction changes several times during the
74 lowering process. In the beginning, in the front-end we have the
75 GENERIC tree TRANSACTION_EXPR. For example,
83 During initial gimplification (gimplify.c) the TRANSACTION_EXPR node is
84 trivially replaced with a GIMPLE_TRANSACTION node.
86 During pass_lower_tm, we examine the body of transactions looking
87 for aborts. Transactions that do not contain an abort may be
88 merged into an outer transaction. We also add a TRY-FINALLY node
89 to arrange for the transaction to be committed on any exit.
91 [??? Think about how this arrangement affects throw-with-commit
92 and throw-with-abort operations. In this case we want the TRY to
93 handle gotos, but not to catch any exceptions because the transaction
94 will already be closed.]
96 GIMPLE_TRANSACTION [label=NULL] {
103 __builtin___tm_abort ();
105 __builtin___tm_commit ();
109 During pass_lower_eh, we create EH regions for the transactions,
110 intermixed with the regular EH stuff. This gives us a nice persistent
111 mapping (all the way through rtl) from transactional memory operation
112 back to the transaction, which allows us to get the abnormal edges
113 correct to model transaction aborts and restarts:
115 GIMPLE_TRANSACTION [label=over]
121 __builtin___tm_abort ();
122 __builtin___tm_commit ();
125 This is the end of all_lowering_passes, and so is what is present
126 during the IPA passes, and through all of the optimization passes.
128 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all
129 functions and mark functions for cloning.
131 At the end of gimple optimization, before exiting SSA form,
132 pass_tm_edges replaces statements that perform transactional
133 memory operations with the appropriate TM builtins, and swap
134 out function calls with their transactional clones. At this
135 point we introduce the abnormal transaction restart edges and
136 complete lowering of the GIMPLE_TRANSACTION node.
138 x = __builtin___tm_start (MAY_ABORT);
140 if (x & abort_transaction)
143 t0 = __builtin___tm_load (global);
145 __builtin___tm_store (&global, t1);
147 __builtin___tm_abort ();
148 __builtin___tm_commit ();
152 static void *expand_regions (struct tm_region
*,
153 void *(*callback
)(struct tm_region
*, void *),
157 /* Return the attributes we want to examine for X, or NULL if it's not
158 something we examine. We look at function types, but allow pointers
159 to function types and function decls and peek through. */
162 get_attrs_for (const_tree x
)
167 switch (TREE_CODE (x
))
170 return TYPE_ATTRIBUTES (TREE_TYPE (x
));
177 if (TREE_CODE (x
) != POINTER_TYPE
)
183 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
189 return TYPE_ATTRIBUTES (x
);
193 /* Return true if X has been marked TM_PURE. */
196 is_tm_pure (const_tree x
)
200 switch (TREE_CODE (x
))
211 if (TREE_CODE (x
) != POINTER_TYPE
)
217 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
222 flags
= flags_from_decl_or_type (x
);
223 return (flags
& ECF_TM_PURE
) != 0;
226 /* Return true if X has been marked TM_IRREVOCABLE. */
229 is_tm_irrevocable (tree x
)
231 tree attrs
= get_attrs_for (x
);
233 if (attrs
&& lookup_attribute ("transaction_unsafe", attrs
))
236 /* A call to the irrevocable builtin is by definition,
238 if (TREE_CODE (x
) == ADDR_EXPR
)
239 x
= TREE_OPERAND (x
, 0);
240 if (TREE_CODE (x
) == FUNCTION_DECL
241 && DECL_BUILT_IN_CLASS (x
) == BUILT_IN_NORMAL
242 && DECL_FUNCTION_CODE (x
) == BUILT_IN_TM_IRREVOCABLE
)
248 /* Return true if X has been marked TM_SAFE. */
251 is_tm_safe (const_tree x
)
255 tree attrs
= get_attrs_for (x
);
258 if (lookup_attribute ("transaction_safe", attrs
))
260 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
267 /* Return true if CALL is const, or tm_pure. */
270 is_tm_pure_call (gimple
*call
)
272 tree fn
= gimple_call_fn (call
);
274 if (TREE_CODE (fn
) == ADDR_EXPR
)
276 fn
= TREE_OPERAND (fn
, 0);
277 gcc_assert (TREE_CODE (fn
) == FUNCTION_DECL
);
282 return is_tm_pure (fn
);
285 /* Return true if X has been marked TM_CALLABLE. */
288 is_tm_callable (tree x
)
290 tree attrs
= get_attrs_for (x
);
293 if (lookup_attribute ("transaction_callable", attrs
))
295 if (lookup_attribute ("transaction_safe", attrs
))
297 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
303 /* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */
306 is_tm_may_cancel_outer (tree x
)
308 tree attrs
= get_attrs_for (x
);
310 return lookup_attribute ("transaction_may_cancel_outer", attrs
) != NULL
;
314 /* Return true for built in functions that "end" a transaction. */
317 is_tm_ending_fndecl (tree fndecl
)
319 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
320 switch (DECL_FUNCTION_CODE (fndecl
))
322 case BUILT_IN_TM_COMMIT
:
323 case BUILT_IN_TM_COMMIT_EH
:
324 case BUILT_IN_TM_ABORT
:
325 case BUILT_IN_TM_IRREVOCABLE
:
334 /* Return true if STMT is a built in function call that "ends" a
338 is_tm_ending (gimple
*stmt
)
342 if (gimple_code (stmt
) != GIMPLE_CALL
)
345 fndecl
= gimple_call_fndecl (stmt
);
346 return (fndecl
!= NULL_TREE
347 && is_tm_ending_fndecl (fndecl
));
350 /* Return true if STMT is a TM load. */
353 is_tm_load (gimple
*stmt
)
357 if (gimple_code (stmt
) != GIMPLE_CALL
)
360 fndecl
= gimple_call_fndecl (stmt
);
361 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
362 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl
)));
365 /* Same as above, but for simple TM loads, that is, not the
366 after-write, after-read, etc optimized variants. */
369 is_tm_simple_load (gimple
*stmt
)
373 if (gimple_code (stmt
) != GIMPLE_CALL
)
376 fndecl
= gimple_call_fndecl (stmt
);
377 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
379 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
380 return (fcode
== BUILT_IN_TM_LOAD_1
381 || fcode
== BUILT_IN_TM_LOAD_2
382 || fcode
== BUILT_IN_TM_LOAD_4
383 || fcode
== BUILT_IN_TM_LOAD_8
384 || fcode
== BUILT_IN_TM_LOAD_FLOAT
385 || fcode
== BUILT_IN_TM_LOAD_DOUBLE
386 || fcode
== BUILT_IN_TM_LOAD_LDOUBLE
387 || fcode
== BUILT_IN_TM_LOAD_M64
388 || fcode
== BUILT_IN_TM_LOAD_M128
389 || fcode
== BUILT_IN_TM_LOAD_M256
);
394 /* Return true if STMT is a TM store. */
397 is_tm_store (gimple
*stmt
)
401 if (gimple_code (stmt
) != GIMPLE_CALL
)
404 fndecl
= gimple_call_fndecl (stmt
);
405 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
406 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl
)));
409 /* Same as above, but for simple TM stores, that is, not the
410 after-write, after-read, etc optimized variants. */
413 is_tm_simple_store (gimple
*stmt
)
417 if (gimple_code (stmt
) != GIMPLE_CALL
)
420 fndecl
= gimple_call_fndecl (stmt
);
421 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
423 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
424 return (fcode
== BUILT_IN_TM_STORE_1
425 || fcode
== BUILT_IN_TM_STORE_2
426 || fcode
== BUILT_IN_TM_STORE_4
427 || fcode
== BUILT_IN_TM_STORE_8
428 || fcode
== BUILT_IN_TM_STORE_FLOAT
429 || fcode
== BUILT_IN_TM_STORE_DOUBLE
430 || fcode
== BUILT_IN_TM_STORE_LDOUBLE
431 || fcode
== BUILT_IN_TM_STORE_M64
432 || fcode
== BUILT_IN_TM_STORE_M128
433 || fcode
== BUILT_IN_TM_STORE_M256
);
438 /* Return true if FNDECL is BUILT_IN_TM_ABORT. */
441 is_tm_abort (tree fndecl
)
444 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
445 && DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_TM_ABORT
);
448 /* Build a GENERIC tree for a user abort. This is called by front ends
449 while transforming the __tm_abort statement. */
452 build_tm_abort_call (location_t loc
, bool is_outer
)
454 return build_call_expr_loc (loc
, builtin_decl_explicit (BUILT_IN_TM_ABORT
), 1,
455 build_int_cst (integer_type_node
,
457 | (is_outer
? AR_OUTERABORT
: 0)));
460 /* Map for aribtrary function replacement under TM, as created
461 by the tm_wrap attribute. */
463 struct tm_wrapper_hasher
: ggc_cache_ptr_hash
<tree_map
>
465 static inline hashval_t
hash (tree_map
*m
) { return m
->hash
; }
467 equal (tree_map
*a
, tree_map
*b
)
469 return a
->base
.from
== b
->base
.from
;
473 keep_cache_entry (tree_map
*&m
)
475 return ggc_marked_p (m
->base
.from
);
479 static GTY((cache
)) hash_table
<tm_wrapper_hasher
> *tm_wrap_map
;
482 record_tm_replacement (tree from
, tree to
)
484 struct tree_map
**slot
, *h
;
486 /* Do not inline wrapper functions that will get replaced in the TM
489 Suppose you have foo() that will get replaced into tmfoo(). Make
490 sure the inliner doesn't try to outsmart us and inline foo()
491 before we get a chance to do the TM replacement. */
492 DECL_UNINLINABLE (from
) = 1;
494 if (tm_wrap_map
== NULL
)
495 tm_wrap_map
= hash_table
<tm_wrapper_hasher
>::create_ggc (32);
497 h
= ggc_alloc
<tree_map
> ();
498 h
->hash
= htab_hash_pointer (from
);
502 slot
= tm_wrap_map
->find_slot_with_hash (h
, h
->hash
, INSERT
);
506 /* Return a TM-aware replacement function for DECL. */
509 find_tm_replacement_function (tree fndecl
)
513 struct tree_map
*h
, in
;
515 in
.base
.from
= fndecl
;
516 in
.hash
= htab_hash_pointer (fndecl
);
517 h
= tm_wrap_map
->find_with_hash (&in
, in
.hash
);
522 /* ??? We may well want TM versions of most of the common <string.h>
523 functions. For now, we've already these two defined. */
524 /* Adjust expand_call_tm() attributes as necessary for the cases
526 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
527 switch (DECL_FUNCTION_CODE (fndecl
))
529 case BUILT_IN_MEMCPY
:
530 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY
);
531 case BUILT_IN_MEMMOVE
:
532 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
);
533 case BUILT_IN_MEMSET
:
534 return builtin_decl_explicit (BUILT_IN_TM_MEMSET
);
542 /* When appropriate, record TM replacement for memory allocation functions.
544 FROM is the FNDECL to wrap. */
546 tm_malloc_replacement (tree from
)
551 if (TREE_CODE (from
) != FUNCTION_DECL
)
554 /* If we have a previous replacement, the user must be explicitly
555 wrapping malloc/calloc/free. They better know what they're
557 if (find_tm_replacement_function (from
))
560 str
= IDENTIFIER_POINTER (DECL_NAME (from
));
562 if (!strcmp (str
, "malloc"))
563 to
= builtin_decl_explicit (BUILT_IN_TM_MALLOC
);
564 else if (!strcmp (str
, "calloc"))
565 to
= builtin_decl_explicit (BUILT_IN_TM_CALLOC
);
566 else if (!strcmp (str
, "free"))
567 to
= builtin_decl_explicit (BUILT_IN_TM_FREE
);
571 TREE_NOTHROW (to
) = 0;
573 record_tm_replacement (from
, to
);
576 /* Diagnostics for tm_safe functions/regions. Called by the front end
577 once we've lowered the function to high-gimple. */
579 /* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq.
580 Process exactly one statement. WI->INFO is set to non-null when in
581 the context of a tm_safe function, and null for a __transaction block. */
583 #define DIAG_TM_OUTER 1
584 #define DIAG_TM_SAFE 2
585 #define DIAG_TM_RELAXED 4
589 unsigned int summary_flags
: 8;
590 unsigned int block_flags
: 8;
591 unsigned int func_flags
: 8;
592 unsigned int saw_volatile
: 1;
596 /* Return true if T is a volatile lvalue of some kind. */
599 volatile_lvalue_p (tree t
)
601 return ((SSA_VAR_P (t
) || REFERENCE_CLASS_P (t
))
602 && TREE_THIS_VOLATILE (TREE_TYPE (t
)));
605 /* Tree callback function for diagnose_tm pass. */
608 diagnose_tm_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
610 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
611 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
614 *walk_subtrees
= false;
615 else if (volatile_lvalue_p (*tp
)
619 if (d
->block_flags
& DIAG_TM_SAFE
)
620 error_at (gimple_location (d
->stmt
),
621 "invalid use of volatile lvalue inside transaction");
622 else if (d
->func_flags
& DIAG_TM_SAFE
)
623 error_at (gimple_location (d
->stmt
),
624 "invalid use of volatile lvalue inside %<transaction_safe%>"
632 is_tm_safe_or_pure (const_tree x
)
634 return is_tm_safe (x
) || is_tm_pure (x
);
638 diagnose_tm_1 (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
639 struct walk_stmt_info
*wi
)
641 gimple
*stmt
= gsi_stmt (*gsi
);
642 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
644 /* Save stmt for use in leaf analysis. */
647 switch (gimple_code (stmt
))
651 tree fn
= gimple_call_fn (stmt
);
653 if ((d
->summary_flags
& DIAG_TM_OUTER
) == 0
654 && is_tm_may_cancel_outer (fn
))
655 error_at (gimple_location (stmt
),
656 "%<transaction_may_cancel_outer%> function call not within"
657 " outer transaction or %<transaction_may_cancel_outer%>");
659 if (d
->summary_flags
& DIAG_TM_SAFE
)
661 bool is_safe
, direct_call_p
;
664 if (TREE_CODE (fn
) == ADDR_EXPR
665 && TREE_CODE (TREE_OPERAND (fn
, 0)) == FUNCTION_DECL
)
667 direct_call_p
= true;
668 replacement
= TREE_OPERAND (fn
, 0);
669 replacement
= find_tm_replacement_function (replacement
);
675 direct_call_p
= false;
676 replacement
= NULL_TREE
;
679 if (is_tm_safe_or_pure (fn
))
681 else if (is_tm_callable (fn
) || is_tm_irrevocable (fn
))
683 /* A function explicitly marked transaction_callable as
684 opposed to transaction_safe is being defined to be
685 unsafe as part of its ABI, regardless of its contents. */
688 else if (direct_call_p
)
690 if (IS_TYPE_OR_DECL_P (fn
)
691 && flags_from_decl_or_type (fn
) & ECF_TM_BUILTIN
)
693 else if (replacement
)
695 /* ??? At present we've been considering replacements
696 merely transaction_callable, and therefore might
697 enter irrevocable. The tm_wrap attribute has not
698 yet made it into the new language spec. */
703 /* ??? Diagnostics for unmarked direct calls moved into
704 the IPA pass. Section 3.2 of the spec details how
705 functions not marked should be considered "implicitly
706 safe" based on having examined the function body. */
712 /* An unmarked indirect call. Consider it unsafe even
713 though optimization may yet figure out how to inline. */
719 if (TREE_CODE (fn
) == ADDR_EXPR
)
720 fn
= TREE_OPERAND (fn
, 0);
721 if (d
->block_flags
& DIAG_TM_SAFE
)
724 error_at (gimple_location (stmt
),
725 "unsafe function call %qD within "
726 "atomic transaction", fn
);
729 if (!DECL_P (fn
) || DECL_NAME (fn
))
730 error_at (gimple_location (stmt
),
731 "unsafe function call %qE within "
732 "atomic transaction", fn
);
734 error_at (gimple_location (stmt
),
735 "unsafe indirect function call within "
736 "atomic transaction");
742 error_at (gimple_location (stmt
),
743 "unsafe function call %qD within "
744 "%<transaction_safe%> function", fn
);
747 if (!DECL_P (fn
) || DECL_NAME (fn
))
748 error_at (gimple_location (stmt
),
749 "unsafe function call %qE within "
750 "%<transaction_safe%> function", fn
);
752 error_at (gimple_location (stmt
),
753 "unsafe indirect function call within "
754 "%<transaction_safe%> function");
763 /* ??? We ought to come up with a way to add attributes to
764 asm statements, and then add "transaction_safe" to it.
765 Either that or get the language spec to resurrect __tm_waiver. */
766 if (d
->block_flags
& DIAG_TM_SAFE
)
767 error_at (gimple_location (stmt
),
768 "asm not allowed in atomic transaction");
769 else if (d
->func_flags
& DIAG_TM_SAFE
)
770 error_at (gimple_location (stmt
),
771 "asm not allowed in %<transaction_safe%> function");
774 case GIMPLE_TRANSACTION
:
776 gtransaction
*trans_stmt
= as_a
<gtransaction
*> (stmt
);
777 unsigned char inner_flags
= DIAG_TM_SAFE
;
779 if (gimple_transaction_subcode (trans_stmt
) & GTMA_IS_RELAXED
)
781 if (d
->block_flags
& DIAG_TM_SAFE
)
782 error_at (gimple_location (stmt
),
783 "relaxed transaction in atomic transaction");
784 else if (d
->func_flags
& DIAG_TM_SAFE
)
785 error_at (gimple_location (stmt
),
786 "relaxed transaction in %<transaction_safe%> function");
787 inner_flags
= DIAG_TM_RELAXED
;
789 else if (gimple_transaction_subcode (trans_stmt
) & GTMA_IS_OUTER
)
792 error_at (gimple_location (stmt
),
793 "outer transaction in transaction");
794 else if (d
->func_flags
& DIAG_TM_OUTER
)
795 error_at (gimple_location (stmt
),
796 "outer transaction in "
797 "%<transaction_may_cancel_outer%> function");
798 else if (d
->func_flags
& DIAG_TM_SAFE
)
799 error_at (gimple_location (stmt
),
800 "outer transaction in %<transaction_safe%> function");
801 inner_flags
|= DIAG_TM_OUTER
;
804 *handled_ops_p
= true;
805 if (gimple_transaction_body (trans_stmt
))
807 struct walk_stmt_info wi_inner
;
808 struct diagnose_tm d_inner
;
810 memset (&d_inner
, 0, sizeof (d_inner
));
811 d_inner
.func_flags
= d
->func_flags
;
812 d_inner
.block_flags
= d
->block_flags
| inner_flags
;
813 d_inner
.summary_flags
= d_inner
.func_flags
| d_inner
.block_flags
;
815 memset (&wi_inner
, 0, sizeof (wi_inner
));
816 wi_inner
.info
= &d_inner
;
818 walk_gimple_seq (gimple_transaction_body (trans_stmt
),
819 diagnose_tm_1
, diagnose_tm_1_op
, &wi_inner
);
832 diagnose_tm_blocks (void)
834 struct walk_stmt_info wi
;
835 struct diagnose_tm d
;
837 memset (&d
, 0, sizeof (d
));
838 if (is_tm_may_cancel_outer (current_function_decl
))
839 d
.func_flags
= DIAG_TM_OUTER
| DIAG_TM_SAFE
;
840 else if (is_tm_safe (current_function_decl
))
841 d
.func_flags
= DIAG_TM_SAFE
;
842 d
.summary_flags
= d
.func_flags
;
844 memset (&wi
, 0, sizeof (wi
));
847 walk_gimple_seq (gimple_body (current_function_decl
),
848 diagnose_tm_1
, diagnose_tm_1_op
, &wi
);
855 const pass_data pass_data_diagnose_tm_blocks
=
857 GIMPLE_PASS
, /* type */
858 "*diagnose_tm_blocks", /* name */
859 OPTGROUP_NONE
, /* optinfo_flags */
860 TV_TRANS_MEM
, /* tv_id */
861 PROP_gimple_any
, /* properties_required */
862 0, /* properties_provided */
863 0, /* properties_destroyed */
864 0, /* todo_flags_start */
865 0, /* todo_flags_finish */
868 class pass_diagnose_tm_blocks
: public gimple_opt_pass
871 pass_diagnose_tm_blocks (gcc::context
*ctxt
)
872 : gimple_opt_pass (pass_data_diagnose_tm_blocks
, ctxt
)
875 /* opt_pass methods: */
876 virtual bool gate (function
*) { return flag_tm
; }
877 virtual unsigned int execute (function
*) { return diagnose_tm_blocks (); }
879 }; // class pass_diagnose_tm_blocks
884 make_pass_diagnose_tm_blocks (gcc::context
*ctxt
)
886 return new pass_diagnose_tm_blocks (ctxt
);
889 /* Instead of instrumenting thread private memory, we save the
890 addresses in a log which we later use to save/restore the addresses
891 upon transaction start/restart.
893 The log is keyed by address, where each element contains individual
894 statements among different code paths that perform the store.
896 This log is later used to generate either plain save/restore of the
897 addresses upon transaction start/restart, or calls to the ITM_L*
900 So for something like:
902 struct large { int x[1000]; };
903 struct large lala = { 0 };
909 We can either save/restore:
912 trxn = _ITM_startTransaction ();
913 if (trxn & a_saveLiveVariables)
914 tmp_lala1 = lala.x[i];
915 else if (a & a_restoreLiveVariables)
916 lala.x[i] = tmp_lala1;
918 or use the logging functions:
921 trxn = _ITM_startTransaction ();
922 _ITM_LU4 (&lala.x[i]);
924 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as
925 far up the dominator tree to shadow all of the writes to a given
926 location (thus reducing the total number of logging calls), but not
927 so high as to be called on a path that does not perform a
930 /* One individual log entry. We may have multiple statements for the
931 same location if neither dominate each other (on different
935 /* Address to save. */
937 /* Entry block for the transaction this address occurs in. */
938 basic_block entry_block
;
939 /* Dominating statements the store occurs in. */
941 /* Initially, while we are building the log, we place a nonzero
942 value here to mean that this address *will* be saved with a
943 save/restore sequence. Later, when generating the save sequence
944 we place the SSA temp generated here. */
949 /* Log entry hashtable helpers. */
951 struct log_entry_hasher
: pointer_hash
<tm_log_entry
>
953 static inline hashval_t
hash (const tm_log_entry
*);
954 static inline bool equal (const tm_log_entry
*, const tm_log_entry
*);
955 static inline void remove (tm_log_entry
*);
958 /* Htab support. Return hash value for a `tm_log_entry'. */
960 log_entry_hasher::hash (const tm_log_entry
*log
)
962 return iterative_hash_expr (log
->addr
, 0);
965 /* Htab support. Return true if two log entries are the same. */
967 log_entry_hasher::equal (const tm_log_entry
*log1
, const tm_log_entry
*log2
)
971 rth: I suggest that we get rid of the component refs etc.
972 I.e. resolve the reference to base + offset.
974 We may need to actually finish a merge with mainline for this,
975 since we'd like to be presented with Richi's MEM_REF_EXPRs more
976 often than not. But in the meantime your tm_log_entry could save
977 the results of get_inner_reference.
979 See: g++.dg/tm/pr46653.C
982 /* Special case plain equality because operand_equal_p() below will
983 return FALSE if the addresses are equal but they have
984 side-effects (e.g. a volatile address). */
985 if (log1
->addr
== log2
->addr
)
988 return operand_equal_p (log1
->addr
, log2
->addr
, 0);
991 /* Htab support. Free one tm_log_entry. */
993 log_entry_hasher::remove (tm_log_entry
*lp
)
995 lp
->stmts
.release ();
1000 /* The actual log. */
1001 static hash_table
<log_entry_hasher
> *tm_log
;
1003 /* Addresses to log with a save/restore sequence. These should be in
1005 static vec
<tree
> tm_log_save_addresses
;
1007 enum thread_memory_type
1011 mem_transaction_local
,
1015 struct tm_new_mem_map
1017 /* SSA_NAME being dereferenced. */
1019 enum thread_memory_type local_new_memory
;
1022 /* Hashtable helpers. */
1024 struct tm_mem_map_hasher
: free_ptr_hash
<tm_new_mem_map
>
1026 static inline hashval_t
hash (const tm_new_mem_map
*);
1027 static inline bool equal (const tm_new_mem_map
*, const tm_new_mem_map
*);
1031 tm_mem_map_hasher::hash (const tm_new_mem_map
*v
)
1033 return (intptr_t)v
->val
>> 4;
1037 tm_mem_map_hasher::equal (const tm_new_mem_map
*v
, const tm_new_mem_map
*c
)
1039 return v
->val
== c
->val
;
1042 /* Map for an SSA_NAME originally pointing to a non aliased new piece
1043 of memory (malloc, alloc, etc). */
1044 static hash_table
<tm_mem_map_hasher
> *tm_new_mem_hash
;
1046 /* Initialize logging data structures. */
1050 tm_log
= new hash_table
<log_entry_hasher
> (10);
1051 tm_new_mem_hash
= new hash_table
<tm_mem_map_hasher
> (5);
1052 tm_log_save_addresses
.create (5);
1055 /* Free logging data structures. */
1057 tm_log_delete (void)
1061 delete tm_new_mem_hash
;
1062 tm_new_mem_hash
= NULL
;
1063 tm_log_save_addresses
.release ();
1066 /* Return true if MEM is a transaction invariant memory for the TM
1067 region starting at REGION_ENTRY_BLOCK. */
1069 transaction_invariant_address_p (const_tree mem
, basic_block region_entry_block
)
1071 if ((TREE_CODE (mem
) == INDIRECT_REF
|| TREE_CODE (mem
) == MEM_REF
)
1072 && TREE_CODE (TREE_OPERAND (mem
, 0)) == SSA_NAME
)
1076 def_bb
= gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem
, 0)));
1077 return def_bb
!= region_entry_block
1078 && dominated_by_p (CDI_DOMINATORS
, region_entry_block
, def_bb
);
1081 mem
= strip_invariant_refs (mem
);
1082 return mem
&& (CONSTANT_CLASS_P (mem
) || decl_address_invariant_p (mem
));
1085 /* Given an address ADDR in STMT, find it in the memory log or add it,
1086 making sure to keep only the addresses highest in the dominator
1089 ENTRY_BLOCK is the entry_block for the transaction.
1091 If we find the address in the log, make sure it's either the same
1092 address, or an equivalent one that dominates ADDR.
1094 If we find the address, but neither ADDR dominates the found
1095 address, nor the found one dominates ADDR, we're on different
1096 execution paths. Add it.
1098 If known, ENTRY_BLOCK is the entry block for the region, otherwise
1101 tm_log_add (basic_block entry_block
, tree addr
, gimple
*stmt
)
1103 tm_log_entry
**slot
;
1104 struct tm_log_entry l
, *lp
;
1107 slot
= tm_log
->find_slot (&l
, INSERT
);
1110 tree type
= TREE_TYPE (addr
);
1112 lp
= XNEW (struct tm_log_entry
);
1116 /* Small invariant addresses can be handled as save/restores. */
1118 && transaction_invariant_address_p (lp
->addr
, entry_block
)
1119 && TYPE_SIZE_UNIT (type
) != NULL
1120 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
))
1121 && ((HOST_WIDE_INT
) tree_to_uhwi (TYPE_SIZE_UNIT (type
))
1122 < PARAM_VALUE (PARAM_TM_MAX_AGGREGATE_SIZE
))
1123 /* We must be able to copy this type normally. I.e., no
1124 special constructors and the like. */
1125 && !TREE_ADDRESSABLE (type
))
1127 lp
->save_var
= create_tmp_reg (TREE_TYPE (lp
->addr
), "tm_save");
1128 lp
->stmts
.create (0);
1129 lp
->entry_block
= entry_block
;
1130 /* Save addresses separately in dominator order so we don't
1131 get confused by overlapping addresses in the save/restore
1133 tm_log_save_addresses
.safe_push (lp
->addr
);
1137 /* Use the logging functions. */
1138 lp
->stmts
.create (5);
1139 lp
->stmts
.quick_push (stmt
);
1140 lp
->save_var
= NULL
;
1150 /* If we're generating a save/restore sequence, we don't care
1151 about statements. */
1155 for (i
= 0; lp
->stmts
.iterate (i
, &oldstmt
); ++i
)
1157 if (stmt
== oldstmt
)
1159 /* We already have a store to the same address, higher up the
1160 dominator tree. Nothing to do. */
1161 if (dominated_by_p (CDI_DOMINATORS
,
1162 gimple_bb (stmt
), gimple_bb (oldstmt
)))
1164 /* We should be processing blocks in dominator tree order. */
1165 gcc_assert (!dominated_by_p (CDI_DOMINATORS
,
1166 gimple_bb (oldstmt
), gimple_bb (stmt
)));
1168 /* Store is on a different code path. */
1169 lp
->stmts
.safe_push (stmt
);
1173 /* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME
1174 result, insert the new statements before GSI. */
1177 gimplify_addr (gimple_stmt_iterator
*gsi
, tree x
)
1179 if (TREE_CODE (x
) == TARGET_MEM_REF
)
1180 x
= tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x
)), x
);
1182 x
= build_fold_addr_expr (x
);
1183 return force_gimple_operand_gsi (gsi
, x
, true, NULL
, true, GSI_SAME_STMT
);
1186 /* Instrument one address with the logging functions.
1187 ADDR is the address to save.
1188 STMT is the statement before which to place it. */
1190 tm_log_emit_stmt (tree addr
, gimple
*stmt
)
1192 tree type
= TREE_TYPE (addr
);
1193 tree size
= TYPE_SIZE_UNIT (type
);
1194 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
1196 enum built_in_function code
= BUILT_IN_TM_LOG
;
1198 if (type
== float_type_node
)
1199 code
= BUILT_IN_TM_LOG_FLOAT
;
1200 else if (type
== double_type_node
)
1201 code
= BUILT_IN_TM_LOG_DOUBLE
;
1202 else if (type
== long_double_type_node
)
1203 code
= BUILT_IN_TM_LOG_LDOUBLE
;
1204 else if (tree_fits_uhwi_p (size
))
1206 unsigned int n
= tree_to_uhwi (size
);
1210 code
= BUILT_IN_TM_LOG_1
;
1213 code
= BUILT_IN_TM_LOG_2
;
1216 code
= BUILT_IN_TM_LOG_4
;
1219 code
= BUILT_IN_TM_LOG_8
;
1222 code
= BUILT_IN_TM_LOG
;
1223 if (TREE_CODE (type
) == VECTOR_TYPE
)
1225 if (n
== 8 && builtin_decl_explicit (BUILT_IN_TM_LOG_M64
))
1226 code
= BUILT_IN_TM_LOG_M64
;
1227 else if (n
== 16 && builtin_decl_explicit (BUILT_IN_TM_LOG_M128
))
1228 code
= BUILT_IN_TM_LOG_M128
;
1229 else if (n
== 32 && builtin_decl_explicit (BUILT_IN_TM_LOG_M256
))
1230 code
= BUILT_IN_TM_LOG_M256
;
1236 addr
= gimplify_addr (&gsi
, addr
);
1237 if (code
== BUILT_IN_TM_LOG
)
1238 log
= gimple_build_call (builtin_decl_explicit (code
), 2, addr
, size
);
1240 log
= gimple_build_call (builtin_decl_explicit (code
), 1, addr
);
1241 gsi_insert_before (&gsi
, log
, GSI_SAME_STMT
);
1244 /* Go through the log and instrument address that must be instrumented
1245 with the logging functions. Leave the save/restore addresses for
1250 hash_table
<log_entry_hasher
>::iterator hi
;
1251 struct tm_log_entry
*lp
;
1253 FOR_EACH_HASH_TABLE_ELEMENT (*tm_log
, lp
, tm_log_entry_t
, hi
)
1260 fprintf (dump_file
, "TM thread private mem logging: ");
1261 print_generic_expr (dump_file
, lp
->addr
, 0);
1262 fprintf (dump_file
, "\n");
1268 fprintf (dump_file
, "DUMPING to variable\n");
1274 fprintf (dump_file
, "DUMPING with logging functions\n");
1275 for (i
= 0; lp
->stmts
.iterate (i
, &stmt
); ++i
)
1276 tm_log_emit_stmt (lp
->addr
, stmt
);
1281 /* Emit the save sequence for the corresponding addresses in the log.
1282 ENTRY_BLOCK is the entry block for the transaction.
1283 BB is the basic block to insert the code in. */
1285 tm_log_emit_saves (basic_block entry_block
, basic_block bb
)
1288 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1290 struct tm_log_entry l
, *lp
;
1292 for (i
= 0; i
< tm_log_save_addresses
.length (); ++i
)
1294 l
.addr
= tm_log_save_addresses
[i
];
1295 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1296 gcc_assert (lp
->save_var
!= NULL
);
1298 /* We only care about variables in the current transaction. */
1299 if (lp
->entry_block
!= entry_block
)
1302 stmt
= gimple_build_assign (lp
->save_var
, unshare_expr (lp
->addr
));
1304 /* Make sure we can create an SSA_NAME for this type. For
1305 instance, aggregates aren't allowed, in which case the system
1306 will create a VOP for us and everything will just work. */
1307 if (is_gimple_reg_type (TREE_TYPE (lp
->save_var
)))
1309 lp
->save_var
= make_ssa_name (lp
->save_var
, stmt
);
1310 gimple_assign_set_lhs (stmt
, lp
->save_var
);
1313 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
1317 /* Emit the restore sequence for the corresponding addresses in the log.
1318 ENTRY_BLOCK is the entry block for the transaction.
1319 BB is the basic block to insert the code in. */
1321 tm_log_emit_restores (basic_block entry_block
, basic_block bb
)
1324 struct tm_log_entry l
, *lp
;
1325 gimple_stmt_iterator gsi
;
1328 for (i
= tm_log_save_addresses
.length () - 1; i
>= 0; i
--)
1330 l
.addr
= tm_log_save_addresses
[i
];
1331 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1332 gcc_assert (lp
->save_var
!= NULL
);
1334 /* We only care about variables in the current transaction. */
1335 if (lp
->entry_block
!= entry_block
)
1338 /* Restores are in LIFO order from the saves in case we have
1340 gsi
= gsi_start_bb (bb
);
1342 stmt
= gimple_build_assign (unshare_expr (lp
->addr
), lp
->save_var
);
1343 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1348 static tree
lower_sequence_tm (gimple_stmt_iterator
*, bool *,
1349 struct walk_stmt_info
*);
1350 static tree
lower_sequence_no_tm (gimple_stmt_iterator
*, bool *,
1351 struct walk_stmt_info
*);
1353 /* Evaluate an address X being dereferenced and determine if it
1354 originally points to a non aliased new chunk of memory (malloc,
1357 Return MEM_THREAD_LOCAL if it points to a thread-local address.
1358 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address.
1359 Return MEM_NON_LOCAL otherwise.
1361 ENTRY_BLOCK is the entry block to the transaction containing the
1362 dereference of X. */
1363 static enum thread_memory_type
1364 thread_private_new_memory (basic_block entry_block
, tree x
)
1366 gimple
*stmt
= NULL
;
1367 enum tree_code code
;
1368 tm_new_mem_map
**slot
;
1369 tm_new_mem_map elt
, *elt_p
;
1371 enum thread_memory_type retval
= mem_transaction_local
;
1374 || TREE_CODE (x
) != SSA_NAME
1375 /* Possible uninitialized use, or a function argument. In
1376 either case, we don't care. */
1377 || SSA_NAME_IS_DEFAULT_DEF (x
))
1378 return mem_non_local
;
1380 /* Look in cache first. */
1382 slot
= tm_new_mem_hash
->find_slot (&elt
, INSERT
);
1385 return elt_p
->local_new_memory
;
1387 /* Optimistically assume the memory is transaction local during
1388 processing. This catches recursion into this variable. */
1389 *slot
= elt_p
= XNEW (tm_new_mem_map
);
1391 elt_p
->local_new_memory
= mem_transaction_local
;
1393 /* Search DEF chain to find the original definition of this address. */
1396 if (ptr_deref_may_alias_global_p (x
))
1398 /* Address escapes. This is not thread-private. */
1399 retval
= mem_non_local
;
1400 goto new_memory_ret
;
1403 stmt
= SSA_NAME_DEF_STMT (x
);
1405 /* If the malloc call is outside the transaction, this is
1407 if (retval
!= mem_thread_local
1408 && !dominated_by_p (CDI_DOMINATORS
, gimple_bb (stmt
), entry_block
))
1409 retval
= mem_thread_local
;
1411 if (is_gimple_assign (stmt
))
1413 code
= gimple_assign_rhs_code (stmt
);
1414 /* x = foo ==> foo */
1415 if (code
== SSA_NAME
)
1416 x
= gimple_assign_rhs1 (stmt
);
1417 /* x = foo + n ==> foo */
1418 else if (code
== POINTER_PLUS_EXPR
)
1419 x
= gimple_assign_rhs1 (stmt
);
1420 /* x = (cast*) foo ==> foo */
1421 else if (code
== VIEW_CONVERT_EXPR
|| CONVERT_EXPR_CODE_P (code
))
1422 x
= gimple_assign_rhs1 (stmt
);
1423 /* x = c ? op1 : op2 == > op1 or op2 just like a PHI */
1424 else if (code
== COND_EXPR
)
1426 tree op1
= gimple_assign_rhs2 (stmt
);
1427 tree op2
= gimple_assign_rhs3 (stmt
);
1428 enum thread_memory_type mem
;
1429 retval
= thread_private_new_memory (entry_block
, op1
);
1430 if (retval
== mem_non_local
)
1431 goto new_memory_ret
;
1432 mem
= thread_private_new_memory (entry_block
, op2
);
1433 retval
= MIN (retval
, mem
);
1434 goto new_memory_ret
;
1438 retval
= mem_non_local
;
1439 goto new_memory_ret
;
1444 if (gimple_code (stmt
) == GIMPLE_PHI
)
1447 enum thread_memory_type mem
;
1448 tree phi_result
= gimple_phi_result (stmt
);
1450 /* If any of the ancestors are non-local, we are sure to
1451 be non-local. Otherwise we can avoid doing anything
1452 and inherit what has already been generated. */
1454 for (i
= 0; i
< gimple_phi_num_args (stmt
); ++i
)
1456 tree op
= PHI_ARG_DEF (stmt
, i
);
1458 /* Exclude self-assignment. */
1459 if (phi_result
== op
)
1462 mem
= thread_private_new_memory (entry_block
, op
);
1463 if (mem
== mem_non_local
)
1466 goto new_memory_ret
;
1468 retval
= MIN (retval
, mem
);
1470 goto new_memory_ret
;
1475 while (TREE_CODE (x
) == SSA_NAME
);
1477 if (stmt
&& is_gimple_call (stmt
) && gimple_call_flags (stmt
) & ECF_MALLOC
)
1478 /* Thread-local or transaction-local. */
1481 retval
= mem_non_local
;
1484 elt_p
->local_new_memory
= retval
;
1488 /* Determine whether X has to be instrumented using a read
1491 ENTRY_BLOCK is the entry block for the region where stmt resides
1492 in. NULL if unknown.
1494 STMT is the statement in which X occurs in. It is used for thread
1495 private memory instrumentation. If no TPM instrumentation is
1496 desired, STMT should be null. */
1498 requires_barrier (basic_block entry_block
, tree x
, gimple
*stmt
)
1501 while (handled_component_p (x
))
1502 x
= TREE_OPERAND (x
, 0);
1504 switch (TREE_CODE (x
))
1509 enum thread_memory_type ret
;
1511 ret
= thread_private_new_memory (entry_block
, TREE_OPERAND (x
, 0));
1512 if (ret
== mem_non_local
)
1514 if (stmt
&& ret
== mem_thread_local
)
1515 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */
1516 tm_log_add (entry_block
, orig
, stmt
);
1518 /* Transaction-locals require nothing at all. For malloc, a
1519 transaction restart frees the memory and we reallocate.
1520 For alloca, the stack pointer gets reset by the retry and
1525 case TARGET_MEM_REF
:
1526 if (TREE_CODE (TMR_BASE (x
)) != ADDR_EXPR
)
1528 x
= TREE_OPERAND (TMR_BASE (x
), 0);
1529 if (TREE_CODE (x
) == PARM_DECL
)
1531 gcc_assert (TREE_CODE (x
) == VAR_DECL
);
1537 if (DECL_BY_REFERENCE (x
))
1539 /* ??? This value is a pointer, but aggregate_value_p has been
1540 jigged to return true which confuses needs_to_live_in_memory.
1541 This ought to be cleaned up generically.
1543 FIXME: Verify this still happens after the next mainline
1544 merge. Testcase ie g++.dg/tm/pr47554.C.
1549 if (is_global_var (x
))
1550 return !TREE_READONLY (x
);
1551 if (/* FIXME: This condition should actually go below in the
1552 tm_log_add() call, however is_call_clobbered() depends on
1553 aliasing info which is not available during
1554 gimplification. Since requires_barrier() gets called
1555 during lower_sequence_tm/gimplification, leave the call
1556 to needs_to_live_in_memory until we eliminate
1557 lower_sequence_tm altogether. */
1558 needs_to_live_in_memory (x
))
1562 /* For local memory that doesn't escape (aka thread private
1563 memory), we can either save the value at the beginning of
1564 the transaction and restore on restart, or call a tm
1565 function to dynamically save and restore on restart
1568 tm_log_add (entry_block
, orig
, stmt
);
1577 /* Mark the GIMPLE_ASSIGN statement as appropriate for being inside
1578 a transaction region. */
1581 examine_assign_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1583 gimple
*stmt
= gsi_stmt (*gsi
);
1585 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_rhs1 (stmt
), NULL
))
1586 *state
|= GTMA_HAVE_LOAD
;
1587 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_lhs (stmt
), NULL
))
1588 *state
|= GTMA_HAVE_STORE
;
1591 /* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */
1594 examine_call_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1596 gimple
*stmt
= gsi_stmt (*gsi
);
1599 if (is_tm_pure_call (stmt
))
1602 /* Check if this call is a transaction abort. */
1603 fn
= gimple_call_fndecl (stmt
);
1604 if (is_tm_abort (fn
))
1605 *state
|= GTMA_HAVE_ABORT
;
1607 /* Note that something may happen. */
1608 *state
|= GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
;
1611 /* Lower a GIMPLE_TRANSACTION statement. */
1614 lower_transaction (gimple_stmt_iterator
*gsi
, struct walk_stmt_info
*wi
)
1617 gtransaction
*stmt
= as_a
<gtransaction
*> (gsi_stmt (*gsi
));
1618 unsigned int *outer_state
= (unsigned int *) wi
->info
;
1619 unsigned int this_state
= 0;
1620 struct walk_stmt_info this_wi
;
1622 /* First, lower the body. The scanning that we do inside gives
1623 us some idea of what we're dealing with. */
1624 memset (&this_wi
, 0, sizeof (this_wi
));
1625 this_wi
.info
= (void *) &this_state
;
1626 walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt
),
1627 lower_sequence_tm
, NULL
, &this_wi
);
1629 /* If there was absolutely nothing transaction related inside the
1630 transaction, we may elide it. Likewise if this is a nested
1631 transaction and does not contain an abort. */
1633 || (!(this_state
& GTMA_HAVE_ABORT
) && outer_state
!= NULL
))
1636 *outer_state
|= this_state
;
1638 gsi_insert_seq_before (gsi
, gimple_transaction_body (stmt
),
1640 gimple_transaction_set_body (stmt
, NULL
);
1642 gsi_remove (gsi
, true);
1643 wi
->removed_stmt
= true;
1647 /* Wrap the body of the transaction in a try-finally node so that
1648 the commit call is always properly called. */
1649 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT
), 0);
1650 if (flag_exceptions
)
1653 gimple_seq n_seq
, e_seq
;
1655 n_seq
= gimple_seq_alloc_with_stmt (g
);
1658 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER
),
1659 1, integer_zero_node
);
1660 ptr
= create_tmp_var (ptr_type_node
);
1661 gimple_call_set_lhs (g
, ptr
);
1662 gimple_seq_add_stmt (&e_seq
, g
);
1664 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH
),
1666 gimple_seq_add_stmt (&e_seq
, g
);
1668 g
= gimple_build_eh_else (n_seq
, e_seq
);
1671 g
= gimple_build_try (gimple_transaction_body (stmt
),
1672 gimple_seq_alloc_with_stmt (g
), GIMPLE_TRY_FINALLY
);
1673 gsi_insert_after (gsi
, g
, GSI_CONTINUE_LINKING
);
1675 gimple_transaction_set_body (stmt
, NULL
);
1677 /* If the transaction calls abort or if this is an outer transaction,
1678 add an "over" label afterwards. */
1679 if ((this_state
& (GTMA_HAVE_ABORT
))
1680 || (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
))
1682 tree label
= create_artificial_label (UNKNOWN_LOCATION
);
1683 gimple_transaction_set_label (stmt
, label
);
1684 gsi_insert_after (gsi
, gimple_build_label (label
), GSI_CONTINUE_LINKING
);
1687 /* Record the set of operations found for use later. */
1688 this_state
|= gimple_transaction_subcode (stmt
) & GTMA_DECLARATION_MASK
;
1689 gimple_transaction_set_subcode (stmt
, this_state
);
1692 /* Iterate through the statements in the sequence, lowering them all
1693 as appropriate for being in a transaction. */
1696 lower_sequence_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1697 struct walk_stmt_info
*wi
)
1699 unsigned int *state
= (unsigned int *) wi
->info
;
1700 gimple
*stmt
= gsi_stmt (*gsi
);
1702 *handled_ops_p
= true;
1703 switch (gimple_code (stmt
))
1706 /* Only memory reads/writes need to be instrumented. */
1707 if (gimple_assign_single_p (stmt
))
1708 examine_assign_tm (state
, gsi
);
1712 examine_call_tm (state
, gsi
);
1716 *state
|= GTMA_MAY_ENTER_IRREVOCABLE
;
1719 case GIMPLE_TRANSACTION
:
1720 lower_transaction (gsi
, wi
);
1724 *handled_ops_p
= !gimple_has_substatements (stmt
);
1731 /* Iterate through the statements in the sequence, lowering them all
1732 as appropriate for being outside of a transaction. */
1735 lower_sequence_no_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1736 struct walk_stmt_info
* wi
)
1738 gimple
*stmt
= gsi_stmt (*gsi
);
1740 if (gimple_code (stmt
) == GIMPLE_TRANSACTION
)
1742 *handled_ops_p
= true;
1743 lower_transaction (gsi
, wi
);
1746 *handled_ops_p
= !gimple_has_substatements (stmt
);
1751 /* Main entry point for flattening GIMPLE_TRANSACTION constructs. After
1752 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has
1753 been moved out, and all the data required for constructing a proper
1754 CFG has been recorded. */
1757 execute_lower_tm (void)
1759 struct walk_stmt_info wi
;
1762 /* Transactional clones aren't created until a later pass. */
1763 gcc_assert (!decl_is_tm_clone (current_function_decl
));
1765 body
= gimple_body (current_function_decl
);
1766 memset (&wi
, 0, sizeof (wi
));
1767 walk_gimple_seq_mod (&body
, lower_sequence_no_tm
, NULL
, &wi
);
1768 gimple_set_body (current_function_decl
, body
);
1775 const pass_data pass_data_lower_tm
=
1777 GIMPLE_PASS
, /* type */
1778 "tmlower", /* name */
1779 OPTGROUP_NONE
, /* optinfo_flags */
1780 TV_TRANS_MEM
, /* tv_id */
1781 PROP_gimple_lcf
, /* properties_required */
1782 0, /* properties_provided */
1783 0, /* properties_destroyed */
1784 0, /* todo_flags_start */
1785 0, /* todo_flags_finish */
1788 class pass_lower_tm
: public gimple_opt_pass
1791 pass_lower_tm (gcc::context
*ctxt
)
1792 : gimple_opt_pass (pass_data_lower_tm
, ctxt
)
1795 /* opt_pass methods: */
1796 virtual bool gate (function
*) { return flag_tm
; }
1797 virtual unsigned int execute (function
*) { return execute_lower_tm (); }
1799 }; // class pass_lower_tm
1804 make_pass_lower_tm (gcc::context
*ctxt
)
1806 return new pass_lower_tm (ctxt
);
1809 /* Collect region information for each transaction. */
1815 /* The field "transaction_stmt" is initially a gtransaction *,
1816 but eventually gets lowered to a gcall *(to BUILT_IN_TM_START).
1818 Helper method to get it as a gtransaction *, with code-checking
1819 in a checked-build. */
1822 get_transaction_stmt () const
1824 return as_a
<gtransaction
*> (transaction_stmt
);
1829 /* Link to the next unnested transaction. */
1830 struct tm_region
*next
;
1832 /* Link to the next inner transaction. */
1833 struct tm_region
*inner
;
1835 /* Link to the next outer transaction. */
1836 struct tm_region
*outer
;
1838 /* The GIMPLE_TRANSACTION statement beginning this transaction.
1839 After TM_MARK, this gets replaced by a call to
1841 Hence this will be either a gtransaction *or a gcall *. */
1842 gimple
*transaction_stmt
;
1844 /* After TM_MARK expands the GIMPLE_TRANSACTION into a call to
1845 BUILT_IN_TM_START, this field is true if the transaction is an
1846 outer transaction. */
1847 bool original_transaction_was_outer
;
1849 /* Return value from BUILT_IN_TM_START. */
1852 /* The entry block to this region. This will always be the first
1853 block of the body of the transaction. */
1854 basic_block entry_block
;
1856 /* The first block after an expanded call to _ITM_beginTransaction. */
1857 basic_block restart_block
;
1859 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK.
1860 These blocks are still a part of the region (i.e., the border is
1861 inclusive). Note that this set is only complete for paths in the CFG
1862 starting at ENTRY_BLOCK, and that there is no exit block recorded for
1863 the edge to the "over" label. */
1866 /* The set of all blocks that have an TM_IRREVOCABLE call. */
1870 /* True if there are pending edge statements to be committed for the
1871 current function being scanned in the tmmark pass. */
1872 bool pending_edge_inserts_p
;
1874 static struct tm_region
*all_tm_regions
;
1875 static bitmap_obstack tm_obstack
;
1878 /* A subroutine of tm_region_init. Record the existence of the
1879 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */
1881 static struct tm_region
*
1882 tm_region_init_0 (struct tm_region
*outer
, basic_block bb
,
1885 struct tm_region
*region
;
1887 region
= (struct tm_region
*)
1888 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1892 region
->next
= outer
->inner
;
1893 outer
->inner
= region
;
1897 region
->next
= all_tm_regions
;
1898 all_tm_regions
= region
;
1900 region
->inner
= NULL
;
1901 region
->outer
= outer
;
1903 region
->transaction_stmt
= stmt
;
1904 region
->original_transaction_was_outer
= false;
1905 region
->tm_state
= NULL
;
1907 /* There are either one or two edges out of the block containing
1908 the GIMPLE_TRANSACTION, one to the actual region and one to the
1909 "over" label if the region contains an abort. The former will
1910 always be the one marked FALLTHRU. */
1911 region
->entry_block
= FALLTHRU_EDGE (bb
)->dest
;
1913 region
->exit_blocks
= BITMAP_ALLOC (&tm_obstack
);
1914 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1919 /* A subroutine of tm_region_init. Record all the exit and
1920 irrevocable blocks in BB into the region's exit_blocks and
1921 irr_blocks bitmaps. Returns the new region being scanned. */
1923 static struct tm_region
*
1924 tm_region_init_1 (struct tm_region
*region
, basic_block bb
)
1926 gimple_stmt_iterator gsi
;
1930 || (!region
->irr_blocks
&& !region
->exit_blocks
))
1933 /* Check to see if this is the end of a region by seeing if it
1934 contains a call to __builtin_tm_commit{,_eh}. Note that the
1935 outermost region for DECL_IS_TM_CLONE need not collect this. */
1936 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
1939 if (gimple_code (g
) == GIMPLE_CALL
)
1941 tree fn
= gimple_call_fndecl (g
);
1942 if (fn
&& DECL_BUILT_IN_CLASS (fn
) == BUILT_IN_NORMAL
)
1944 if ((DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT
1945 || DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT_EH
)
1946 && region
->exit_blocks
)
1948 bitmap_set_bit (region
->exit_blocks
, bb
->index
);
1949 region
= region
->outer
;
1952 if (DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_IRREVOCABLE
)
1953 bitmap_set_bit (region
->irr_blocks
, bb
->index
);
1960 /* Collect all of the transaction regions within the current function
1961 and record them in ALL_TM_REGIONS. The REGION parameter may specify
1962 an "outermost" region for use by tm clones. */
1965 tm_region_init (struct tm_region
*region
)
1971 auto_vec
<basic_block
> queue
;
1972 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
1973 struct tm_region
*old_region
;
1974 auto_vec
<tm_region
*> bb_regions
;
1976 all_tm_regions
= region
;
1977 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1979 /* We could store this information in bb->aux, but we may get called
1980 through get_all_tm_blocks() from another pass that may be already
1982 bb_regions
.safe_grow_cleared (last_basic_block_for_fn (cfun
));
1984 queue
.safe_push (bb
);
1985 bb_regions
[bb
->index
] = region
;
1989 region
= bb_regions
[bb
->index
];
1990 bb_regions
[bb
->index
] = NULL
;
1992 /* Record exit and irrevocable blocks. */
1993 region
= tm_region_init_1 (region
, bb
);
1995 /* Check for the last statement in the block beginning a new region. */
1997 old_region
= region
;
1999 if (gtransaction
*trans_stmt
= dyn_cast
<gtransaction
*> (g
))
2000 region
= tm_region_init_0 (region
, bb
, trans_stmt
);
2002 /* Process subsequent blocks. */
2003 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2004 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2006 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2007 queue
.safe_push (e
->dest
);
2009 /* If the current block started a new region, make sure that only
2010 the entry block of the new region is associated with this region.
2011 Other successors are still part of the old region. */
2012 if (old_region
!= region
&& e
->dest
!= region
->entry_block
)
2013 bb_regions
[e
->dest
->index
] = old_region
;
2015 bb_regions
[e
->dest
->index
] = region
;
2018 while (!queue
.is_empty ());
2019 BITMAP_FREE (visited_blocks
);
2022 /* The "gate" function for all transactional memory expansion and optimization
2023 passes. We collect region information for each top-level transaction, and
2024 if we don't find any, we skip all of the TM passes. Each region will have
2025 all of the exit blocks recorded, and the originating statement. */
2033 calculate_dominance_info (CDI_DOMINATORS
);
2034 bitmap_obstack_initialize (&tm_obstack
);
2036 /* If the function is a TM_CLONE, then the entire function is the region. */
2037 if (decl_is_tm_clone (current_function_decl
))
2039 struct tm_region
*region
= (struct tm_region
*)
2040 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
2041 memset (region
, 0, sizeof (*region
));
2042 region
->entry_block
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
2043 /* For a clone, the entire function is the region. But even if
2044 we don't need to record any exit blocks, we may need to
2045 record irrevocable blocks. */
2046 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
2048 tm_region_init (region
);
2052 tm_region_init (NULL
);
2054 /* If we didn't find any regions, cleanup and skip the whole tree
2055 of tm-related optimizations. */
2056 if (all_tm_regions
== NULL
)
2058 bitmap_obstack_release (&tm_obstack
);
2068 const pass_data pass_data_tm_init
=
2070 GIMPLE_PASS
, /* type */
2071 "*tminit", /* name */
2072 OPTGROUP_NONE
, /* optinfo_flags */
2073 TV_TRANS_MEM
, /* tv_id */
2074 ( PROP_ssa
| PROP_cfg
), /* properties_required */
2075 0, /* properties_provided */
2076 0, /* properties_destroyed */
2077 0, /* todo_flags_start */
2078 0, /* todo_flags_finish */
2081 class pass_tm_init
: public gimple_opt_pass
2084 pass_tm_init (gcc::context
*ctxt
)
2085 : gimple_opt_pass (pass_data_tm_init
, ctxt
)
2088 /* opt_pass methods: */
2089 virtual bool gate (function
*) { return gate_tm_init (); }
2091 }; // class pass_tm_init
2096 make_pass_tm_init (gcc::context
*ctxt
)
2098 return new pass_tm_init (ctxt
);
2101 /* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region
2102 represented by STATE. */
2105 transaction_subcode_ior (struct tm_region
*region
, unsigned flags
)
2107 if (region
&& region
->transaction_stmt
)
2109 gtransaction
*transaction_stmt
= region
->get_transaction_stmt ();
2110 flags
|= gimple_transaction_subcode (transaction_stmt
);
2111 gimple_transaction_set_subcode (transaction_stmt
, flags
);
2115 /* Construct a memory load in a transactional context. Return the
2116 gimple statement performing the load, or NULL if there is no
2117 TM_LOAD builtin of the appropriate size to do the load.
2119 LOC is the location to use for the new statement(s). */
2122 build_tm_load (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2124 enum built_in_function code
= END_BUILTINS
;
2125 tree t
, type
= TREE_TYPE (rhs
), decl
;
2128 if (type
== float_type_node
)
2129 code
= BUILT_IN_TM_LOAD_FLOAT
;
2130 else if (type
== double_type_node
)
2131 code
= BUILT_IN_TM_LOAD_DOUBLE
;
2132 else if (type
== long_double_type_node
)
2133 code
= BUILT_IN_TM_LOAD_LDOUBLE
;
2134 else if (TYPE_SIZE_UNIT (type
) != NULL
2135 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2137 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2140 code
= BUILT_IN_TM_LOAD_1
;
2143 code
= BUILT_IN_TM_LOAD_2
;
2146 code
= BUILT_IN_TM_LOAD_4
;
2149 code
= BUILT_IN_TM_LOAD_8
;
2154 if (code
== END_BUILTINS
)
2156 decl
= targetm
.vectorize
.builtin_tm_load (type
);
2161 decl
= builtin_decl_explicit (code
);
2163 t
= gimplify_addr (gsi
, rhs
);
2164 gcall
= gimple_build_call (decl
, 1, t
);
2165 gimple_set_location (gcall
, loc
);
2167 t
= TREE_TYPE (TREE_TYPE (decl
));
2168 if (useless_type_conversion_p (type
, t
))
2170 gimple_call_set_lhs (gcall
, lhs
);
2171 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2178 temp
= create_tmp_reg (t
);
2179 gimple_call_set_lhs (gcall
, temp
);
2180 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2182 t
= fold_build1 (VIEW_CONVERT_EXPR
, type
, temp
);
2183 g
= gimple_build_assign (lhs
, t
);
2184 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2191 /* Similarly for storing TYPE in a transactional context. */
2194 build_tm_store (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2196 enum built_in_function code
= END_BUILTINS
;
2197 tree t
, fn
, type
= TREE_TYPE (rhs
), simple_type
;
2200 if (type
== float_type_node
)
2201 code
= BUILT_IN_TM_STORE_FLOAT
;
2202 else if (type
== double_type_node
)
2203 code
= BUILT_IN_TM_STORE_DOUBLE
;
2204 else if (type
== long_double_type_node
)
2205 code
= BUILT_IN_TM_STORE_LDOUBLE
;
2206 else if (TYPE_SIZE_UNIT (type
) != NULL
2207 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2209 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2212 code
= BUILT_IN_TM_STORE_1
;
2215 code
= BUILT_IN_TM_STORE_2
;
2218 code
= BUILT_IN_TM_STORE_4
;
2221 code
= BUILT_IN_TM_STORE_8
;
2226 if (code
== END_BUILTINS
)
2228 fn
= targetm
.vectorize
.builtin_tm_store (type
);
2233 fn
= builtin_decl_explicit (code
);
2235 simple_type
= TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn
))));
2237 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
2239 /* Handle the easy initialization to zero. */
2240 if (!CONSTRUCTOR_ELTS (rhs
))
2241 rhs
= build_int_cst (simple_type
, 0);
2244 /* ...otherwise punt to the caller and probably use
2245 BUILT_IN_TM_MEMMOVE, because we can't wrap a
2246 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce
2251 else if (!useless_type_conversion_p (simple_type
, type
))
2256 temp
= create_tmp_reg (simple_type
);
2257 t
= fold_build1 (VIEW_CONVERT_EXPR
, simple_type
, rhs
);
2258 g
= gimple_build_assign (temp
, t
);
2259 gimple_set_location (g
, loc
);
2260 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2265 t
= gimplify_addr (gsi
, lhs
);
2266 gcall
= gimple_build_call (fn
, 2, t
, rhs
);
2267 gimple_set_location (gcall
, loc
);
2268 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2274 /* Expand an assignment statement into transactional builtins. */
2277 expand_assign_tm (struct tm_region
*region
, gimple_stmt_iterator
*gsi
)
2279 gimple
*stmt
= gsi_stmt (*gsi
);
2280 location_t loc
= gimple_location (stmt
);
2281 tree lhs
= gimple_assign_lhs (stmt
);
2282 tree rhs
= gimple_assign_rhs1 (stmt
);
2283 bool store_p
= requires_barrier (region
->entry_block
, lhs
, NULL
);
2284 bool load_p
= requires_barrier (region
->entry_block
, rhs
, NULL
);
2285 gimple
*gcall
= NULL
;
2287 if (!load_p
&& !store_p
)
2289 /* Add thread private addresses to log if applicable. */
2290 requires_barrier (region
->entry_block
, lhs
, stmt
);
2295 // Remove original load/store statement.
2296 gsi_remove (gsi
, true);
2298 if (load_p
&& !store_p
)
2300 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2301 gcall
= build_tm_load (loc
, lhs
, rhs
, gsi
);
2303 else if (store_p
&& !load_p
)
2305 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2306 gcall
= build_tm_store (loc
, lhs
, rhs
, gsi
);
2310 tree lhs_addr
, rhs_addr
, tmp
;
2313 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2315 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2317 /* ??? Figure out if there's any possible overlap between the LHS
2318 and the RHS and if not, use MEMCPY. */
2320 if (load_p
&& is_gimple_reg (lhs
))
2322 tmp
= create_tmp_var (TREE_TYPE (lhs
));
2323 lhs_addr
= build_fold_addr_expr (tmp
);
2328 lhs_addr
= gimplify_addr (gsi
, lhs
);
2330 rhs_addr
= gimplify_addr (gsi
, rhs
);
2331 gcall
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
),
2332 3, lhs_addr
, rhs_addr
,
2333 TYPE_SIZE_UNIT (TREE_TYPE (lhs
)));
2334 gimple_set_location (gcall
, loc
);
2335 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2339 gcall
= gimple_build_assign (lhs
, tmp
);
2340 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2344 /* Now that we have the load/store in its instrumented form, add
2345 thread private addresses to the log if applicable. */
2347 requires_barrier (region
->entry_block
, lhs
, gcall
);
2349 // The calls to build_tm_{store,load} above inserted the instrumented
2350 // call into the stream.
2351 // gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2355 /* Expand a call statement as appropriate for a transaction. That is,
2356 either verify that the call does not affect the transaction, or
2357 redirect the call to a clone that handles transactions, or change
2358 the transaction state to IRREVOCABLE. Return true if the call is
2359 one of the builtins that end a transaction. */
2362 expand_call_tm (struct tm_region
*region
,
2363 gimple_stmt_iterator
*gsi
)
2365 gcall
*stmt
= as_a
<gcall
*> (gsi_stmt (*gsi
));
2366 tree lhs
= gimple_call_lhs (stmt
);
2368 struct cgraph_node
*node
;
2369 bool retval
= false;
2371 fn_decl
= gimple_call_fndecl (stmt
);
2373 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMCPY
)
2374 || fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
))
2375 transaction_subcode_ior (region
, GTMA_HAVE_STORE
| GTMA_HAVE_LOAD
);
2376 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMSET
))
2377 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2379 if (is_tm_pure_call (stmt
))
2383 retval
= is_tm_ending_fndecl (fn_decl
);
2386 /* Assume all non-const/pure calls write to memory, except
2387 transaction ending builtins. */
2388 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2391 /* For indirect calls, we already generated a call into the runtime. */
2394 tree fn
= gimple_call_fn (stmt
);
2396 /* We are guaranteed never to go irrevocable on a safe or pure
2397 call, and the pure call was handled above. */
2398 if (is_tm_safe (fn
))
2401 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2406 node
= cgraph_node::get (fn_decl
);
2407 /* All calls should have cgraph here. */
2410 /* We can have a nodeless call here if some pass after IPA-tm
2411 added uninstrumented calls. For example, loop distribution
2412 can transform certain loop constructs into __builtin_mem*
2413 calls. In this case, see if we have a suitable TM
2414 replacement and fill in the gaps. */
2415 gcc_assert (DECL_BUILT_IN_CLASS (fn_decl
) == BUILT_IN_NORMAL
);
2416 enum built_in_function code
= DECL_FUNCTION_CODE (fn_decl
);
2417 gcc_assert (code
== BUILT_IN_MEMCPY
2418 || code
== BUILT_IN_MEMMOVE
2419 || code
== BUILT_IN_MEMSET
);
2421 tree repl
= find_tm_replacement_function (fn_decl
);
2424 gimple_call_set_fndecl (stmt
, repl
);
2426 node
= cgraph_node::create (repl
);
2427 node
->local
.tm_may_enter_irr
= false;
2428 return expand_call_tm (region
, gsi
);
2432 if (node
->local
.tm_may_enter_irr
)
2433 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2435 if (is_tm_abort (fn_decl
))
2437 transaction_subcode_ior (region
, GTMA_HAVE_ABORT
);
2441 /* Instrument the store if needed.
2443 If the assignment happens inside the function call (return slot
2444 optimization), there is no instrumentation to be done, since
2445 the callee should have done the right thing. */
2446 if (lhs
&& requires_barrier (region
->entry_block
, lhs
, stmt
)
2447 && !gimple_call_return_slot_opt_p (stmt
))
2449 tree tmp
= create_tmp_reg (TREE_TYPE (lhs
));
2450 location_t loc
= gimple_location (stmt
);
2451 edge fallthru_edge
= NULL
;
2452 gassign
*assign_stmt
;
2454 /* Remember if the call was going to throw. */
2455 if (stmt_can_throw_internal (stmt
))
2459 basic_block bb
= gimple_bb (stmt
);
2461 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2462 if (e
->flags
& EDGE_FALLTHRU
)
2469 gimple_call_set_lhs (stmt
, tmp
);
2471 assign_stmt
= gimple_build_assign (lhs
, tmp
);
2472 gimple_set_location (assign_stmt
, loc
);
2474 /* We cannot throw in the middle of a BB. If the call was going
2475 to throw, place the instrumentation on the fallthru edge, so
2476 the call remains the last statement in the block. */
2479 gimple_seq fallthru_seq
= gimple_seq_alloc_with_stmt (assign_stmt
);
2480 gimple_stmt_iterator fallthru_gsi
= gsi_start (fallthru_seq
);
2481 expand_assign_tm (region
, &fallthru_gsi
);
2482 gsi_insert_seq_on_edge (fallthru_edge
, fallthru_seq
);
2483 pending_edge_inserts_p
= true;
2487 gsi_insert_after (gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
2488 expand_assign_tm (region
, gsi
);
2491 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2498 /* Expand all statements in BB as appropriate for being inside
2502 expand_block_tm (struct tm_region
*region
, basic_block bb
)
2504 gimple_stmt_iterator gsi
;
2506 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2508 gimple
*stmt
= gsi_stmt (gsi
);
2509 switch (gimple_code (stmt
))
2512 /* Only memory reads/writes need to be instrumented. */
2513 if (gimple_assign_single_p (stmt
)
2514 && !gimple_clobber_p (stmt
))
2516 expand_assign_tm (region
, &gsi
);
2522 if (expand_call_tm (region
, &gsi
))
2532 if (!gsi_end_p (gsi
))
2537 /* Return the list of basic-blocks in REGION.
2539 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks
2540 following a TM_IRREVOCABLE call.
2542 INCLUDE_UNINSTRUMENTED_P is TRUE if we should include the
2543 uninstrumented code path blocks in the list of basic blocks
2544 returned, false otherwise. */
2546 static vec
<basic_block
>
2547 get_tm_region_blocks (basic_block entry_block
,
2550 bitmap all_region_blocks
,
2551 bool stop_at_irrevocable_p
,
2552 bool include_uninstrumented_p
= true)
2554 vec
<basic_block
> bbs
= vNULL
;
2558 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
2561 bbs
.safe_push (entry_block
);
2562 bitmap_set_bit (visited_blocks
, entry_block
->index
);
2566 basic_block bb
= bbs
[i
++];
2569 bitmap_bit_p (exit_blocks
, bb
->index
))
2572 if (stop_at_irrevocable_p
2574 && bitmap_bit_p (irr_blocks
, bb
->index
))
2577 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2578 if ((include_uninstrumented_p
2579 || !(e
->flags
& EDGE_TM_UNINSTRUMENTED
))
2580 && !bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2582 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2583 bbs
.safe_push (e
->dest
);
2586 while (i
< bbs
.length ());
2588 if (all_region_blocks
)
2589 bitmap_ior_into (all_region_blocks
, visited_blocks
);
2591 BITMAP_FREE (visited_blocks
);
2595 // Callback data for collect_bb2reg.
2598 vec
<tm_region
*> *bb2reg
;
2599 bool include_uninstrumented_p
;
2602 // Callback for expand_regions, collect innermost region data for each bb.
2604 collect_bb2reg (struct tm_region
*region
, void *data
)
2606 struct bb2reg_stuff
*stuff
= (struct bb2reg_stuff
*)data
;
2607 vec
<tm_region
*> *bb2reg
= stuff
->bb2reg
;
2608 vec
<basic_block
> queue
;
2612 queue
= get_tm_region_blocks (region
->entry_block
,
2613 region
->exit_blocks
,
2616 /*stop_at_irr_p=*/true,
2617 stuff
->include_uninstrumented_p
);
2619 // We expect expand_region to perform a post-order traversal of the region
2620 // tree. Therefore the last region seen for any bb is the innermost.
2621 FOR_EACH_VEC_ELT (queue
, i
, bb
)
2622 (*bb2reg
)[bb
->index
] = region
;
2628 // Returns a vector, indexed by BB->INDEX, of the innermost tm_region to
2629 // which a basic block belongs. Note that we only consider the instrumented
2630 // code paths for the region; the uninstrumented code paths are ignored if
2631 // INCLUDE_UNINSTRUMENTED_P is false.
2633 // ??? This data is very similar to the bb_regions array that is collected
2634 // during tm_region_init. Or, rather, this data is similar to what could
2635 // be used within tm_region_init. The actual computation in tm_region_init
2636 // begins and ends with bb_regions entirely full of NULL pointers, due to
2637 // the way in which pointers are swapped in and out of the array.
2639 // ??? Our callers expect that blocks are not shared between transactions.
2640 // When the optimizers get too smart, and blocks are shared, then during
2641 // the tm_mark phase we'll add log entries to only one of the two transactions,
2642 // and in the tm_edge phase we'll add edges to the CFG that create invalid
2643 // cycles. The symptom being SSA defs that do not dominate their uses.
2644 // Note that the optimizers were locally correct with their transformation,
2645 // as we have no info within the program that suggests that the blocks cannot
2648 // ??? There is currently a hack inside tree-ssa-pre.c to work around the
2649 // only known instance of this block sharing.
2651 static vec
<tm_region
*>
2652 get_bb_regions_instrumented (bool traverse_clones
,
2653 bool include_uninstrumented_p
)
2655 unsigned n
= last_basic_block_for_fn (cfun
);
2656 struct bb2reg_stuff stuff
;
2657 vec
<tm_region
*> ret
;
2660 ret
.safe_grow_cleared (n
);
2661 stuff
.bb2reg
= &ret
;
2662 stuff
.include_uninstrumented_p
= include_uninstrumented_p
;
2663 expand_regions (all_tm_regions
, collect_bb2reg
, &stuff
, traverse_clones
);
2668 /* Set the IN_TRANSACTION for all gimple statements that appear in a
2672 compute_transaction_bits (void)
2674 struct tm_region
*region
;
2675 vec
<basic_block
> queue
;
2679 /* ?? Perhaps we need to abstract gate_tm_init further, because we
2680 certainly don't need it to calculate CDI_DOMINATOR info. */
2683 FOR_EACH_BB_FN (bb
, cfun
)
2684 bb
->flags
&= ~BB_IN_TRANSACTION
;
2686 for (region
= all_tm_regions
; region
; region
= region
->next
)
2688 queue
= get_tm_region_blocks (region
->entry_block
,
2689 region
->exit_blocks
,
2692 /*stop_at_irr_p=*/true);
2693 for (i
= 0; queue
.iterate (i
, &bb
); ++i
)
2694 bb
->flags
|= BB_IN_TRANSACTION
;
2699 bitmap_obstack_release (&tm_obstack
);
2702 /* Replace the GIMPLE_TRANSACTION in this region with the corresponding
2703 call to BUILT_IN_TM_START. */
2706 expand_transaction (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2708 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2709 basic_block transaction_bb
= gimple_bb (region
->transaction_stmt
);
2710 tree tm_state
= region
->tm_state
;
2711 tree tm_state_type
= TREE_TYPE (tm_state
);
2712 edge abort_edge
= NULL
;
2713 edge inst_edge
= NULL
;
2714 edge uninst_edge
= NULL
;
2715 edge fallthru_edge
= NULL
;
2717 // Identify the various successors of the transaction start.
2721 FOR_EACH_EDGE (e
, i
, transaction_bb
->succs
)
2723 if (e
->flags
& EDGE_TM_ABORT
)
2725 else if (e
->flags
& EDGE_TM_UNINSTRUMENTED
)
2729 if (e
->flags
& EDGE_FALLTHRU
)
2734 /* ??? There are plenty of bits here we're not computing. */
2736 int subcode
= gimple_transaction_subcode (region
->get_transaction_stmt ());
2738 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2739 flags
|= PR_DOESGOIRREVOCABLE
;
2740 if ((subcode
& GTMA_MAY_ENTER_IRREVOCABLE
) == 0)
2741 flags
|= PR_HASNOIRREVOCABLE
;
2742 /* If the transaction does not have an abort in lexical scope and is not
2743 marked as an outer transaction, then it will never abort. */
2744 if ((subcode
& GTMA_HAVE_ABORT
) == 0 && (subcode
& GTMA_IS_OUTER
) == 0)
2745 flags
|= PR_HASNOABORT
;
2746 if ((subcode
& GTMA_HAVE_STORE
) == 0)
2747 flags
|= PR_READONLY
;
2748 if (inst_edge
&& !(subcode
& GTMA_HAS_NO_INSTRUMENTATION
))
2749 flags
|= PR_INSTRUMENTEDCODE
;
2751 flags
|= PR_UNINSTRUMENTEDCODE
;
2752 if (subcode
& GTMA_IS_OUTER
)
2753 region
->original_transaction_was_outer
= true;
2754 tree t
= build_int_cst (tm_state_type
, flags
);
2755 gcall
*call
= gimple_build_call (tm_start
, 1, t
);
2756 gimple_call_set_lhs (call
, tm_state
);
2757 gimple_set_location (call
, gimple_location (region
->transaction_stmt
));
2759 // Replace the GIMPLE_TRANSACTION with the call to BUILT_IN_TM_START.
2760 gimple_stmt_iterator gsi
= gsi_last_bb (transaction_bb
);
2761 gcc_assert (gsi_stmt (gsi
) == region
->transaction_stmt
);
2762 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
2763 gsi_remove (&gsi
, true);
2764 region
->transaction_stmt
= call
;
2767 // Generate log saves.
2768 if (!tm_log_save_addresses
.is_empty ())
2769 tm_log_emit_saves (region
->entry_block
, transaction_bb
);
2771 // In the beginning, we've no tests to perform on transaction restart.
2772 // Note that after this point, transaction_bb becomes the "most recent
2773 // block containing tests for the transaction".
2774 region
->restart_block
= region
->entry_block
;
2776 // Generate log restores.
2777 if (!tm_log_save_addresses
.is_empty ())
2779 basic_block test_bb
= create_empty_bb (transaction_bb
);
2780 basic_block code_bb
= create_empty_bb (test_bb
);
2781 basic_block join_bb
= create_empty_bb (code_bb
);
2782 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2783 add_bb_to_loop (code_bb
, transaction_bb
->loop_father
);
2784 add_bb_to_loop (join_bb
, transaction_bb
->loop_father
);
2785 if (region
->restart_block
== region
->entry_block
)
2786 region
->restart_block
= test_bb
;
2788 tree t1
= create_tmp_reg (tm_state_type
);
2789 tree t2
= build_int_cst (tm_state_type
, A_RESTORELIVEVARIABLES
);
2790 gimple
*stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2791 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2792 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2794 t2
= build_int_cst (tm_state_type
, 0);
2795 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2796 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2798 tm_log_emit_restores (region
->entry_block
, code_bb
);
2800 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2801 edge et
= make_edge (test_bb
, code_bb
, EDGE_TRUE_VALUE
);
2802 edge ef
= make_edge (test_bb
, join_bb
, EDGE_FALSE_VALUE
);
2803 redirect_edge_pred (fallthru_edge
, join_bb
);
2805 join_bb
->frequency
= test_bb
->frequency
= transaction_bb
->frequency
;
2806 join_bb
->count
= test_bb
->count
= transaction_bb
->count
;
2808 ei
->probability
= PROB_ALWAYS
;
2809 et
->probability
= PROB_LIKELY
;
2810 ef
->probability
= PROB_UNLIKELY
;
2811 et
->count
= apply_probability (test_bb
->count
, et
->probability
);
2812 ef
->count
= apply_probability (test_bb
->count
, ef
->probability
);
2814 code_bb
->count
= et
->count
;
2815 code_bb
->frequency
= EDGE_FREQUENCY (et
);
2817 transaction_bb
= join_bb
;
2820 // If we have an ABORT edge, create a test to perform the abort.
2823 basic_block test_bb
= create_empty_bb (transaction_bb
);
2824 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2825 if (region
->restart_block
== region
->entry_block
)
2826 region
->restart_block
= test_bb
;
2828 tree t1
= create_tmp_reg (tm_state_type
);
2829 tree t2
= build_int_cst (tm_state_type
, A_ABORTTRANSACTION
);
2830 gimple
*stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2831 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2832 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2834 t2
= build_int_cst (tm_state_type
, 0);
2835 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2836 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2838 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2839 test_bb
->frequency
= transaction_bb
->frequency
;
2840 test_bb
->count
= transaction_bb
->count
;
2841 ei
->probability
= PROB_ALWAYS
;
2843 // Not abort edge. If both are live, chose one at random as we'll
2844 // we'll be fixing that up below.
2845 redirect_edge_pred (fallthru_edge
, test_bb
);
2846 fallthru_edge
->flags
= EDGE_FALSE_VALUE
;
2847 fallthru_edge
->probability
= PROB_VERY_LIKELY
;
2848 fallthru_edge
->count
2849 = apply_probability (test_bb
->count
, fallthru_edge
->probability
);
2852 redirect_edge_pred (abort_edge
, test_bb
);
2853 abort_edge
->flags
= EDGE_TRUE_VALUE
;
2854 abort_edge
->probability
= PROB_VERY_UNLIKELY
;
2856 = apply_probability (test_bb
->count
, abort_edge
->probability
);
2858 transaction_bb
= test_bb
;
2861 // If we have both instrumented and uninstrumented code paths, select one.
2862 if (inst_edge
&& uninst_edge
)
2864 basic_block test_bb
= create_empty_bb (transaction_bb
);
2865 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2866 if (region
->restart_block
== region
->entry_block
)
2867 region
->restart_block
= test_bb
;
2869 tree t1
= create_tmp_reg (tm_state_type
);
2870 tree t2
= build_int_cst (tm_state_type
, A_RUNUNINSTRUMENTEDCODE
);
2872 gimple
*stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2873 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2874 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2876 t2
= build_int_cst (tm_state_type
, 0);
2877 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2878 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2880 // Create the edge into test_bb first, as we want to copy values
2881 // out of the fallthru edge.
2882 edge e
= make_edge (transaction_bb
, test_bb
, fallthru_edge
->flags
);
2883 e
->probability
= fallthru_edge
->probability
;
2884 test_bb
->count
= e
->count
= fallthru_edge
->count
;
2885 test_bb
->frequency
= EDGE_FREQUENCY (e
);
2887 // Now update the edges to the inst/uninist implementations.
2888 // For now assume that the paths are equally likely. When using HTM,
2889 // we'll try the uninst path first and fallback to inst path if htm
2890 // buffers are exceeded. Without HTM we start with the inst path and
2891 // use the uninst path when falling back to serial mode.
2892 redirect_edge_pred (inst_edge
, test_bb
);
2893 inst_edge
->flags
= EDGE_FALSE_VALUE
;
2894 inst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2896 = apply_probability (test_bb
->count
, inst_edge
->probability
);
2898 redirect_edge_pred (uninst_edge
, test_bb
);
2899 uninst_edge
->flags
= EDGE_TRUE_VALUE
;
2900 uninst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2902 = apply_probability (test_bb
->count
, uninst_edge
->probability
);
2905 // If we have no previous special cases, and we have PHIs at the beginning
2906 // of the atomic region, this means we have a loop at the beginning of the
2907 // atomic region that shares the first block. This can cause problems with
2908 // the transaction restart abnormal edges to be added in the tm_edges pass.
2909 // Solve this by adding a new empty block to receive the abnormal edges.
2910 if (region
->restart_block
== region
->entry_block
2911 && phi_nodes (region
->entry_block
))
2913 basic_block empty_bb
= create_empty_bb (transaction_bb
);
2914 region
->restart_block
= empty_bb
;
2915 add_bb_to_loop (empty_bb
, transaction_bb
->loop_father
);
2917 redirect_edge_pred (fallthru_edge
, empty_bb
);
2918 make_edge (transaction_bb
, empty_bb
, EDGE_FALLTHRU
);
2924 /* Generate the temporary to be used for the return value of
2925 BUILT_IN_TM_START. */
2928 generate_tm_state (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2930 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2932 create_tmp_reg (TREE_TYPE (TREE_TYPE (tm_start
)), "tm_state");
2934 // Reset the subcode, post optimizations. We'll fill this in
2935 // again as we process blocks.
2936 if (region
->exit_blocks
)
2938 gtransaction
*transaction_stmt
= region
->get_transaction_stmt ();
2939 unsigned int subcode
= gimple_transaction_subcode (transaction_stmt
);
2941 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2942 subcode
&= (GTMA_DECLARATION_MASK
| GTMA_DOES_GO_IRREVOCABLE
2943 | GTMA_MAY_ENTER_IRREVOCABLE
2944 | GTMA_HAS_NO_INSTRUMENTATION
);
2946 subcode
&= GTMA_DECLARATION_MASK
;
2947 gimple_transaction_set_subcode (transaction_stmt
, subcode
);
2953 // Propagate flags from inner transactions outwards.
2955 propagate_tm_flags_out (struct tm_region
*region
)
2959 propagate_tm_flags_out (region
->inner
);
2961 if (region
->outer
&& region
->outer
->transaction_stmt
)
2964 = gimple_transaction_subcode (region
->get_transaction_stmt ());
2965 s
&= (GTMA_HAVE_ABORT
| GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
2966 | GTMA_MAY_ENTER_IRREVOCABLE
);
2967 s
|= gimple_transaction_subcode (region
->outer
->get_transaction_stmt ());
2968 gimple_transaction_set_subcode (region
->outer
->get_transaction_stmt (),
2972 propagate_tm_flags_out (region
->next
);
2975 /* Entry point to the MARK phase of TM expansion. Here we replace
2976 transactional memory statements with calls to builtins, and function
2977 calls with their transactional clones (if available). But we don't
2978 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */
2981 execute_tm_mark (void)
2983 pending_edge_inserts_p
= false;
2985 expand_regions (all_tm_regions
, generate_tm_state
, NULL
,
2986 /*traverse_clones=*/true);
2990 vec
<tm_region
*> bb_regions
2991 = get_bb_regions_instrumented (/*traverse_clones=*/true,
2992 /*include_uninstrumented_p=*/false);
2993 struct tm_region
*r
;
2996 // Expand memory operations into calls into the runtime.
2997 // This collects log entries as well.
2998 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3002 if (r
->transaction_stmt
)
3005 = gimple_transaction_subcode (r
->get_transaction_stmt ());
3007 /* If we're sure to go irrevocable, there won't be
3008 anything to expand, since the run-time will go
3009 irrevocable right away. */
3010 if (sub
& GTMA_DOES_GO_IRREVOCABLE
3011 && sub
& GTMA_MAY_ENTER_IRREVOCABLE
)
3014 expand_block_tm (r
, BASIC_BLOCK_FOR_FN (cfun
, i
));
3018 bb_regions
.release ();
3020 // Propagate flags from inner transactions outwards.
3021 propagate_tm_flags_out (all_tm_regions
);
3023 // Expand GIMPLE_TRANSACTIONs into calls into the runtime.
3024 expand_regions (all_tm_regions
, expand_transaction
, NULL
,
3025 /*traverse_clones=*/false);
3030 if (pending_edge_inserts_p
)
3031 gsi_commit_edge_inserts ();
3032 free_dominance_info (CDI_DOMINATORS
);
3038 const pass_data pass_data_tm_mark
=
3040 GIMPLE_PASS
, /* type */
3041 "tmmark", /* name */
3042 OPTGROUP_NONE
, /* optinfo_flags */
3043 TV_TRANS_MEM
, /* tv_id */
3044 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3045 0, /* properties_provided */
3046 0, /* properties_destroyed */
3047 0, /* todo_flags_start */
3048 TODO_update_ssa
, /* todo_flags_finish */
3051 class pass_tm_mark
: public gimple_opt_pass
3054 pass_tm_mark (gcc::context
*ctxt
)
3055 : gimple_opt_pass (pass_data_tm_mark
, ctxt
)
3058 /* opt_pass methods: */
3059 virtual unsigned int execute (function
*) { return execute_tm_mark (); }
3061 }; // class pass_tm_mark
3066 make_pass_tm_mark (gcc::context
*ctxt
)
3068 return new pass_tm_mark (ctxt
);
3072 /* Create an abnormal edge from STMT at iter, splitting the block
3073 as necessary. Adjust *PNEXT as needed for the split block. */
3076 split_bb_make_tm_edge (gimple
*stmt
, basic_block dest_bb
,
3077 gimple_stmt_iterator iter
, gimple_stmt_iterator
*pnext
)
3079 basic_block bb
= gimple_bb (stmt
);
3080 if (!gsi_one_before_end_p (iter
))
3082 edge e
= split_block (bb
, stmt
);
3083 *pnext
= gsi_start_bb (e
->dest
);
3085 make_edge (bb
, dest_bb
, EDGE_ABNORMAL
);
3087 // Record the need for the edge for the benefit of the rtl passes.
3088 if (cfun
->gimple_df
->tm_restart
== NULL
)
3089 cfun
->gimple_df
->tm_restart
3090 = hash_table
<tm_restart_hasher
>::create_ggc (31);
3092 struct tm_restart_node dummy
;
3094 dummy
.label_or_list
= gimple_block_label (dest_bb
);
3096 tm_restart_node
**slot
= cfun
->gimple_df
->tm_restart
->find_slot (&dummy
,
3098 struct tm_restart_node
*n
= *slot
;
3101 n
= ggc_alloc
<tm_restart_node
> ();
3106 tree old
= n
->label_or_list
;
3107 if (TREE_CODE (old
) == LABEL_DECL
)
3108 old
= tree_cons (NULL
, old
, NULL
);
3109 n
->label_or_list
= tree_cons (NULL
, dummy
.label_or_list
, old
);
3113 /* Split block BB as necessary for every builtin function we added, and
3114 wire up the abnormal back edges implied by the transaction restart. */
3117 expand_block_edges (struct tm_region
*const region
, basic_block bb
)
3119 gimple_stmt_iterator gsi
, next_gsi
;
3121 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi
= next_gsi
)
3123 gimple
*stmt
= gsi_stmt (gsi
);
3127 gsi_next (&next_gsi
);
3129 // ??? Shouldn't we split for any non-pure, non-irrevocable function?
3130 call_stmt
= dyn_cast
<gcall
*> (stmt
);
3132 || (gimple_call_flags (call_stmt
) & ECF_TM_BUILTIN
) == 0)
3135 if (DECL_FUNCTION_CODE (gimple_call_fndecl (call_stmt
))
3136 == BUILT_IN_TM_ABORT
)
3138 // If we have a ``_transaction_cancel [[outer]]'', there is only
3139 // one abnormal edge: to the transaction marked OUTER.
3140 // All compiler-generated instances of BUILT_IN_TM_ABORT have a
3141 // constant argument, which we can examine here. Users invoking
3142 // TM_ABORT directly get what they deserve.
3143 tree arg
= gimple_call_arg (call_stmt
, 0);
3144 if (TREE_CODE (arg
) == INTEGER_CST
3145 && (TREE_INT_CST_LOW (arg
) & AR_OUTERABORT
) != 0
3146 && !decl_is_tm_clone (current_function_decl
))
3148 // Find the GTMA_IS_OUTER transaction.
3149 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3150 if (o
->original_transaction_was_outer
)
3152 split_bb_make_tm_edge (call_stmt
, o
->restart_block
,
3157 // Otherwise, the front-end should have semantically checked
3158 // outer aborts, but in either case the target region is not
3159 // within this function.
3163 // Non-outer, TM aborts have an abnormal edge to the inner-most
3164 // transaction, the one being aborted;
3165 split_bb_make_tm_edge (call_stmt
, region
->restart_block
, gsi
,
3169 // All TM builtins have an abnormal edge to the outer-most transaction.
3170 // We never restart inner transactions. For tm clones, we know a-priori
3171 // that the outer-most transaction is outside the function.
3172 if (decl_is_tm_clone (current_function_decl
))
3175 if (cfun
->gimple_df
->tm_restart
== NULL
)
3176 cfun
->gimple_df
->tm_restart
3177 = hash_table
<tm_restart_hasher
>::create_ggc (31);
3179 // All TM builtins have an abnormal edge to the outer-most transaction.
3180 // We never restart inner transactions.
3181 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3184 split_bb_make_tm_edge (call_stmt
, o
->restart_block
, gsi
, &next_gsi
);
3188 // Delete any tail-call annotation that may have been added.
3189 // The tail-call pass may have mis-identified the commit as being
3190 // a candidate because we had not yet added this restart edge.
3191 gimple_call_set_tail (call_stmt
, false);
3195 /* Entry point to the final expansion of transactional nodes. */
3199 const pass_data pass_data_tm_edges
=
3201 GIMPLE_PASS
, /* type */
3202 "tmedge", /* name */
3203 OPTGROUP_NONE
, /* optinfo_flags */
3204 TV_TRANS_MEM
, /* tv_id */
3205 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3206 0, /* properties_provided */
3207 0, /* properties_destroyed */
3208 0, /* todo_flags_start */
3209 TODO_update_ssa
, /* todo_flags_finish */
3212 class pass_tm_edges
: public gimple_opt_pass
3215 pass_tm_edges (gcc::context
*ctxt
)
3216 : gimple_opt_pass (pass_data_tm_edges
, ctxt
)
3219 /* opt_pass methods: */
3220 virtual unsigned int execute (function
*);
3222 }; // class pass_tm_edges
3225 pass_tm_edges::execute (function
*fun
)
3227 vec
<tm_region
*> bb_regions
3228 = get_bb_regions_instrumented (/*traverse_clones=*/false,
3229 /*include_uninstrumented_p=*/true);
3230 struct tm_region
*r
;
3233 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3235 expand_block_edges (r
, BASIC_BLOCK_FOR_FN (fun
, i
));
3237 bb_regions
.release ();
3239 /* We've got to release the dominance info now, to indicate that it
3240 must be rebuilt completely. Otherwise we'll crash trying to update
3241 the SSA web in the TODO section following this pass. */
3242 free_dominance_info (CDI_DOMINATORS
);
3243 bitmap_obstack_release (&tm_obstack
);
3244 all_tm_regions
= NULL
;
3252 make_pass_tm_edges (gcc::context
*ctxt
)
3254 return new pass_tm_edges (ctxt
);
3257 /* Helper function for expand_regions. Expand REGION and recurse to
3258 the inner region. Call CALLBACK on each region. CALLBACK returns
3259 NULL to continue the traversal, otherwise a non-null value which
3260 this function will return as well. TRAVERSE_CLONES is true if we
3261 should traverse transactional clones. */
3264 expand_regions_1 (struct tm_region
*region
,
3265 void *(*callback
)(struct tm_region
*, void *),
3267 bool traverse_clones
)
3269 void *retval
= NULL
;
3270 if (region
->exit_blocks
3271 || (traverse_clones
&& decl_is_tm_clone (current_function_decl
)))
3273 retval
= callback (region
, data
);
3279 retval
= expand_regions (region
->inner
, callback
, data
, traverse_clones
);
3286 /* Traverse the regions enclosed and including REGION. Execute
3287 CALLBACK for each region, passing DATA. CALLBACK returns NULL to
3288 continue the traversal, otherwise a non-null value which this
3289 function will return as well. TRAVERSE_CLONES is true if we should
3290 traverse transactional clones. */
3293 expand_regions (struct tm_region
*region
,
3294 void *(*callback
)(struct tm_region
*, void *),
3296 bool traverse_clones
)
3298 void *retval
= NULL
;
3301 retval
= expand_regions_1 (region
, callback
, data
, traverse_clones
);
3304 region
= region
->next
;
3310 /* A unique TM memory operation. */
3313 /* Unique ID that all memory operations to the same location have. */
3314 unsigned int value_id
;
3315 /* Address of load/store. */
3319 /* TM memory operation hashtable helpers. */
3321 struct tm_memop_hasher
: free_ptr_hash
<tm_memop
>
3323 static inline hashval_t
hash (const tm_memop
*);
3324 static inline bool equal (const tm_memop
*, const tm_memop
*);
3327 /* Htab support. Return a hash value for a `tm_memop'. */
3329 tm_memop_hasher::hash (const tm_memop
*mem
)
3331 tree addr
= mem
->addr
;
3332 /* We drill down to the SSA_NAME/DECL for the hash, but equality is
3333 actually done with operand_equal_p (see tm_memop_eq). */
3334 if (TREE_CODE (addr
) == ADDR_EXPR
)
3335 addr
= TREE_OPERAND (addr
, 0);
3336 return iterative_hash_expr (addr
, 0);
3339 /* Htab support. Return true if two tm_memop's are the same. */
3341 tm_memop_hasher::equal (const tm_memop
*mem1
, const tm_memop
*mem2
)
3343 return operand_equal_p (mem1
->addr
, mem2
->addr
, 0);
3346 /* Sets for solving data flow equations in the memory optimization pass. */
3347 struct tm_memopt_bitmaps
3349 /* Stores available to this BB upon entry. Basically, stores that
3350 dominate this BB. */
3351 bitmap store_avail_in
;
3352 /* Stores available at the end of this BB. */
3353 bitmap store_avail_out
;
3354 bitmap store_antic_in
;
3355 bitmap store_antic_out
;
3356 /* Reads available to this BB upon entry. Basically, reads that
3357 dominate this BB. */
3358 bitmap read_avail_in
;
3359 /* Reads available at the end of this BB. */
3360 bitmap read_avail_out
;
3361 /* Reads performed in this BB. */
3363 /* Writes performed in this BB. */
3366 /* Temporary storage for pass. */
3367 /* Is the current BB in the worklist? */
3368 bool avail_in_worklist_p
;
3369 /* Have we visited this BB? */
3373 static bitmap_obstack tm_memopt_obstack
;
3375 /* Unique counter for TM loads and stores. Loads and stores of the
3376 same address get the same ID. */
3377 static unsigned int tm_memopt_value_id
;
3378 static hash_table
<tm_memop_hasher
> *tm_memopt_value_numbers
;
3380 #define STORE_AVAIL_IN(BB) \
3381 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in
3382 #define STORE_AVAIL_OUT(BB) \
3383 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out
3384 #define STORE_ANTIC_IN(BB) \
3385 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in
3386 #define STORE_ANTIC_OUT(BB) \
3387 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out
3388 #define READ_AVAIL_IN(BB) \
3389 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in
3390 #define READ_AVAIL_OUT(BB) \
3391 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out
3392 #define READ_LOCAL(BB) \
3393 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local
3394 #define STORE_LOCAL(BB) \
3395 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local
3396 #define AVAIL_IN_WORKLIST_P(BB) \
3397 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p
3398 #define BB_VISITED_P(BB) \
3399 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p
3401 /* Given a TM load/store in STMT, return the value number for the address
3405 tm_memopt_value_number (gimple
*stmt
, enum insert_option op
)
3407 struct tm_memop tmpmem
, *mem
;
3410 gcc_assert (is_tm_load (stmt
) || is_tm_store (stmt
));
3411 tmpmem
.addr
= gimple_call_arg (stmt
, 0);
3412 slot
= tm_memopt_value_numbers
->find_slot (&tmpmem
, op
);
3415 else if (op
== INSERT
)
3417 mem
= XNEW (struct tm_memop
);
3419 mem
->value_id
= tm_memopt_value_id
++;
3420 mem
->addr
= tmpmem
.addr
;
3424 return mem
->value_id
;
3427 /* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */
3430 tm_memopt_accumulate_memops (basic_block bb
)
3432 gimple_stmt_iterator gsi
;
3434 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3436 gimple
*stmt
= gsi_stmt (gsi
);
3440 if (is_tm_store (stmt
))
3441 bits
= STORE_LOCAL (bb
);
3442 else if (is_tm_load (stmt
))
3443 bits
= READ_LOCAL (bb
);
3447 loc
= tm_memopt_value_number (stmt
, INSERT
);
3448 bitmap_set_bit (bits
, loc
);
3451 fprintf (dump_file
, "TM memopt (%s): value num=%d, BB=%d, addr=",
3452 is_tm_load (stmt
) ? "LOAD" : "STORE", loc
,
3453 gimple_bb (stmt
)->index
);
3454 print_generic_expr (dump_file
, gimple_call_arg (stmt
, 0), 0);
3455 fprintf (dump_file
, "\n");
3460 /* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */
3463 dump_tm_memopt_set (const char *set_name
, bitmap bits
)
3467 const char *comma
= "";
3469 fprintf (dump_file
, "TM memopt: %s: [", set_name
);
3470 EXECUTE_IF_SET_IN_BITMAP (bits
, 0, i
, bi
)
3472 hash_table
<tm_memop_hasher
>::iterator hi
;
3473 struct tm_memop
*mem
= NULL
;
3475 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */
3476 FOR_EACH_HASH_TABLE_ELEMENT (*tm_memopt_value_numbers
, mem
, tm_memop_t
, hi
)
3477 if (mem
->value_id
== i
)
3479 gcc_assert (mem
->value_id
== i
);
3480 fprintf (dump_file
, "%s", comma
);
3482 print_generic_expr (dump_file
, mem
->addr
, 0);
3484 fprintf (dump_file
, "]\n");
3487 /* Prettily dump all of the memopt sets in BLOCKS. */
3490 dump_tm_memopt_sets (vec
<basic_block
> blocks
)
3495 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3497 fprintf (dump_file
, "------------BB %d---------\n", bb
->index
);
3498 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb
));
3499 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb
));
3500 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb
));
3501 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb
));
3502 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb
));
3503 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb
));
3507 /* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */
3510 tm_memopt_compute_avin (basic_block bb
)
3515 /* Seed with the AVOUT of any predecessor. */
3516 for (ix
= 0; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3518 e
= EDGE_PRED (bb
, ix
);
3519 /* Make sure we have already visited this BB, and is thus
3522 If e->src->aux is NULL, this predecessor is actually on an
3523 enclosing transaction. We only care about the current
3524 transaction, so ignore it. */
3525 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3527 bitmap_copy (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3528 bitmap_copy (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3533 for (; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3535 e
= EDGE_PRED (bb
, ix
);
3536 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3538 bitmap_and_into (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3539 bitmap_and_into (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3543 BB_VISITED_P (bb
) = true;
3546 /* Compute the STORE_ANTIC_IN for the basic block BB. */
3549 tm_memopt_compute_antin (basic_block bb
)
3554 /* Seed with the ANTIC_OUT of any successor. */
3555 for (ix
= 0; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3557 e
= EDGE_SUCC (bb
, ix
);
3558 /* Make sure we have already visited this BB, and is thus
3560 if (BB_VISITED_P (e
->dest
))
3562 bitmap_copy (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3567 for (; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3569 e
= EDGE_SUCC (bb
, ix
);
3570 if (BB_VISITED_P (e
->dest
))
3571 bitmap_and_into (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3574 BB_VISITED_P (bb
) = true;
3577 /* Compute the AVAIL sets for every basic block in BLOCKS.
3579 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows:
3581 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb])
3582 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors])
3584 This is basically what we do in lcm's compute_available(), but here
3585 we calculate two sets of sets (one for STOREs and one for READs),
3586 and we work on a region instead of the entire CFG.
3588 REGION is the TM region.
3589 BLOCKS are the basic blocks in the region. */
3592 tm_memopt_compute_available (struct tm_region
*region
,
3593 vec
<basic_block
> blocks
)
3596 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3597 unsigned int qlen
, i
;
3601 /* Allocate a worklist array/queue. Entries are only added to the
3602 list if they were not already on the list. So the size is
3603 bounded by the number of basic blocks in the region. */
3604 qlen
= blocks
.length () - 1;
3605 qin
= qout
= worklist
=
3606 XNEWVEC (basic_block
, qlen
);
3608 /* Put every block in the region on the worklist. */
3609 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3611 /* Seed AVAIL_OUT with the LOCAL set. */
3612 bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_LOCAL (bb
));
3613 bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_LOCAL (bb
));
3615 AVAIL_IN_WORKLIST_P (bb
) = true;
3616 /* No need to insert the entry block, since it has an AVIN of
3617 null, and an AVOUT that has already been seeded in. */
3618 if (bb
!= region
->entry_block
)
3622 /* The entry block has been initialized with the local sets. */
3623 BB_VISITED_P (region
->entry_block
) = true;
3626 qend
= &worklist
[qlen
];
3628 /* Iterate until the worklist is empty. */
3631 /* Take the first entry off the worklist. */
3638 /* This block can be added to the worklist again if necessary. */
3639 AVAIL_IN_WORKLIST_P (bb
) = false;
3640 tm_memopt_compute_avin (bb
);
3642 /* Note: We do not add the LOCAL sets here because we already
3643 seeded the AVAIL_OUT sets with them. */
3644 changed
= bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_AVAIL_IN (bb
));
3645 changed
|= bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_AVAIL_IN (bb
));
3647 && (region
->exit_blocks
== NULL
3648 || !bitmap_bit_p (region
->exit_blocks
, bb
->index
)))
3649 /* If the out state of this block changed, then we need to add
3650 its successors to the worklist if they are not already in. */
3651 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3652 if (!AVAIL_IN_WORKLIST_P (e
->dest
)
3653 && e
->dest
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
3656 AVAIL_IN_WORKLIST_P (e
->dest
) = true;
3667 dump_tm_memopt_sets (blocks
);
3670 /* Compute ANTIC sets for every basic block in BLOCKS.
3672 We compute STORE_ANTIC_OUT as follows:
3674 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb])
3675 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors])
3677 REGION is the TM region.
3678 BLOCKS are the basic blocks in the region. */
3681 tm_memopt_compute_antic (struct tm_region
*region
,
3682 vec
<basic_block
> blocks
)
3685 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3690 /* Allocate a worklist array/queue. Entries are only added to the
3691 list if they were not already on the list. So the size is
3692 bounded by the number of basic blocks in the region. */
3693 qin
= qout
= worklist
= XNEWVEC (basic_block
, blocks
.length ());
3695 for (qlen
= 0, i
= blocks
.length () - 1; i
>= 0; --i
)
3699 /* Seed ANTIC_OUT with the LOCAL set. */
3700 bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_LOCAL (bb
));
3702 /* Put every block in the region on the worklist. */
3703 AVAIL_IN_WORKLIST_P (bb
) = true;
3704 /* No need to insert exit blocks, since their ANTIC_IN is NULL,
3705 and their ANTIC_OUT has already been seeded in. */
3706 if (region
->exit_blocks
3707 && !bitmap_bit_p (region
->exit_blocks
, bb
->index
))
3714 /* The exit blocks have been initialized with the local sets. */
3715 if (region
->exit_blocks
)
3719 EXECUTE_IF_SET_IN_BITMAP (region
->exit_blocks
, 0, i
, bi
)
3720 BB_VISITED_P (BASIC_BLOCK_FOR_FN (cfun
, i
)) = true;
3724 qend
= &worklist
[qlen
];
3726 /* Iterate until the worklist is empty. */
3729 /* Take the first entry off the worklist. */
3736 /* This block can be added to the worklist again if necessary. */
3737 AVAIL_IN_WORKLIST_P (bb
) = false;
3738 tm_memopt_compute_antin (bb
);
3740 /* Note: We do not add the LOCAL sets here because we already
3741 seeded the ANTIC_OUT sets with them. */
3742 if (bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_ANTIC_IN (bb
))
3743 && bb
!= region
->entry_block
)
3744 /* If the out state of this block changed, then we need to add
3745 its predecessors to the worklist if they are not already in. */
3746 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
3747 if (!AVAIL_IN_WORKLIST_P (e
->src
))
3750 AVAIL_IN_WORKLIST_P (e
->src
) = true;
3761 dump_tm_memopt_sets (blocks
);
3764 /* Offsets of load variants from TM_LOAD. For example,
3765 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*.
3766 See gtm-builtins.def. */
3767 #define TRANSFORM_RAR 1
3768 #define TRANSFORM_RAW 2
3769 #define TRANSFORM_RFW 3
3770 /* Offsets of store variants from TM_STORE. */
3771 #define TRANSFORM_WAR 1
3772 #define TRANSFORM_WAW 2
3774 /* Inform about a load/store optimization. */
3777 dump_tm_memopt_transform (gimple
*stmt
)
3781 fprintf (dump_file
, "TM memopt: transforming: ");
3782 print_gimple_stmt (dump_file
, stmt
, 0, 0);
3783 fprintf (dump_file
, "\n");
3787 /* Perform a read/write optimization. Replaces the TM builtin in STMT
3788 by a builtin that is OFFSET entries down in the builtins table in
3789 gtm-builtins.def. */
3792 tm_memopt_transform_stmt (unsigned int offset
,
3794 gimple_stmt_iterator
*gsi
)
3796 tree fn
= gimple_call_fn (stmt
);
3797 gcc_assert (TREE_CODE (fn
) == ADDR_EXPR
);
3798 TREE_OPERAND (fn
, 0)
3799 = builtin_decl_explicit ((enum built_in_function
)
3800 (DECL_FUNCTION_CODE (TREE_OPERAND (fn
, 0))
3802 gimple_call_set_fn (stmt
, fn
);
3803 gsi_replace (gsi
, stmt
, true);
3804 dump_tm_memopt_transform (stmt
);
3807 /* Perform the actual TM memory optimization transformations in the
3808 basic blocks in BLOCKS. */
3811 tm_memopt_transform_blocks (vec
<basic_block
> blocks
)
3815 gimple_stmt_iterator gsi
;
3817 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3819 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3821 gimple
*stmt
= gsi_stmt (gsi
);
3822 bitmap read_avail
= READ_AVAIL_IN (bb
);
3823 bitmap store_avail
= STORE_AVAIL_IN (bb
);
3824 bitmap store_antic
= STORE_ANTIC_OUT (bb
);
3827 if (is_tm_simple_load (stmt
))
3829 gcall
*call_stmt
= as_a
<gcall
*> (stmt
);
3830 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3831 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3832 tm_memopt_transform_stmt (TRANSFORM_RAW
, call_stmt
, &gsi
);
3833 else if (store_antic
&& bitmap_bit_p (store_antic
, loc
))
3835 tm_memopt_transform_stmt (TRANSFORM_RFW
, call_stmt
, &gsi
);
3836 bitmap_set_bit (store_avail
, loc
);
3838 else if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3839 tm_memopt_transform_stmt (TRANSFORM_RAR
, call_stmt
, &gsi
);
3841 bitmap_set_bit (read_avail
, loc
);
3843 else if (is_tm_simple_store (stmt
))
3845 gcall
*call_stmt
= as_a
<gcall
*> (stmt
);
3846 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3847 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3848 tm_memopt_transform_stmt (TRANSFORM_WAW
, call_stmt
, &gsi
);
3851 if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3852 tm_memopt_transform_stmt (TRANSFORM_WAR
, call_stmt
, &gsi
);
3853 bitmap_set_bit (store_avail
, loc
);
3860 /* Return a new set of bitmaps for a BB. */
3862 static struct tm_memopt_bitmaps
*
3863 tm_memopt_init_sets (void)
3865 struct tm_memopt_bitmaps
*b
3866 = XOBNEW (&tm_memopt_obstack
.obstack
, struct tm_memopt_bitmaps
);
3867 b
->store_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3868 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3869 b
->store_antic_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3870 b
->store_antic_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3871 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3872 b
->read_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3873 b
->read_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3874 b
->read_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3875 b
->store_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3879 /* Free sets computed for each BB. */
3882 tm_memopt_free_sets (vec
<basic_block
> blocks
)
3887 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3891 /* Clear the visited bit for every basic block in BLOCKS. */
3894 tm_memopt_clear_visited (vec
<basic_block
> blocks
)
3899 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3900 BB_VISITED_P (bb
) = false;
3903 /* Replace TM load/stores with hints for the runtime. We handle
3904 things like read-after-write, write-after-read, read-after-read,
3905 read-for-write, etc. */
3908 execute_tm_memopt (void)
3910 struct tm_region
*region
;
3911 vec
<basic_block
> bbs
;
3913 tm_memopt_value_id
= 0;
3914 tm_memopt_value_numbers
= new hash_table
<tm_memop_hasher
> (10);
3916 for (region
= all_tm_regions
; region
; region
= region
->next
)
3918 /* All the TM stores/loads in the current region. */
3922 bitmap_obstack_initialize (&tm_memopt_obstack
);
3924 /* Save all BBs for the current region. */
3925 bbs
= get_tm_region_blocks (region
->entry_block
,
3926 region
->exit_blocks
,
3931 /* Collect all the memory operations. */
3932 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
3934 bb
->aux
= tm_memopt_init_sets ();
3935 tm_memopt_accumulate_memops (bb
);
3938 /* Solve data flow equations and transform each block accordingly. */
3939 tm_memopt_clear_visited (bbs
);
3940 tm_memopt_compute_available (region
, bbs
);
3941 tm_memopt_clear_visited (bbs
);
3942 tm_memopt_compute_antic (region
, bbs
);
3943 tm_memopt_transform_blocks (bbs
);
3945 tm_memopt_free_sets (bbs
);
3947 bitmap_obstack_release (&tm_memopt_obstack
);
3948 tm_memopt_value_numbers
->empty ();
3951 delete tm_memopt_value_numbers
;
3952 tm_memopt_value_numbers
= NULL
;
3958 const pass_data pass_data_tm_memopt
=
3960 GIMPLE_PASS
, /* type */
3961 "tmmemopt", /* name */
3962 OPTGROUP_NONE
, /* optinfo_flags */
3963 TV_TRANS_MEM
, /* tv_id */
3964 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3965 0, /* properties_provided */
3966 0, /* properties_destroyed */
3967 0, /* todo_flags_start */
3968 0, /* todo_flags_finish */
3971 class pass_tm_memopt
: public gimple_opt_pass
3974 pass_tm_memopt (gcc::context
*ctxt
)
3975 : gimple_opt_pass (pass_data_tm_memopt
, ctxt
)
3978 /* opt_pass methods: */
3979 virtual bool gate (function
*) { return flag_tm
&& optimize
> 0; }
3980 virtual unsigned int execute (function
*) { return execute_tm_memopt (); }
3982 }; // class pass_tm_memopt
3987 make_pass_tm_memopt (gcc::context
*ctxt
)
3989 return new pass_tm_memopt (ctxt
);
3993 /* Interprocedual analysis for the creation of transactional clones.
3994 The aim of this pass is to find which functions are referenced in
3995 a non-irrevocable transaction context, and for those over which
3996 we have control (or user directive), create a version of the
3997 function which uses only the transactional interface to reference
3998 protected memories. This analysis proceeds in several steps:
4000 (1) Collect the set of all possible transactional clones:
4002 (a) For all local public functions marked tm_callable, push
4003 it onto the tm_callee queue.
4005 (b) For all local functions, scan for calls in transaction blocks.
4006 Push the caller and callee onto the tm_caller and tm_callee
4007 queues. Count the number of callers for each callee.
4009 (c) For each local function on the callee list, assume we will
4010 create a transactional clone. Push *all* calls onto the
4011 callee queues; count the number of clone callers separately
4012 to the number of original callers.
4014 (2) Propagate irrevocable status up the dominator tree:
4016 (a) Any external function on the callee list that is not marked
4017 tm_callable is irrevocable. Push all callers of such onto
4020 (b) For each function on the worklist, mark each block that
4021 contains an irrevocable call. Use the AND operator to
4022 propagate that mark up the dominator tree.
4024 (c) If we reach the entry block for a possible transactional
4025 clone, then the transactional clone is irrevocable, and
4026 we should not create the clone after all. Push all
4027 callers onto the worklist.
4029 (d) Place tm_irrevocable calls at the beginning of the relevant
4030 blocks. Special case here is the entry block for the entire
4031 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for
4032 the library to begin the region in serial mode. Decrement
4033 the call count for all callees in the irrevocable region.
4035 (3) Create the transactional clones:
4037 Any tm_callee that still has a non-zero call count is cloned.
4040 /* This structure is stored in the AUX field of each cgraph_node. */
4041 struct tm_ipa_cg_data
4043 /* The clone of the function that got created. */
4044 struct cgraph_node
*clone
;
4046 /* The tm regions in the normal function. */
4047 struct tm_region
*all_tm_regions
;
4049 /* The blocks of the normal/clone functions that contain irrevocable
4050 calls, or blocks that are post-dominated by irrevocable calls. */
4051 bitmap irrevocable_blocks_normal
;
4052 bitmap irrevocable_blocks_clone
;
4054 /* The blocks of the normal function that are involved in transactions. */
4055 bitmap transaction_blocks_normal
;
4057 /* The number of callers to the transactional clone of this function
4058 from normal and transactional clones respectively. */
4059 unsigned tm_callers_normal
;
4060 unsigned tm_callers_clone
;
4062 /* True if all calls to this function's transactional clone
4063 are irrevocable. Also automatically true if the function
4064 has no transactional clone. */
4065 bool is_irrevocable
;
4067 /* Flags indicating the presence of this function in various queues. */
4068 bool in_callee_queue
;
4071 /* Flags indicating the kind of scan desired while in the worklist. */
4072 bool want_irr_scan_normal
;
4075 typedef vec
<cgraph_node
*> cgraph_node_queue
;
4077 /* Return the ipa data associated with NODE, allocating zeroed memory
4078 if necessary. TRAVERSE_ALIASES is true if we must traverse aliases
4079 and set *NODE accordingly. */
4081 static struct tm_ipa_cg_data
*
4082 get_cg_data (struct cgraph_node
**node
, bool traverse_aliases
)
4084 struct tm_ipa_cg_data
*d
;
4086 if (traverse_aliases
&& (*node
)->alias
)
4087 *node
= (*node
)->get_alias_target ();
4089 d
= (struct tm_ipa_cg_data
*) (*node
)->aux
;
4093 d
= (struct tm_ipa_cg_data
*)
4094 obstack_alloc (&tm_obstack
.obstack
, sizeof (*d
));
4095 (*node
)->aux
= (void *) d
;
4096 memset (d
, 0, sizeof (*d
));
4102 /* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that
4103 it is already present. */
4106 maybe_push_queue (struct cgraph_node
*node
,
4107 cgraph_node_queue
*queue_p
, bool *in_queue_p
)
4112 queue_p
->safe_push (node
);
4116 /* Duplicate the basic blocks in QUEUE for use in the uninstrumented
4117 code path. QUEUE are the basic blocks inside the transaction
4118 represented in REGION.
4120 Later in split_code_paths() we will add the conditional to choose
4121 between the two alternatives. */
4124 ipa_uninstrument_transaction (struct tm_region
*region
,
4125 vec
<basic_block
> queue
)
4127 gimple
*transaction
= region
->transaction_stmt
;
4128 basic_block transaction_bb
= gimple_bb (transaction
);
4129 int n
= queue
.length ();
4130 basic_block
*new_bbs
= XNEWVEC (basic_block
, n
);
4132 copy_bbs (queue
.address (), n
, new_bbs
, NULL
, 0, NULL
, NULL
, transaction_bb
,
4134 edge e
= make_edge (transaction_bb
, new_bbs
[0], EDGE_TM_UNINSTRUMENTED
);
4135 add_phi_args_after_copy (new_bbs
, n
, e
);
4137 // Now we will have a GIMPLE_ATOMIC with 3 possible edges out of it.
4138 // a) EDGE_FALLTHRU into the transaction
4139 // b) EDGE_TM_ABORT out of the transaction
4140 // c) EDGE_TM_UNINSTRUMENTED into the uninstrumented blocks.
4145 /* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone.
4146 Queue all callees within block BB. */
4149 ipa_tm_scan_calls_block (cgraph_node_queue
*callees_p
,
4150 basic_block bb
, bool for_clone
)
4152 gimple_stmt_iterator gsi
;
4154 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4156 gimple
*stmt
= gsi_stmt (gsi
);
4157 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4159 tree fndecl
= gimple_call_fndecl (stmt
);
4162 struct tm_ipa_cg_data
*d
;
4164 struct cgraph_node
*node
;
4166 if (is_tm_ending_fndecl (fndecl
))
4168 if (find_tm_replacement_function (fndecl
))
4171 node
= cgraph_node::get (fndecl
);
4172 gcc_assert (node
!= NULL
);
4173 d
= get_cg_data (&node
, true);
4175 pcallers
= (for_clone
? &d
->tm_callers_clone
4176 : &d
->tm_callers_normal
);
4179 maybe_push_queue (node
, callees_p
, &d
->in_callee_queue
);
4185 /* Scan all calls in NODE that are within a transaction region,
4186 and push the resulting nodes into the callee queue. */
4189 ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data
*d
,
4190 cgraph_node_queue
*callees_p
)
4192 struct tm_region
*r
;
4194 d
->transaction_blocks_normal
= BITMAP_ALLOC (&tm_obstack
);
4195 d
->all_tm_regions
= all_tm_regions
;
4197 for (r
= all_tm_regions
; r
; r
= r
->next
)
4199 vec
<basic_block
> bbs
;
4203 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
, NULL
,
4204 d
->transaction_blocks_normal
, false);
4206 // Generate the uninstrumented code path for this transaction.
4207 ipa_uninstrument_transaction (r
, bbs
);
4209 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
4210 ipa_tm_scan_calls_block (callees_p
, bb
, false);
4215 // ??? copy_bbs should maintain cgraph edges for the blocks as it is
4216 // copying them, rather than forcing us to do this externally.
4217 cgraph_edge::rebuild_edges ();
4219 // ??? In ipa_uninstrument_transaction we don't try to update dominators
4220 // because copy_bbs doesn't return a VEC like iterate_fix_dominators expects.
4221 // Instead, just release dominators here so update_ssa recomputes them.
4222 free_dominance_info (CDI_DOMINATORS
);
4224 // When building the uninstrumented code path, copy_bbs will have invoked
4225 // create_new_def_for starting an "ssa update context". There is only one
4226 // instance of this context, so resolve ssa updates before moving on to
4227 // the next function.
4228 update_ssa (TODO_update_ssa
);
4231 /* Scan all calls in NODE as if this is the transactional clone,
4232 and push the destinations into the callee queue. */
4235 ipa_tm_scan_calls_clone (struct cgraph_node
*node
,
4236 cgraph_node_queue
*callees_p
)
4238 struct function
*fn
= DECL_STRUCT_FUNCTION (node
->decl
);
4241 FOR_EACH_BB_FN (bb
, fn
)
4242 ipa_tm_scan_calls_block (callees_p
, bb
, true);
4245 /* The function NODE has been detected to be irrevocable. Push all
4246 of its callers onto WORKLIST for the purpose of re-scanning them. */
4249 ipa_tm_note_irrevocable (struct cgraph_node
*node
,
4250 cgraph_node_queue
*worklist_p
)
4252 struct tm_ipa_cg_data
*d
= get_cg_data (&node
, true);
4253 struct cgraph_edge
*e
;
4255 d
->is_irrevocable
= true;
4257 for (e
= node
->callers
; e
; e
= e
->next_caller
)
4260 struct cgraph_node
*caller
;
4262 /* Don't examine recursive calls. */
4263 if (e
->caller
== node
)
4265 /* Even if we think we can go irrevocable, believe the user
4267 if (is_tm_safe_or_pure (e
->caller
->decl
))
4271 d
= get_cg_data (&caller
, true);
4273 /* Check if the callee is in a transactional region. If so,
4274 schedule the function for normal re-scan as well. */
4275 bb
= gimple_bb (e
->call_stmt
);
4276 gcc_assert (bb
!= NULL
);
4277 if (d
->transaction_blocks_normal
4278 && bitmap_bit_p (d
->transaction_blocks_normal
, bb
->index
))
4279 d
->want_irr_scan_normal
= true;
4281 maybe_push_queue (caller
, worklist_p
, &d
->in_worklist
);
4285 /* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement
4286 within the block is irrevocable. */
4289 ipa_tm_scan_irr_block (basic_block bb
)
4291 gimple_stmt_iterator gsi
;
4294 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4296 gimple
*stmt
= gsi_stmt (gsi
);
4297 switch (gimple_code (stmt
))
4300 if (gimple_assign_single_p (stmt
))
4302 tree lhs
= gimple_assign_lhs (stmt
);
4303 tree rhs
= gimple_assign_rhs1 (stmt
);
4304 if (volatile_lvalue_p (lhs
) || volatile_lvalue_p (rhs
))
4311 tree lhs
= gimple_call_lhs (stmt
);
4312 if (lhs
&& volatile_lvalue_p (lhs
))
4315 if (is_tm_pure_call (stmt
))
4318 fn
= gimple_call_fn (stmt
);
4320 /* Functions with the attribute are by definition irrevocable. */
4321 if (is_tm_irrevocable (fn
))
4324 /* For direct function calls, go ahead and check for replacement
4325 functions, or transitive irrevocable functions. For indirect
4326 functions, we'll ask the runtime. */
4327 if (TREE_CODE (fn
) == ADDR_EXPR
)
4329 struct tm_ipa_cg_data
*d
;
4330 struct cgraph_node
*node
;
4332 fn
= TREE_OPERAND (fn
, 0);
4333 if (is_tm_ending_fndecl (fn
))
4335 if (find_tm_replacement_function (fn
))
4338 node
= cgraph_node::get (fn
);
4339 d
= get_cg_data (&node
, true);
4341 /* Return true if irrevocable, but above all, believe
4343 if (d
->is_irrevocable
4344 && !is_tm_safe_or_pure (fn
))
4351 /* ??? The Approved Method of indicating that an inline
4352 assembly statement is not relevant to the transaction
4353 is to wrap it in a __tm_waiver block. This is not
4354 yet implemented, so we can't check for it. */
4355 if (is_tm_safe (current_function_decl
))
4357 tree t
= build1 (NOP_EXPR
, void_type_node
, size_zero_node
);
4358 SET_EXPR_LOCATION (t
, gimple_location (stmt
));
4359 error ("%Kasm not allowed in %<transaction_safe%> function", t
);
4371 /* For each of the blocks seeded witin PQUEUE, walk the CFG looking
4372 for new irrevocable blocks, marking them in NEW_IRR. Don't bother
4373 scanning past OLD_IRR or EXIT_BLOCKS. */
4376 ipa_tm_scan_irr_blocks (vec
<basic_block
> *pqueue
, bitmap new_irr
,
4377 bitmap old_irr
, bitmap exit_blocks
)
4379 bool any_new_irr
= false;
4382 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4386 basic_block bb
= pqueue
->pop ();
4388 /* Don't re-scan blocks we know already are irrevocable. */
4389 if (old_irr
&& bitmap_bit_p (old_irr
, bb
->index
))
4392 if (ipa_tm_scan_irr_block (bb
))
4394 bitmap_set_bit (new_irr
, bb
->index
);
4397 else if (exit_blocks
== NULL
|| !bitmap_bit_p (exit_blocks
, bb
->index
))
4399 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4400 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
4402 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
4403 pqueue
->safe_push (e
->dest
);
4407 while (!pqueue
->is_empty ());
4409 BITMAP_FREE (visited_blocks
);
4414 /* Propagate the irrevocable property both up and down the dominator tree.
4415 BB is the current block being scanned; EXIT_BLOCKS are the edges of the
4416 TM regions; OLD_IRR are the results of a previous scan of the dominator
4417 tree which has been fully propagated; NEW_IRR is the set of new blocks
4418 which are gaining the irrevocable property during the current scan. */
4421 ipa_tm_propagate_irr (basic_block entry_block
, bitmap new_irr
,
4422 bitmap old_irr
, bitmap exit_blocks
)
4424 vec
<basic_block
> bbs
;
4425 bitmap all_region_blocks
;
4427 /* If this block is in the old set, no need to rescan. */
4428 if (old_irr
&& bitmap_bit_p (old_irr
, entry_block
->index
))
4431 all_region_blocks
= BITMAP_ALLOC (&tm_obstack
);
4432 bbs
= get_tm_region_blocks (entry_block
, exit_blocks
, NULL
,
4433 all_region_blocks
, false);
4436 basic_block bb
= bbs
.pop ();
4437 bool this_irr
= bitmap_bit_p (new_irr
, bb
->index
);
4438 bool all_son_irr
= false;
4442 /* Propagate up. If my children are, I am too, but we must have
4443 at least one child that is. */
4446 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4448 if (!bitmap_bit_p (new_irr
, e
->dest
->index
))
4450 all_son_irr
= false;
4458 /* Add block to new_irr if it hasn't already been processed. */
4459 if (!old_irr
|| !bitmap_bit_p (old_irr
, bb
->index
))
4461 bitmap_set_bit (new_irr
, bb
->index
);
4467 /* Propagate down to everyone we immediately dominate. */
4471 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
4473 son
= next_dom_son (CDI_DOMINATORS
, son
))
4475 /* Make sure block is actually in a TM region, and it
4476 isn't already in old_irr. */
4477 if ((!old_irr
|| !bitmap_bit_p (old_irr
, son
->index
))
4478 && bitmap_bit_p (all_region_blocks
, son
->index
))
4479 bitmap_set_bit (new_irr
, son
->index
);
4483 while (!bbs
.is_empty ());
4485 BITMAP_FREE (all_region_blocks
);
4490 ipa_tm_decrement_clone_counts (basic_block bb
, bool for_clone
)
4492 gimple_stmt_iterator gsi
;
4494 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4496 gimple
*stmt
= gsi_stmt (gsi
);
4497 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4499 tree fndecl
= gimple_call_fndecl (stmt
);
4502 struct tm_ipa_cg_data
*d
;
4504 struct cgraph_node
*tnode
;
4506 if (is_tm_ending_fndecl (fndecl
))
4508 if (find_tm_replacement_function (fndecl
))
4511 tnode
= cgraph_node::get (fndecl
);
4512 d
= get_cg_data (&tnode
, true);
4514 pcallers
= (for_clone
? &d
->tm_callers_clone
4515 : &d
->tm_callers_normal
);
4517 gcc_assert (*pcallers
> 0);
4524 /* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions,
4525 as well as other irrevocable actions such as inline assembly. Mark all
4526 such blocks as irrevocable and decrement the number of calls to
4527 transactional clones. Return true if, for the transactional clone, the
4528 entire function is irrevocable. */
4531 ipa_tm_scan_irr_function (struct cgraph_node
*node
, bool for_clone
)
4533 struct tm_ipa_cg_data
*d
;
4534 bitmap new_irr
, old_irr
;
4537 /* Builtin operators (operator new, and such). */
4538 if (DECL_STRUCT_FUNCTION (node
->decl
) == NULL
4539 || DECL_STRUCT_FUNCTION (node
->decl
)->cfg
== NULL
)
4542 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
4543 calculate_dominance_info (CDI_DOMINATORS
);
4545 d
= get_cg_data (&node
, true);
4546 auto_vec
<basic_block
, 10> queue
;
4547 new_irr
= BITMAP_ALLOC (&tm_obstack
);
4549 /* Scan each tm region, propagating irrevocable status through the tree. */
4552 old_irr
= d
->irrevocable_blocks_clone
;
4553 queue
.quick_push (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)));
4554 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
, NULL
))
4556 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
4559 ret
= bitmap_bit_p (new_irr
,
4560 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
))->index
);
4565 struct tm_region
*region
;
4567 old_irr
= d
->irrevocable_blocks_normal
;
4568 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
4570 queue
.quick_push (region
->entry_block
);
4571 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
,
4572 region
->exit_blocks
))
4573 ipa_tm_propagate_irr (region
->entry_block
, new_irr
, old_irr
,
4574 region
->exit_blocks
);
4578 /* If we found any new irrevocable blocks, reduce the call count for
4579 transactional clones within the irrevocable blocks. Save the new
4580 set of irrevocable blocks for next time. */
4581 if (!bitmap_empty_p (new_irr
))
4583 bitmap_iterator bmi
;
4586 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4587 ipa_tm_decrement_clone_counts (BASIC_BLOCK_FOR_FN (cfun
, i
),
4592 bitmap_ior_into (old_irr
, new_irr
);
4593 BITMAP_FREE (new_irr
);
4596 d
->irrevocable_blocks_clone
= new_irr
;
4598 d
->irrevocable_blocks_normal
= new_irr
;
4600 if (dump_file
&& new_irr
)
4603 bitmap_iterator bmi
;
4606 dname
= lang_hooks
.decl_printable_name (current_function_decl
, 2);
4607 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4608 fprintf (dump_file
, "%s: bb %d goes irrevocable\n", dname
, i
);
4612 BITMAP_FREE (new_irr
);
4619 /* Return true if, for the transactional clone of NODE, any call
4620 may enter irrevocable mode. */
4623 ipa_tm_mayenterirr_function (struct cgraph_node
*node
)
4625 struct tm_ipa_cg_data
*d
;
4629 d
= get_cg_data (&node
, true);
4631 flags
= flags_from_decl_or_type (decl
);
4633 /* Handle some TM builtins. Ordinarily these aren't actually generated
4634 at this point, but handling these functions when written in by the
4635 user makes it easier to build unit tests. */
4636 if (flags
& ECF_TM_BUILTIN
)
4639 /* Filter out all functions that are marked. */
4640 if (flags
& ECF_TM_PURE
)
4642 if (is_tm_safe (decl
))
4644 if (is_tm_irrevocable (decl
))
4646 if (is_tm_callable (decl
))
4648 if (find_tm_replacement_function (decl
))
4651 /* If we aren't seeing the final version of the function we don't
4652 know what it will contain at runtime. */
4653 if (node
->get_availability () < AVAIL_AVAILABLE
)
4656 /* If the function must go irrevocable, then of course true. */
4657 if (d
->is_irrevocable
)
4660 /* If there are any blocks marked irrevocable, then the function
4661 as a whole may enter irrevocable. */
4662 if (d
->irrevocable_blocks_clone
)
4665 /* We may have previously marked this function as tm_may_enter_irr;
4666 see pass_diagnose_tm_blocks. */
4667 if (node
->local
.tm_may_enter_irr
)
4670 /* Recurse on the main body for aliases. In general, this will
4671 result in one of the bits above being set so that we will not
4672 have to recurse next time. */
4674 return ipa_tm_mayenterirr_function (cgraph_node::get (node
->thunk
.alias
));
4676 /* What remains is unmarked local functions without items that force
4677 the function to go irrevocable. */
4681 /* Diagnose calls from transaction_safe functions to unmarked
4682 functions that are determined to not be safe. */
4685 ipa_tm_diagnose_tm_safe (struct cgraph_node
*node
)
4687 struct cgraph_edge
*e
;
4689 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4690 if (!is_tm_callable (e
->callee
->decl
)
4691 && e
->callee
->local
.tm_may_enter_irr
)
4692 error_at (gimple_location (e
->call_stmt
),
4693 "unsafe function call %qD within "
4694 "%<transaction_safe%> function", e
->callee
->decl
);
4697 /* Diagnose call from atomic transactions to unmarked functions
4698 that are determined to not be safe. */
4701 ipa_tm_diagnose_transaction (struct cgraph_node
*node
,
4702 struct tm_region
*all_tm_regions
)
4704 struct tm_region
*r
;
4706 for (r
= all_tm_regions
; r
; r
= r
->next
)
4707 if (gimple_transaction_subcode (r
->get_transaction_stmt ())
4710 /* Atomic transactions can be nested inside relaxed. */
4712 ipa_tm_diagnose_transaction (node
, r
->inner
);
4716 vec
<basic_block
> bbs
;
4717 gimple_stmt_iterator gsi
;
4721 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
,
4722 r
->irr_blocks
, NULL
, false);
4724 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
4725 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4727 gimple
*stmt
= gsi_stmt (gsi
);
4730 if (gimple_code (stmt
) == GIMPLE_ASM
)
4732 error_at (gimple_location (stmt
),
4733 "asm not allowed in atomic transaction");
4737 if (!is_gimple_call (stmt
))
4739 fndecl
= gimple_call_fndecl (stmt
);
4741 /* Indirect function calls have been diagnosed already. */
4745 /* Stop at the end of the transaction. */
4746 if (is_tm_ending_fndecl (fndecl
))
4748 if (bitmap_bit_p (r
->exit_blocks
, bb
->index
))
4753 /* Marked functions have been diagnosed already. */
4754 if (is_tm_pure_call (stmt
))
4756 if (is_tm_callable (fndecl
))
4759 if (cgraph_node::local_info (fndecl
)->tm_may_enter_irr
)
4760 error_at (gimple_location (stmt
),
4761 "unsafe function call %qD within "
4762 "atomic transaction", fndecl
);
4769 /* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in
4770 OLD_DECL. The returned value is a freshly malloced pointer that
4771 should be freed by the caller. */
4774 tm_mangle (tree old_asm_id
)
4776 const char *old_asm_name
;
4779 struct demangle_component
*dc
;
4782 /* Determine if the symbol is already a valid C++ mangled name. Do this
4783 even for C, which might be interfacing with C++ code via appropriately
4784 ugly identifiers. */
4785 /* ??? We could probably do just as well checking for "_Z" and be done. */
4786 old_asm_name
= IDENTIFIER_POINTER (old_asm_id
);
4787 dc
= cplus_demangle_v3_components (old_asm_name
, DMGL_NO_OPTS
, &alloc
);
4794 sprintf (length
, "%u", IDENTIFIER_LENGTH (old_asm_id
));
4795 tm_name
= concat ("_ZGTt", length
, old_asm_name
, NULL
);
4799 old_asm_name
+= 2; /* Skip _Z */
4803 case DEMANGLE_COMPONENT_TRANSACTION_CLONE
:
4804 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE
:
4805 /* Don't play silly games, you! */
4808 case DEMANGLE_COMPONENT_HIDDEN_ALIAS
:
4809 /* I'd really like to know if we can ever be passed one of
4810 these from the C++ front end. The Logical Thing would
4811 seem that hidden-alias should be outer-most, so that we
4812 get hidden-alias of a transaction-clone and not vice-versa. */
4820 tm_name
= concat ("_ZGTt", old_asm_name
, NULL
);
4824 new_asm_id
= get_identifier (tm_name
);
4831 ipa_tm_mark_force_output_node (struct cgraph_node
*node
)
4833 node
->mark_force_output ();
4834 node
->analyzed
= true;
4838 ipa_tm_mark_forced_by_abi_node (struct cgraph_node
*node
)
4840 node
->forced_by_abi
= true;
4841 node
->analyzed
= true;
4844 /* Callback data for ipa_tm_create_version_alias. */
4845 struct create_version_alias_info
4847 struct cgraph_node
*old_node
;
4851 /* A subroutine of ipa_tm_create_version, called via
4852 cgraph_for_node_and_aliases. Create new tm clones for each of
4853 the existing aliases. */
4855 ipa_tm_create_version_alias (struct cgraph_node
*node
, void *data
)
4857 struct create_version_alias_info
*info
4858 = (struct create_version_alias_info
*)data
;
4859 tree old_decl
, new_decl
, tm_name
;
4860 struct cgraph_node
*new_node
;
4862 if (!node
->cpp_implicit_alias
)
4865 old_decl
= node
->decl
;
4866 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4867 new_decl
= build_decl (DECL_SOURCE_LOCATION (old_decl
),
4868 TREE_CODE (old_decl
), tm_name
,
4869 TREE_TYPE (old_decl
));
4871 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4872 SET_DECL_RTL (new_decl
, NULL
);
4874 /* Based loosely on C++'s make_alias_for(). */
4875 TREE_PUBLIC (new_decl
) = TREE_PUBLIC (old_decl
);
4876 DECL_CONTEXT (new_decl
) = DECL_CONTEXT (old_decl
);
4877 DECL_LANG_SPECIFIC (new_decl
) = DECL_LANG_SPECIFIC (old_decl
);
4878 TREE_READONLY (new_decl
) = TREE_READONLY (old_decl
);
4879 DECL_EXTERNAL (new_decl
) = 0;
4880 DECL_ARTIFICIAL (new_decl
) = 1;
4881 TREE_ADDRESSABLE (new_decl
) = 1;
4882 TREE_USED (new_decl
) = 1;
4883 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4885 /* Perform the same remapping to the comdat group. */
4886 if (DECL_ONE_ONLY (new_decl
))
4887 varpool_node::get (new_decl
)->set_comdat_group
4888 (tm_mangle (decl_comdat_group_id (old_decl
)));
4890 new_node
= cgraph_node::create_same_body_alias (new_decl
, info
->new_decl
);
4891 new_node
->tm_clone
= true;
4892 new_node
->externally_visible
= info
->old_node
->externally_visible
;
4893 new_node
->no_reorder
= info
->old_node
->no_reorder
;
4894 /* ?? Do not traverse aliases here. */
4895 get_cg_data (&node
, false)->clone
= new_node
;
4897 record_tm_clone_pair (old_decl
, new_decl
);
4899 if (info
->old_node
->force_output
4900 || info
->old_node
->ref_list
.first_referring ())
4901 ipa_tm_mark_force_output_node (new_node
);
4902 if (info
->old_node
->forced_by_abi
)
4903 ipa_tm_mark_forced_by_abi_node (new_node
);
4907 /* Create a copy of the function (possibly declaration only) of OLD_NODE,
4908 appropriate for the transactional clone. */
4911 ipa_tm_create_version (struct cgraph_node
*old_node
)
4913 tree new_decl
, old_decl
, tm_name
;
4914 struct cgraph_node
*new_node
;
4916 old_decl
= old_node
->decl
;
4917 new_decl
= copy_node (old_decl
);
4919 /* DECL_ASSEMBLER_NAME needs to be set before we call
4920 cgraph_copy_node_for_versioning below, because cgraph_node will
4921 fill the assembler_name_hash. */
4922 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4923 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4924 SET_DECL_RTL (new_decl
, NULL
);
4925 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4927 /* Perform the same remapping to the comdat group. */
4928 if (DECL_ONE_ONLY (new_decl
))
4929 varpool_node::get (new_decl
)->set_comdat_group
4930 (tm_mangle (DECL_COMDAT_GROUP (old_decl
)));
4932 gcc_assert (!old_node
->ipa_transforms_to_apply
.exists ());
4933 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
4934 new_node
->local
.local
= false;
4935 new_node
->externally_visible
= old_node
->externally_visible
;
4936 new_node
->lowered
= true;
4937 new_node
->tm_clone
= 1;
4938 if (!old_node
->implicit_section
)
4939 new_node
->set_section (old_node
->get_section ());
4940 get_cg_data (&old_node
, true)->clone
= new_node
;
4942 if (old_node
->get_availability () >= AVAIL_INTERPOSABLE
)
4944 /* Remap extern inline to static inline. */
4945 /* ??? Is it worth trying to use make_decl_one_only? */
4946 if (DECL_DECLARED_INLINE_P (new_decl
) && DECL_EXTERNAL (new_decl
))
4948 DECL_EXTERNAL (new_decl
) = 0;
4949 TREE_PUBLIC (new_decl
) = 0;
4950 DECL_WEAK (new_decl
) = 0;
4953 tree_function_versioning (old_decl
, new_decl
,
4958 record_tm_clone_pair (old_decl
, new_decl
);
4960 symtab
->call_cgraph_insertion_hooks (new_node
);
4961 if (old_node
->force_output
4962 || old_node
->ref_list
.first_referring ())
4963 ipa_tm_mark_force_output_node (new_node
);
4964 if (old_node
->forced_by_abi
)
4965 ipa_tm_mark_forced_by_abi_node (new_node
);
4967 /* Do the same thing, but for any aliases of the original node. */
4969 struct create_version_alias_info data
;
4970 data
.old_node
= old_node
;
4971 data
.new_decl
= new_decl
;
4972 old_node
->call_for_symbol_thunks_and_aliases (ipa_tm_create_version_alias
,
4977 /* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */
4980 ipa_tm_insert_irr_call (struct cgraph_node
*node
, struct tm_region
*region
,
4983 gimple_stmt_iterator gsi
;
4986 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4988 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
),
4989 1, build_int_cst (NULL_TREE
, MODE_SERIALIRREVOCABLE
));
4991 split_block_after_labels (bb
);
4992 gsi
= gsi_after_labels (bb
);
4993 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
4995 node
->create_edge (cgraph_node::get_create
4996 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
)),
4998 compute_call_stmt_bb_frequency (node
->decl
,
5002 /* Construct a call to TM_GETTMCLONE and insert it before GSI. */
5005 ipa_tm_insert_gettmclone_call (struct cgraph_node
*node
,
5006 struct tm_region
*region
,
5007 gimple_stmt_iterator
*gsi
, gcall
*stmt
)
5009 tree gettm_fn
, ret
, old_fn
, callfn
;
5014 old_fn
= gimple_call_fn (stmt
);
5016 if (TREE_CODE (old_fn
) == ADDR_EXPR
)
5018 tree fndecl
= TREE_OPERAND (old_fn
, 0);
5019 tree clone
= get_tm_clone_pair (fndecl
);
5021 /* By transforming the call into a TM_GETTMCLONE, we are
5022 technically taking the address of the original function and
5023 its clone. Explain this so inlining will know this function
5025 cgraph_node::get (fndecl
)->mark_address_taken () ;
5027 cgraph_node::get (clone
)->mark_address_taken ();
5030 safe
= is_tm_safe (TREE_TYPE (old_fn
));
5031 gettm_fn
= builtin_decl_explicit (safe
? BUILT_IN_TM_GETTMCLONE_SAFE
5032 : BUILT_IN_TM_GETTMCLONE_IRR
);
5033 ret
= create_tmp_var (ptr_type_node
);
5036 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
5038 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */
5039 if (TREE_CODE (old_fn
) == OBJ_TYPE_REF
)
5040 old_fn
= OBJ_TYPE_REF_EXPR (old_fn
);
5042 g
= gimple_build_call (gettm_fn
, 1, old_fn
);
5043 ret
= make_ssa_name (ret
, g
);
5044 gimple_call_set_lhs (g
, ret
);
5046 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
5048 node
->create_edge (cgraph_node::get_create (gettm_fn
), g
, 0,
5049 compute_call_stmt_bb_frequency (node
->decl
,
5052 /* Cast return value from tm_gettmclone* into appropriate function
5054 callfn
= create_tmp_var (TREE_TYPE (old_fn
));
5055 g2
= gimple_build_assign (callfn
,
5056 fold_build1 (NOP_EXPR
, TREE_TYPE (callfn
), ret
));
5057 callfn
= make_ssa_name (callfn
, g2
);
5058 gimple_assign_set_lhs (g2
, callfn
);
5059 gsi_insert_before (gsi
, g2
, GSI_SAME_STMT
);
5061 /* ??? This is a hack to preserve the NOTHROW bit on the call,
5062 which we would have derived from the decl. Failure to save
5063 this bit means we might have to split the basic block. */
5064 if (gimple_call_nothrow_p (stmt
))
5065 gimple_call_set_nothrow (stmt
, true);
5067 gimple_call_set_fn (stmt
, callfn
);
5069 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS
5070 for a call statement. Fix it. */
5072 tree lhs
= gimple_call_lhs (stmt
);
5073 tree rettype
= TREE_TYPE (gimple_call_fntype (stmt
));
5075 && !useless_type_conversion_p (TREE_TYPE (lhs
), rettype
))
5079 temp
= create_tmp_reg (rettype
);
5080 gimple_call_set_lhs (stmt
, temp
);
5082 g2
= gimple_build_assign (lhs
,
5083 fold_build1 (VIEW_CONVERT_EXPR
,
5084 TREE_TYPE (lhs
), temp
));
5085 gsi_insert_after (gsi
, g2
, GSI_SAME_STMT
);
5090 cgraph_edge
*e
= cgraph_node::get (current_function_decl
)->get_edge (stmt
);
5091 if (e
&& e
->indirect_info
)
5092 e
->indirect_info
->polymorphic
= false;
5097 /* Helper function for ipa_tm_transform_calls*. Given a call
5098 statement in GSI which resides inside transaction REGION, redirect
5099 the call to either its wrapper function, or its clone. */
5102 ipa_tm_transform_calls_redirect (struct cgraph_node
*node
,
5103 struct tm_region
*region
,
5104 gimple_stmt_iterator
*gsi
,
5105 bool *need_ssa_rename_p
)
5107 gcall
*stmt
= as_a
<gcall
*> (gsi_stmt (*gsi
));
5108 struct cgraph_node
*new_node
;
5109 struct cgraph_edge
*e
= node
->get_edge (stmt
);
5110 tree fndecl
= gimple_call_fndecl (stmt
);
5112 /* For indirect calls, pass the address through the runtime. */
5115 *need_ssa_rename_p
|=
5116 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5120 /* Handle some TM builtins. Ordinarily these aren't actually generated
5121 at this point, but handling these functions when written in by the
5122 user makes it easier to build unit tests. */
5123 if (flags_from_decl_or_type (fndecl
) & ECF_TM_BUILTIN
)
5126 /* Fixup recursive calls inside clones. */
5127 /* ??? Why did cgraph_copy_node_for_versioning update the call edges
5128 for recursion but not update the call statements themselves? */
5129 if (e
->caller
== e
->callee
&& decl_is_tm_clone (current_function_decl
))
5131 gimple_call_set_fndecl (stmt
, current_function_decl
);
5135 /* If there is a replacement, use it. */
5136 fndecl
= find_tm_replacement_function (fndecl
);
5139 new_node
= cgraph_node::get_create (fndecl
);
5141 /* ??? Mark all transaction_wrap functions tm_may_enter_irr.
5143 We can't do this earlier in record_tm_replacement because
5144 cgraph_remove_unreachable_nodes is called before we inject
5145 references to the node. Further, we can't do this in some
5146 nice central place in ipa_tm_execute because we don't have
5147 the exact list of wrapper functions that would be used.
5148 Marking more wrappers than necessary results in the creation
5149 of unnecessary cgraph_nodes, which can cause some of the
5150 other IPA passes to crash.
5152 We do need to mark these nodes so that we get the proper
5153 result in expand_call_tm. */
5154 /* ??? This seems broken. How is it that we're marking the
5155 CALLEE as may_enter_irr? Surely we should be marking the
5156 CALLER. Also note that find_tm_replacement_function also
5157 contains mappings into the TM runtime, e.g. memcpy. These
5158 we know won't go irrevocable. */
5159 new_node
->local
.tm_may_enter_irr
= 1;
5163 struct tm_ipa_cg_data
*d
;
5164 struct cgraph_node
*tnode
= e
->callee
;
5166 d
= get_cg_data (&tnode
, true);
5167 new_node
= d
->clone
;
5169 /* As we've already skipped pure calls and appropriate builtins,
5170 and we've already marked irrevocable blocks, if we can't come
5171 up with a static replacement, then ask the runtime. */
5172 if (new_node
== NULL
)
5174 *need_ssa_rename_p
|=
5175 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5179 fndecl
= new_node
->decl
;
5182 e
->redirect_callee (new_node
);
5183 gimple_call_set_fndecl (stmt
, fndecl
);
5186 /* Helper function for ipa_tm_transform_calls. For a given BB,
5187 install calls to tm_irrevocable when IRR_BLOCKS are reached,
5188 redirect other calls to the generated transactional clone. */
5191 ipa_tm_transform_calls_1 (struct cgraph_node
*node
, struct tm_region
*region
,
5192 basic_block bb
, bitmap irr_blocks
)
5194 gimple_stmt_iterator gsi
;
5195 bool need_ssa_rename
= false;
5197 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5199 ipa_tm_insert_irr_call (node
, region
, bb
);
5203 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5205 gimple
*stmt
= gsi_stmt (gsi
);
5207 if (!is_gimple_call (stmt
))
5209 if (is_tm_pure_call (stmt
))
5212 /* Redirect edges to the appropriate replacement or clone. */
5213 ipa_tm_transform_calls_redirect (node
, region
, &gsi
, &need_ssa_rename
);
5216 return need_ssa_rename
;
5219 /* Walk the CFG for REGION, beginning at BB. Install calls to
5220 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to
5221 the generated transactional clone. */
5224 ipa_tm_transform_calls (struct cgraph_node
*node
, struct tm_region
*region
,
5225 basic_block bb
, bitmap irr_blocks
)
5227 bool need_ssa_rename
= false;
5230 auto_vec
<basic_block
> queue
;
5231 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
5233 queue
.safe_push (bb
);
5239 ipa_tm_transform_calls_1 (node
, region
, bb
, irr_blocks
);
5241 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5244 if (region
&& bitmap_bit_p (region
->exit_blocks
, bb
->index
))
5247 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
5248 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
5250 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
5251 queue
.safe_push (e
->dest
);
5254 while (!queue
.is_empty ());
5256 BITMAP_FREE (visited_blocks
);
5258 return need_ssa_rename
;
5261 /* Transform the calls within the TM regions within NODE. */
5264 ipa_tm_transform_transaction (struct cgraph_node
*node
)
5266 struct tm_ipa_cg_data
*d
;
5267 struct tm_region
*region
;
5268 bool need_ssa_rename
= false;
5270 d
= get_cg_data (&node
, true);
5272 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5273 calculate_dominance_info (CDI_DOMINATORS
);
5275 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
5277 /* If we're sure to go irrevocable, don't transform anything. */
5278 if (d
->irrevocable_blocks_normal
5279 && bitmap_bit_p (d
->irrevocable_blocks_normal
,
5280 region
->entry_block
->index
))
5282 transaction_subcode_ior (region
, GTMA_DOES_GO_IRREVOCABLE
5283 | GTMA_MAY_ENTER_IRREVOCABLE
5284 | GTMA_HAS_NO_INSTRUMENTATION
);
5289 ipa_tm_transform_calls (node
, region
, region
->entry_block
,
5290 d
->irrevocable_blocks_normal
);
5293 if (need_ssa_rename
)
5294 update_ssa (TODO_update_ssa_only_virtuals
);
5299 /* Transform the calls within the transactional clone of NODE. */
5302 ipa_tm_transform_clone (struct cgraph_node
*node
)
5304 struct tm_ipa_cg_data
*d
;
5305 bool need_ssa_rename
;
5307 d
= get_cg_data (&node
, true);
5309 /* If this function makes no calls and has no irrevocable blocks,
5310 then there's nothing to do. */
5311 /* ??? Remove non-aborting top-level transactions. */
5312 if (!node
->callees
&& !node
->indirect_calls
&& !d
->irrevocable_blocks_clone
)
5315 push_cfun (DECL_STRUCT_FUNCTION (d
->clone
->decl
));
5316 calculate_dominance_info (CDI_DOMINATORS
);
5319 ipa_tm_transform_calls (d
->clone
, NULL
,
5320 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
5321 d
->irrevocable_blocks_clone
);
5323 if (need_ssa_rename
)
5324 update_ssa (TODO_update_ssa_only_virtuals
);
5329 /* Main entry point for the transactional memory IPA pass. */
5332 ipa_tm_execute (void)
5334 cgraph_node_queue tm_callees
= cgraph_node_queue ();
5335 /* List of functions that will go irrevocable. */
5336 cgraph_node_queue irr_worklist
= cgraph_node_queue ();
5338 struct cgraph_node
*node
;
5339 struct tm_ipa_cg_data
*d
;
5340 enum availability a
;
5343 cgraph_node::checking_verify_cgraph_nodes ();
5345 bitmap_obstack_initialize (&tm_obstack
);
5346 initialize_original_copy_tables ();
5348 /* For all local functions marked tm_callable, queue them. */
5349 FOR_EACH_DEFINED_FUNCTION (node
)
5350 if (is_tm_callable (node
->decl
)
5351 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5353 d
= get_cg_data (&node
, true);
5354 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5357 /* For all local reachable functions... */
5358 FOR_EACH_DEFINED_FUNCTION (node
)
5360 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5362 /* ... marked tm_pure, record that fact for the runtime by
5363 indicating that the pure function is its own tm_callable.
5364 No need to do this if the function's address can't be taken. */
5365 if (is_tm_pure (node
->decl
))
5367 if (!node
->local
.local
)
5368 record_tm_clone_pair (node
->decl
, node
->decl
);
5372 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5373 calculate_dominance_info (CDI_DOMINATORS
);
5375 tm_region_init (NULL
);
5378 d
= get_cg_data (&node
, true);
5380 /* Scan for calls that are in each transaction, and
5381 generate the uninstrumented code path. */
5382 ipa_tm_scan_calls_transaction (d
, &tm_callees
);
5384 /* Put it in the worklist so we can scan the function
5385 later (ipa_tm_scan_irr_function) and mark the
5386 irrevocable blocks. */
5387 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5388 d
->want_irr_scan_normal
= true;
5394 /* For every local function on the callee list, scan as if we will be
5395 creating a transactional clone, queueing all new functions we find
5397 for (i
= 0; i
< tm_callees
.length (); ++i
)
5399 node
= tm_callees
[i
];
5400 a
= node
->get_availability ();
5401 d
= get_cg_data (&node
, true);
5403 /* Put it in the worklist so we can scan the function later
5404 (ipa_tm_scan_irr_function) and mark the irrevocable
5406 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5408 /* Some callees cannot be arbitrarily cloned. These will always be
5409 irrevocable. Mark these now, so that we need not scan them. */
5410 if (is_tm_irrevocable (node
->decl
))
5411 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5412 else if (a
<= AVAIL_NOT_AVAILABLE
5413 && !is_tm_safe_or_pure (node
->decl
))
5414 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5415 else if (a
>= AVAIL_INTERPOSABLE
)
5417 if (!tree_versionable_function_p (node
->decl
))
5418 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5419 else if (!d
->is_irrevocable
)
5421 /* If this is an alias, make sure its base is queued as well.
5422 we need not scan the callees now, as the base will do. */
5425 node
= cgraph_node::get (node
->thunk
.alias
);
5426 d
= get_cg_data (&node
, true);
5427 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5431 /* Add all nodes called by this function into
5432 tm_callees as well. */
5433 ipa_tm_scan_calls_clone (node
, &tm_callees
);
5438 /* Iterate scans until no more work to be done. Prefer not to use
5439 vec::pop because the worklist tends to follow a breadth-first
5440 search of the callgraph, which should allow convergance with a
5441 minimum number of scans. But we also don't want the worklist
5442 array to grow without bound, so we shift the array up periodically. */
5443 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5445 if (i
> 256 && i
== irr_worklist
.length () / 8)
5447 irr_worklist
.block_remove (0, i
);
5451 node
= irr_worklist
[i
];
5452 d
= get_cg_data (&node
, true);
5453 d
->in_worklist
= false;
5455 if (d
->want_irr_scan_normal
)
5457 d
->want_irr_scan_normal
= false;
5458 ipa_tm_scan_irr_function (node
, false);
5460 if (d
->in_callee_queue
&& ipa_tm_scan_irr_function (node
, true))
5461 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5464 /* For every function on the callee list, collect the tm_may_enter_irr
5466 irr_worklist
.truncate (0);
5467 for (i
= 0; i
< tm_callees
.length (); ++i
)
5469 node
= tm_callees
[i
];
5470 if (ipa_tm_mayenterirr_function (node
))
5472 d
= get_cg_data (&node
, true);
5473 gcc_assert (d
->in_worklist
== false);
5474 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5478 /* Propagate the tm_may_enter_irr bit to callers until stable. */
5479 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5481 struct cgraph_node
*caller
;
5482 struct cgraph_edge
*e
;
5483 struct ipa_ref
*ref
;
5485 if (i
> 256 && i
== irr_worklist
.length () / 8)
5487 irr_worklist
.block_remove (0, i
);
5491 node
= irr_worklist
[i
];
5492 d
= get_cg_data (&node
, true);
5493 d
->in_worklist
= false;
5494 node
->local
.tm_may_enter_irr
= true;
5496 /* Propagate back to normal callers. */
5497 for (e
= node
->callers
; e
; e
= e
->next_caller
)
5500 if (!is_tm_safe_or_pure (caller
->decl
)
5501 && !caller
->local
.tm_may_enter_irr
)
5503 d
= get_cg_data (&caller
, true);
5504 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5508 /* Propagate back to referring aliases as well. */
5509 FOR_EACH_ALIAS (node
, ref
)
5511 caller
= dyn_cast
<cgraph_node
*> (ref
->referring
);
5512 if (!caller
->local
.tm_may_enter_irr
)
5514 /* ?? Do not traverse aliases here. */
5515 d
= get_cg_data (&caller
, false);
5516 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5521 /* Now validate all tm_safe functions, and all atomic regions in
5523 FOR_EACH_DEFINED_FUNCTION (node
)
5525 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5527 d
= get_cg_data (&node
, true);
5528 if (is_tm_safe (node
->decl
))
5529 ipa_tm_diagnose_tm_safe (node
);
5530 else if (d
->all_tm_regions
)
5531 ipa_tm_diagnose_transaction (node
, d
->all_tm_regions
);
5534 /* Create clones. Do those that are not irrevocable and have a
5535 positive call count. Do those publicly visible functions that
5536 the user directed us to clone. */
5537 for (i
= 0; i
< tm_callees
.length (); ++i
)
5541 node
= tm_callees
[i
];
5542 if (node
->cpp_implicit_alias
)
5545 a
= node
->get_availability ();
5546 d
= get_cg_data (&node
, true);
5548 if (a
<= AVAIL_NOT_AVAILABLE
)
5549 doit
= is_tm_callable (node
->decl
);
5550 else if (a
<= AVAIL_AVAILABLE
&& is_tm_callable (node
->decl
))
5552 else if (!d
->is_irrevocable
5553 && d
->tm_callers_normal
+ d
->tm_callers_clone
> 0)
5557 ipa_tm_create_version (node
);
5560 /* Redirect calls to the new clones, and insert irrevocable marks. */
5561 for (i
= 0; i
< tm_callees
.length (); ++i
)
5563 node
= tm_callees
[i
];
5566 d
= get_cg_data (&node
, true);
5568 ipa_tm_transform_clone (node
);
5571 FOR_EACH_DEFINED_FUNCTION (node
)
5573 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5575 d
= get_cg_data (&node
, true);
5576 if (d
->all_tm_regions
)
5577 ipa_tm_transform_transaction (node
);
5580 /* Free and clear all data structures. */
5581 tm_callees
.release ();
5582 irr_worklist
.release ();
5583 bitmap_obstack_release (&tm_obstack
);
5584 free_original_copy_tables ();
5586 FOR_EACH_FUNCTION (node
)
5589 cgraph_node::checking_verify_cgraph_nodes ();
5596 const pass_data pass_data_ipa_tm
=
5598 SIMPLE_IPA_PASS
, /* type */
5600 OPTGROUP_NONE
, /* optinfo_flags */
5601 TV_TRANS_MEM
, /* tv_id */
5602 ( PROP_ssa
| PROP_cfg
), /* properties_required */
5603 0, /* properties_provided */
5604 0, /* properties_destroyed */
5605 0, /* todo_flags_start */
5606 0, /* todo_flags_finish */
5609 class pass_ipa_tm
: public simple_ipa_opt_pass
5612 pass_ipa_tm (gcc::context
*ctxt
)
5613 : simple_ipa_opt_pass (pass_data_ipa_tm
, ctxt
)
5616 /* opt_pass methods: */
5617 virtual bool gate (function
*) { return flag_tm
; }
5618 virtual unsigned int execute (function
*) { return ipa_tm_execute (); }
5620 }; // class pass_ipa_tm
5624 simple_ipa_opt_pass
*
5625 make_pass_ipa_tm (gcc::context
*ctxt
)
5627 return new pass_ipa_tm (ctxt
);
5630 #include "gt-trans-mem.h"