1 /* Passes for transactional memory support.
2 Copyright (C) 2008-2014 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
23 #include "hash-table.h"
31 #include "hard-reg-set.h"
34 #include "dominance.h"
36 #include "basic-block.h"
37 #include "tree-ssa-alias.h"
38 #include "internal-fn.h"
40 #include "gimple-expr.h"
47 #include "gimple-iterator.h"
48 #include "gimplify-me.h"
49 #include "gimple-walk.h"
50 #include "gimple-ssa.h"
52 #include "plugin-api.h"
56 #include "stringpool.h"
57 #include "tree-ssanames.h"
58 #include "tree-into-ssa.h"
59 #include "tree-pass.h"
60 #include "tree-inline.h"
61 #include "diagnostic-core.h"
64 #include "trans-mem.h"
67 #include "langhooks.h"
68 #include "gimple-pretty-print.h"
70 #include "tree-ssa-address.h"
73 #define A_RUNINSTRUMENTEDCODE 0x0001
74 #define A_RUNUNINSTRUMENTEDCODE 0x0002
75 #define A_SAVELIVEVARIABLES 0x0004
76 #define A_RESTORELIVEVARIABLES 0x0008
77 #define A_ABORTTRANSACTION 0x0010
79 #define AR_USERABORT 0x0001
80 #define AR_USERRETRY 0x0002
81 #define AR_TMCONFLICT 0x0004
82 #define AR_EXCEPTIONBLOCKABORT 0x0008
83 #define AR_OUTERABORT 0x0010
85 #define MODE_SERIALIRREVOCABLE 0x0000
88 /* The representation of a transaction changes several times during the
89 lowering process. In the beginning, in the front-end we have the
90 GENERIC tree TRANSACTION_EXPR. For example,
98 During initial gimplification (gimplify.c) the TRANSACTION_EXPR node is
99 trivially replaced with a GIMPLE_TRANSACTION node.
101 During pass_lower_tm, we examine the body of transactions looking
102 for aborts. Transactions that do not contain an abort may be
103 merged into an outer transaction. We also add a TRY-FINALLY node
104 to arrange for the transaction to be committed on any exit.
106 [??? Think about how this arrangement affects throw-with-commit
107 and throw-with-abort operations. In this case we want the TRY to
108 handle gotos, but not to catch any exceptions because the transaction
109 will already be closed.]
111 GIMPLE_TRANSACTION [label=NULL] {
118 __builtin___tm_abort ();
120 __builtin___tm_commit ();
124 During pass_lower_eh, we create EH regions for the transactions,
125 intermixed with the regular EH stuff. This gives us a nice persistent
126 mapping (all the way through rtl) from transactional memory operation
127 back to the transaction, which allows us to get the abnormal edges
128 correct to model transaction aborts and restarts:
130 GIMPLE_TRANSACTION [label=over]
136 __builtin___tm_abort ();
137 __builtin___tm_commit ();
140 This is the end of all_lowering_passes, and so is what is present
141 during the IPA passes, and through all of the optimization passes.
143 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all
144 functions and mark functions for cloning.
146 At the end of gimple optimization, before exiting SSA form,
147 pass_tm_edges replaces statements that perform transactional
148 memory operations with the appropriate TM builtins, and swap
149 out function calls with their transactional clones. At this
150 point we introduce the abnormal transaction restart edges and
151 complete lowering of the GIMPLE_TRANSACTION node.
153 x = __builtin___tm_start (MAY_ABORT);
155 if (x & abort_transaction)
158 t0 = __builtin___tm_load (global);
160 __builtin___tm_store (&global, t1);
162 __builtin___tm_abort ();
163 __builtin___tm_commit ();
167 static void *expand_regions (struct tm_region
*,
168 void *(*callback
)(struct tm_region
*, void *),
172 /* Return the attributes we want to examine for X, or NULL if it's not
173 something we examine. We look at function types, but allow pointers
174 to function types and function decls and peek through. */
177 get_attrs_for (const_tree x
)
179 switch (TREE_CODE (x
))
182 return TYPE_ATTRIBUTES (TREE_TYPE (x
));
189 if (TREE_CODE (x
) != POINTER_TYPE
)
195 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
201 return TYPE_ATTRIBUTES (x
);
205 /* Return true if X has been marked TM_PURE. */
208 is_tm_pure (const_tree x
)
212 switch (TREE_CODE (x
))
223 if (TREE_CODE (x
) != POINTER_TYPE
)
229 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
234 flags
= flags_from_decl_or_type (x
);
235 return (flags
& ECF_TM_PURE
) != 0;
238 /* Return true if X has been marked TM_IRREVOCABLE. */
241 is_tm_irrevocable (tree x
)
243 tree attrs
= get_attrs_for (x
);
245 if (attrs
&& lookup_attribute ("transaction_unsafe", attrs
))
248 /* A call to the irrevocable builtin is by definition,
250 if (TREE_CODE (x
) == ADDR_EXPR
)
251 x
= TREE_OPERAND (x
, 0);
252 if (TREE_CODE (x
) == FUNCTION_DECL
253 && DECL_BUILT_IN_CLASS (x
) == BUILT_IN_NORMAL
254 && DECL_FUNCTION_CODE (x
) == BUILT_IN_TM_IRREVOCABLE
)
260 /* Return true if X has been marked TM_SAFE. */
263 is_tm_safe (const_tree x
)
267 tree attrs
= get_attrs_for (x
);
270 if (lookup_attribute ("transaction_safe", attrs
))
272 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
279 /* Return true if CALL is const, or tm_pure. */
282 is_tm_pure_call (gimple call
)
284 tree fn
= gimple_call_fn (call
);
286 if (TREE_CODE (fn
) == ADDR_EXPR
)
288 fn
= TREE_OPERAND (fn
, 0);
289 gcc_assert (TREE_CODE (fn
) == FUNCTION_DECL
);
294 return is_tm_pure (fn
);
297 /* Return true if X has been marked TM_CALLABLE. */
300 is_tm_callable (tree x
)
302 tree attrs
= get_attrs_for (x
);
305 if (lookup_attribute ("transaction_callable", attrs
))
307 if (lookup_attribute ("transaction_safe", attrs
))
309 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
315 /* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */
318 is_tm_may_cancel_outer (tree x
)
320 tree attrs
= get_attrs_for (x
);
322 return lookup_attribute ("transaction_may_cancel_outer", attrs
) != NULL
;
326 /* Return true for built in functions that "end" a transaction. */
329 is_tm_ending_fndecl (tree fndecl
)
331 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
332 switch (DECL_FUNCTION_CODE (fndecl
))
334 case BUILT_IN_TM_COMMIT
:
335 case BUILT_IN_TM_COMMIT_EH
:
336 case BUILT_IN_TM_ABORT
:
337 case BUILT_IN_TM_IRREVOCABLE
:
346 /* Return true if STMT is a built in function call that "ends" a
350 is_tm_ending (gimple stmt
)
354 if (gimple_code (stmt
) != GIMPLE_CALL
)
357 fndecl
= gimple_call_fndecl (stmt
);
358 return (fndecl
!= NULL_TREE
359 && is_tm_ending_fndecl (fndecl
));
362 /* Return true if STMT is a TM load. */
365 is_tm_load (gimple stmt
)
369 if (gimple_code (stmt
) != GIMPLE_CALL
)
372 fndecl
= gimple_call_fndecl (stmt
);
373 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
374 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl
)));
377 /* Same as above, but for simple TM loads, that is, not the
378 after-write, after-read, etc optimized variants. */
381 is_tm_simple_load (gimple stmt
)
385 if (gimple_code (stmt
) != GIMPLE_CALL
)
388 fndecl
= gimple_call_fndecl (stmt
);
389 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
391 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
392 return (fcode
== BUILT_IN_TM_LOAD_1
393 || fcode
== BUILT_IN_TM_LOAD_2
394 || fcode
== BUILT_IN_TM_LOAD_4
395 || fcode
== BUILT_IN_TM_LOAD_8
396 || fcode
== BUILT_IN_TM_LOAD_FLOAT
397 || fcode
== BUILT_IN_TM_LOAD_DOUBLE
398 || fcode
== BUILT_IN_TM_LOAD_LDOUBLE
399 || fcode
== BUILT_IN_TM_LOAD_M64
400 || fcode
== BUILT_IN_TM_LOAD_M128
401 || fcode
== BUILT_IN_TM_LOAD_M256
);
406 /* Return true if STMT is a TM store. */
409 is_tm_store (gimple stmt
)
413 if (gimple_code (stmt
) != GIMPLE_CALL
)
416 fndecl
= gimple_call_fndecl (stmt
);
417 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
418 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl
)));
421 /* Same as above, but for simple TM stores, that is, not the
422 after-write, after-read, etc optimized variants. */
425 is_tm_simple_store (gimple stmt
)
429 if (gimple_code (stmt
) != GIMPLE_CALL
)
432 fndecl
= gimple_call_fndecl (stmt
);
433 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
435 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
436 return (fcode
== BUILT_IN_TM_STORE_1
437 || fcode
== BUILT_IN_TM_STORE_2
438 || fcode
== BUILT_IN_TM_STORE_4
439 || fcode
== BUILT_IN_TM_STORE_8
440 || fcode
== BUILT_IN_TM_STORE_FLOAT
441 || fcode
== BUILT_IN_TM_STORE_DOUBLE
442 || fcode
== BUILT_IN_TM_STORE_LDOUBLE
443 || fcode
== BUILT_IN_TM_STORE_M64
444 || fcode
== BUILT_IN_TM_STORE_M128
445 || fcode
== BUILT_IN_TM_STORE_M256
);
450 /* Return true if FNDECL is BUILT_IN_TM_ABORT. */
453 is_tm_abort (tree fndecl
)
456 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
457 && DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_TM_ABORT
);
460 /* Build a GENERIC tree for a user abort. This is called by front ends
461 while transforming the __tm_abort statement. */
464 build_tm_abort_call (location_t loc
, bool is_outer
)
466 return build_call_expr_loc (loc
, builtin_decl_explicit (BUILT_IN_TM_ABORT
), 1,
467 build_int_cst (integer_type_node
,
469 | (is_outer
? AR_OUTERABORT
: 0)));
472 /* Map for aribtrary function replacement under TM, as created
473 by the tm_wrap attribute. */
475 static GTY((if_marked ("tree_map_marked_p"), param_is (struct tree_map
)))
479 record_tm_replacement (tree from
, tree to
)
481 struct tree_map
**slot
, *h
;
483 /* Do not inline wrapper functions that will get replaced in the TM
486 Suppose you have foo() that will get replaced into tmfoo(). Make
487 sure the inliner doesn't try to outsmart us and inline foo()
488 before we get a chance to do the TM replacement. */
489 DECL_UNINLINABLE (from
) = 1;
491 if (tm_wrap_map
== NULL
)
492 tm_wrap_map
= htab_create_ggc (32, tree_map_hash
, tree_map_eq
, 0);
494 h
= ggc_alloc
<tree_map
> ();
495 h
->hash
= htab_hash_pointer (from
);
499 slot
= (struct tree_map
**)
500 htab_find_slot_with_hash (tm_wrap_map
, h
, h
->hash
, INSERT
);
504 /* Return a TM-aware replacement function for DECL. */
507 find_tm_replacement_function (tree fndecl
)
511 struct tree_map
*h
, in
;
513 in
.base
.from
= fndecl
;
514 in
.hash
= htab_hash_pointer (fndecl
);
515 h
= (struct tree_map
*) htab_find_with_hash (tm_wrap_map
, &in
, in
.hash
);
520 /* ??? We may well want TM versions of most of the common <string.h>
521 functions. For now, we've already these two defined. */
522 /* Adjust expand_call_tm() attributes as necessary for the cases
524 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
525 switch (DECL_FUNCTION_CODE (fndecl
))
527 case BUILT_IN_MEMCPY
:
528 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY
);
529 case BUILT_IN_MEMMOVE
:
530 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
);
531 case BUILT_IN_MEMSET
:
532 return builtin_decl_explicit (BUILT_IN_TM_MEMSET
);
540 /* When appropriate, record TM replacement for memory allocation functions.
542 FROM is the FNDECL to wrap. */
544 tm_malloc_replacement (tree from
)
549 if (TREE_CODE (from
) != FUNCTION_DECL
)
552 /* If we have a previous replacement, the user must be explicitly
553 wrapping malloc/calloc/free. They better know what they're
555 if (find_tm_replacement_function (from
))
558 str
= IDENTIFIER_POINTER (DECL_NAME (from
));
560 if (!strcmp (str
, "malloc"))
561 to
= builtin_decl_explicit (BUILT_IN_TM_MALLOC
);
562 else if (!strcmp (str
, "calloc"))
563 to
= builtin_decl_explicit (BUILT_IN_TM_CALLOC
);
564 else if (!strcmp (str
, "free"))
565 to
= builtin_decl_explicit (BUILT_IN_TM_FREE
);
569 TREE_NOTHROW (to
) = 0;
571 record_tm_replacement (from
, to
);
574 /* Diagnostics for tm_safe functions/regions. Called by the front end
575 once we've lowered the function to high-gimple. */
577 /* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq.
578 Process exactly one statement. WI->INFO is set to non-null when in
579 the context of a tm_safe function, and null for a __transaction block. */
581 #define DIAG_TM_OUTER 1
582 #define DIAG_TM_SAFE 2
583 #define DIAG_TM_RELAXED 4
587 unsigned int summary_flags
: 8;
588 unsigned int block_flags
: 8;
589 unsigned int func_flags
: 8;
590 unsigned int saw_volatile
: 1;
594 /* Return true if T is a volatile variable of some kind. */
597 volatile_var_p (tree t
)
599 return (SSA_VAR_P (t
)
600 && TREE_THIS_VOLATILE (TREE_TYPE (t
)));
603 /* Tree callback function for diagnose_tm pass. */
606 diagnose_tm_1_op (tree
*tp
, int *walk_subtrees ATTRIBUTE_UNUSED
,
609 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
610 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
612 if (volatile_var_p (*tp
)
613 && d
->block_flags
& DIAG_TM_SAFE
617 error_at (gimple_location (d
->stmt
),
618 "invalid volatile use of %qD inside transaction",
626 is_tm_safe_or_pure (const_tree x
)
628 return is_tm_safe (x
) || is_tm_pure (x
);
632 diagnose_tm_1 (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
633 struct walk_stmt_info
*wi
)
635 gimple stmt
= gsi_stmt (*gsi
);
636 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
638 /* Save stmt for use in leaf analysis. */
641 switch (gimple_code (stmt
))
645 tree fn
= gimple_call_fn (stmt
);
647 if ((d
->summary_flags
& DIAG_TM_OUTER
) == 0
648 && is_tm_may_cancel_outer (fn
))
649 error_at (gimple_location (stmt
),
650 "%<transaction_may_cancel_outer%> function call not within"
651 " outer transaction or %<transaction_may_cancel_outer%>");
653 if (d
->summary_flags
& DIAG_TM_SAFE
)
655 bool is_safe
, direct_call_p
;
658 if (TREE_CODE (fn
) == ADDR_EXPR
659 && TREE_CODE (TREE_OPERAND (fn
, 0)) == FUNCTION_DECL
)
661 direct_call_p
= true;
662 replacement
= TREE_OPERAND (fn
, 0);
663 replacement
= find_tm_replacement_function (replacement
);
669 direct_call_p
= false;
670 replacement
= NULL_TREE
;
673 if (is_tm_safe_or_pure (fn
))
675 else if (is_tm_callable (fn
) || is_tm_irrevocable (fn
))
677 /* A function explicitly marked transaction_callable as
678 opposed to transaction_safe is being defined to be
679 unsafe as part of its ABI, regardless of its contents. */
682 else if (direct_call_p
)
684 if (IS_TYPE_OR_DECL_P (fn
)
685 && flags_from_decl_or_type (fn
) & ECF_TM_BUILTIN
)
687 else if (replacement
)
689 /* ??? At present we've been considering replacements
690 merely transaction_callable, and therefore might
691 enter irrevocable. The tm_wrap attribute has not
692 yet made it into the new language spec. */
697 /* ??? Diagnostics for unmarked direct calls moved into
698 the IPA pass. Section 3.2 of the spec details how
699 functions not marked should be considered "implicitly
700 safe" based on having examined the function body. */
706 /* An unmarked indirect call. Consider it unsafe even
707 though optimization may yet figure out how to inline. */
713 if (TREE_CODE (fn
) == ADDR_EXPR
)
714 fn
= TREE_OPERAND (fn
, 0);
715 if (d
->block_flags
& DIAG_TM_SAFE
)
718 error_at (gimple_location (stmt
),
719 "unsafe function call %qD within "
720 "atomic transaction", fn
);
723 if (!DECL_P (fn
) || DECL_NAME (fn
))
724 error_at (gimple_location (stmt
),
725 "unsafe function call %qE within "
726 "atomic transaction", fn
);
728 error_at (gimple_location (stmt
),
729 "unsafe indirect function call within "
730 "atomic transaction");
736 error_at (gimple_location (stmt
),
737 "unsafe function call %qD within "
738 "%<transaction_safe%> function", fn
);
741 if (!DECL_P (fn
) || DECL_NAME (fn
))
742 error_at (gimple_location (stmt
),
743 "unsafe function call %qE within "
744 "%<transaction_safe%> function", fn
);
746 error_at (gimple_location (stmt
),
747 "unsafe indirect function call within "
748 "%<transaction_safe%> function");
757 /* ??? We ought to come up with a way to add attributes to
758 asm statements, and then add "transaction_safe" to it.
759 Either that or get the language spec to resurrect __tm_waiver. */
760 if (d
->block_flags
& DIAG_TM_SAFE
)
761 error_at (gimple_location (stmt
),
762 "asm not allowed in atomic transaction");
763 else if (d
->func_flags
& DIAG_TM_SAFE
)
764 error_at (gimple_location (stmt
),
765 "asm not allowed in %<transaction_safe%> function");
768 case GIMPLE_TRANSACTION
:
770 gtransaction
*trans_stmt
= as_a
<gtransaction
*> (stmt
);
771 unsigned char inner_flags
= DIAG_TM_SAFE
;
773 if (gimple_transaction_subcode (trans_stmt
) & GTMA_IS_RELAXED
)
775 if (d
->block_flags
& DIAG_TM_SAFE
)
776 error_at (gimple_location (stmt
),
777 "relaxed transaction in atomic transaction");
778 else if (d
->func_flags
& DIAG_TM_SAFE
)
779 error_at (gimple_location (stmt
),
780 "relaxed transaction in %<transaction_safe%> function");
781 inner_flags
= DIAG_TM_RELAXED
;
783 else if (gimple_transaction_subcode (trans_stmt
) & GTMA_IS_OUTER
)
786 error_at (gimple_location (stmt
),
787 "outer transaction in transaction");
788 else if (d
->func_flags
& DIAG_TM_OUTER
)
789 error_at (gimple_location (stmt
),
790 "outer transaction in "
791 "%<transaction_may_cancel_outer%> function");
792 else if (d
->func_flags
& DIAG_TM_SAFE
)
793 error_at (gimple_location (stmt
),
794 "outer transaction in %<transaction_safe%> function");
795 inner_flags
|= DIAG_TM_OUTER
;
798 *handled_ops_p
= true;
799 if (gimple_transaction_body (trans_stmt
))
801 struct walk_stmt_info wi_inner
;
802 struct diagnose_tm d_inner
;
804 memset (&d_inner
, 0, sizeof (d_inner
));
805 d_inner
.func_flags
= d
->func_flags
;
806 d_inner
.block_flags
= d
->block_flags
| inner_flags
;
807 d_inner
.summary_flags
= d_inner
.func_flags
| d_inner
.block_flags
;
809 memset (&wi_inner
, 0, sizeof (wi_inner
));
810 wi_inner
.info
= &d_inner
;
812 walk_gimple_seq (gimple_transaction_body (trans_stmt
),
813 diagnose_tm_1
, diagnose_tm_1_op
, &wi_inner
);
826 diagnose_tm_blocks (void)
828 struct walk_stmt_info wi
;
829 struct diagnose_tm d
;
831 memset (&d
, 0, sizeof (d
));
832 if (is_tm_may_cancel_outer (current_function_decl
))
833 d
.func_flags
= DIAG_TM_OUTER
| DIAG_TM_SAFE
;
834 else if (is_tm_safe (current_function_decl
))
835 d
.func_flags
= DIAG_TM_SAFE
;
836 d
.summary_flags
= d
.func_flags
;
838 memset (&wi
, 0, sizeof (wi
));
841 walk_gimple_seq (gimple_body (current_function_decl
),
842 diagnose_tm_1
, diagnose_tm_1_op
, &wi
);
849 const pass_data pass_data_diagnose_tm_blocks
=
851 GIMPLE_PASS
, /* type */
852 "*diagnose_tm_blocks", /* name */
853 OPTGROUP_NONE
, /* optinfo_flags */
854 TV_TRANS_MEM
, /* tv_id */
855 PROP_gimple_any
, /* properties_required */
856 0, /* properties_provided */
857 0, /* properties_destroyed */
858 0, /* todo_flags_start */
859 0, /* todo_flags_finish */
862 class pass_diagnose_tm_blocks
: public gimple_opt_pass
865 pass_diagnose_tm_blocks (gcc::context
*ctxt
)
866 : gimple_opt_pass (pass_data_diagnose_tm_blocks
, ctxt
)
869 /* opt_pass methods: */
870 virtual bool gate (function
*) { return flag_tm
; }
871 virtual unsigned int execute (function
*) { return diagnose_tm_blocks (); }
873 }; // class pass_diagnose_tm_blocks
878 make_pass_diagnose_tm_blocks (gcc::context
*ctxt
)
880 return new pass_diagnose_tm_blocks (ctxt
);
883 /* Instead of instrumenting thread private memory, we save the
884 addresses in a log which we later use to save/restore the addresses
885 upon transaction start/restart.
887 The log is keyed by address, where each element contains individual
888 statements among different code paths that perform the store.
890 This log is later used to generate either plain save/restore of the
891 addresses upon transaction start/restart, or calls to the ITM_L*
894 So for something like:
896 struct large { int x[1000]; };
897 struct large lala = { 0 };
903 We can either save/restore:
906 trxn = _ITM_startTransaction ();
907 if (trxn & a_saveLiveVariables)
908 tmp_lala1 = lala.x[i];
909 else if (a & a_restoreLiveVariables)
910 lala.x[i] = tmp_lala1;
912 or use the logging functions:
915 trxn = _ITM_startTransaction ();
916 _ITM_LU4 (&lala.x[i]);
918 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as
919 far up the dominator tree to shadow all of the writes to a given
920 location (thus reducing the total number of logging calls), but not
921 so high as to be called on a path that does not perform a
924 /* One individual log entry. We may have multiple statements for the
925 same location if neither dominate each other (on different
927 typedef struct tm_log_entry
929 /* Address to save. */
931 /* Entry block for the transaction this address occurs in. */
932 basic_block entry_block
;
933 /* Dominating statements the store occurs in. */
935 /* Initially, while we are building the log, we place a nonzero
936 value here to mean that this address *will* be saved with a
937 save/restore sequence. Later, when generating the save sequence
938 we place the SSA temp generated here. */
943 /* Log entry hashtable helpers. */
945 struct log_entry_hasher
947 typedef tm_log_entry value_type
;
948 typedef tm_log_entry compare_type
;
949 static inline hashval_t
hash (const value_type
*);
950 static inline bool equal (const value_type
*, const compare_type
*);
951 static inline void remove (value_type
*);
954 /* Htab support. Return hash value for a `tm_log_entry'. */
956 log_entry_hasher::hash (const value_type
*log
)
958 return iterative_hash_expr (log
->addr
, 0);
961 /* Htab support. Return true if two log entries are the same. */
963 log_entry_hasher::equal (const value_type
*log1
, const compare_type
*log2
)
967 rth: I suggest that we get rid of the component refs etc.
968 I.e. resolve the reference to base + offset.
970 We may need to actually finish a merge with mainline for this,
971 since we'd like to be presented with Richi's MEM_REF_EXPRs more
972 often than not. But in the meantime your tm_log_entry could save
973 the results of get_inner_reference.
975 See: g++.dg/tm/pr46653.C
978 /* Special case plain equality because operand_equal_p() below will
979 return FALSE if the addresses are equal but they have
980 side-effects (e.g. a volatile address). */
981 if (log1
->addr
== log2
->addr
)
984 return operand_equal_p (log1
->addr
, log2
->addr
, 0);
987 /* Htab support. Free one tm_log_entry. */
989 log_entry_hasher::remove (value_type
*lp
)
991 lp
->stmts
.release ();
996 /* The actual log. */
997 static hash_table
<log_entry_hasher
> *tm_log
;
999 /* Addresses to log with a save/restore sequence. These should be in
1001 static vec
<tree
> tm_log_save_addresses
;
1003 enum thread_memory_type
1007 mem_transaction_local
,
1011 typedef struct tm_new_mem_map
1013 /* SSA_NAME being dereferenced. */
1015 enum thread_memory_type local_new_memory
;
1018 /* Hashtable helpers. */
1020 struct tm_mem_map_hasher
: typed_free_remove
<tm_new_mem_map_t
>
1022 typedef tm_new_mem_map_t value_type
;
1023 typedef tm_new_mem_map_t compare_type
;
1024 static inline hashval_t
hash (const value_type
*);
1025 static inline bool equal (const value_type
*, const compare_type
*);
1029 tm_mem_map_hasher::hash (const value_type
*v
)
1031 return (intptr_t)v
->val
>> 4;
1035 tm_mem_map_hasher::equal (const value_type
*v
, const compare_type
*c
)
1037 return v
->val
== c
->val
;
1040 /* Map for an SSA_NAME originally pointing to a non aliased new piece
1041 of memory (malloc, alloc, etc). */
1042 static hash_table
<tm_mem_map_hasher
> *tm_new_mem_hash
;
1044 /* Initialize logging data structures. */
1048 tm_log
= new hash_table
<log_entry_hasher
> (10);
1049 tm_new_mem_hash
= new hash_table
<tm_mem_map_hasher
> (5);
1050 tm_log_save_addresses
.create (5);
1053 /* Free logging data structures. */
1055 tm_log_delete (void)
1059 delete tm_new_mem_hash
;
1060 tm_new_mem_hash
= NULL
;
1061 tm_log_save_addresses
.release ();
1064 /* Return true if MEM is a transaction invariant memory for the TM
1065 region starting at REGION_ENTRY_BLOCK. */
1067 transaction_invariant_address_p (const_tree mem
, basic_block region_entry_block
)
1069 if ((TREE_CODE (mem
) == INDIRECT_REF
|| TREE_CODE (mem
) == MEM_REF
)
1070 && TREE_CODE (TREE_OPERAND (mem
, 0)) == SSA_NAME
)
1074 def_bb
= gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem
, 0)));
1075 return def_bb
!= region_entry_block
1076 && dominated_by_p (CDI_DOMINATORS
, region_entry_block
, def_bb
);
1079 mem
= strip_invariant_refs (mem
);
1080 return mem
&& (CONSTANT_CLASS_P (mem
) || decl_address_invariant_p (mem
));
1083 /* Given an address ADDR in STMT, find it in the memory log or add it,
1084 making sure to keep only the addresses highest in the dominator
1087 ENTRY_BLOCK is the entry_block for the transaction.
1089 If we find the address in the log, make sure it's either the same
1090 address, or an equivalent one that dominates ADDR.
1092 If we find the address, but neither ADDR dominates the found
1093 address, nor the found one dominates ADDR, we're on different
1094 execution paths. Add it.
1096 If known, ENTRY_BLOCK is the entry block for the region, otherwise
1099 tm_log_add (basic_block entry_block
, tree addr
, gimple stmt
)
1101 tm_log_entry
**slot
;
1102 struct tm_log_entry l
, *lp
;
1105 slot
= tm_log
->find_slot (&l
, INSERT
);
1108 tree type
= TREE_TYPE (addr
);
1110 lp
= XNEW (struct tm_log_entry
);
1114 /* Small invariant addresses can be handled as save/restores. */
1116 && transaction_invariant_address_p (lp
->addr
, entry_block
)
1117 && TYPE_SIZE_UNIT (type
) != NULL
1118 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
))
1119 && ((HOST_WIDE_INT
) tree_to_uhwi (TYPE_SIZE_UNIT (type
))
1120 < PARAM_VALUE (PARAM_TM_MAX_AGGREGATE_SIZE
))
1121 /* We must be able to copy this type normally. I.e., no
1122 special constructors and the like. */
1123 && !TREE_ADDRESSABLE (type
))
1125 lp
->save_var
= create_tmp_reg (TREE_TYPE (lp
->addr
), "tm_save");
1126 lp
->stmts
.create (0);
1127 lp
->entry_block
= entry_block
;
1128 /* Save addresses separately in dominator order so we don't
1129 get confused by overlapping addresses in the save/restore
1131 tm_log_save_addresses
.safe_push (lp
->addr
);
1135 /* Use the logging functions. */
1136 lp
->stmts
.create (5);
1137 lp
->stmts
.quick_push (stmt
);
1138 lp
->save_var
= NULL
;
1148 /* If we're generating a save/restore sequence, we don't care
1149 about statements. */
1153 for (i
= 0; lp
->stmts
.iterate (i
, &oldstmt
); ++i
)
1155 if (stmt
== oldstmt
)
1157 /* We already have a store to the same address, higher up the
1158 dominator tree. Nothing to do. */
1159 if (dominated_by_p (CDI_DOMINATORS
,
1160 gimple_bb (stmt
), gimple_bb (oldstmt
)))
1162 /* We should be processing blocks in dominator tree order. */
1163 gcc_assert (!dominated_by_p (CDI_DOMINATORS
,
1164 gimple_bb (oldstmt
), gimple_bb (stmt
)));
1166 /* Store is on a different code path. */
1167 lp
->stmts
.safe_push (stmt
);
1171 /* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME
1172 result, insert the new statements before GSI. */
1175 gimplify_addr (gimple_stmt_iterator
*gsi
, tree x
)
1177 if (TREE_CODE (x
) == TARGET_MEM_REF
)
1178 x
= tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x
)), x
);
1180 x
= build_fold_addr_expr (x
);
1181 return force_gimple_operand_gsi (gsi
, x
, true, NULL
, true, GSI_SAME_STMT
);
1184 /* Instrument one address with the logging functions.
1185 ADDR is the address to save.
1186 STMT is the statement before which to place it. */
1188 tm_log_emit_stmt (tree addr
, gimple stmt
)
1190 tree type
= TREE_TYPE (addr
);
1191 tree size
= TYPE_SIZE_UNIT (type
);
1192 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
1194 enum built_in_function code
= BUILT_IN_TM_LOG
;
1196 if (type
== float_type_node
)
1197 code
= BUILT_IN_TM_LOG_FLOAT
;
1198 else if (type
== double_type_node
)
1199 code
= BUILT_IN_TM_LOG_DOUBLE
;
1200 else if (type
== long_double_type_node
)
1201 code
= BUILT_IN_TM_LOG_LDOUBLE
;
1202 else if (tree_fits_uhwi_p (size
))
1204 unsigned int n
= tree_to_uhwi (size
);
1208 code
= BUILT_IN_TM_LOG_1
;
1211 code
= BUILT_IN_TM_LOG_2
;
1214 code
= BUILT_IN_TM_LOG_4
;
1217 code
= BUILT_IN_TM_LOG_8
;
1220 code
= BUILT_IN_TM_LOG
;
1221 if (TREE_CODE (type
) == VECTOR_TYPE
)
1223 if (n
== 8 && builtin_decl_explicit (BUILT_IN_TM_LOG_M64
))
1224 code
= BUILT_IN_TM_LOG_M64
;
1225 else if (n
== 16 && builtin_decl_explicit (BUILT_IN_TM_LOG_M128
))
1226 code
= BUILT_IN_TM_LOG_M128
;
1227 else if (n
== 32 && builtin_decl_explicit (BUILT_IN_TM_LOG_M256
))
1228 code
= BUILT_IN_TM_LOG_M256
;
1234 addr
= gimplify_addr (&gsi
, addr
);
1235 if (code
== BUILT_IN_TM_LOG
)
1236 log
= gimple_build_call (builtin_decl_explicit (code
), 2, addr
, size
);
1238 log
= gimple_build_call (builtin_decl_explicit (code
), 1, addr
);
1239 gsi_insert_before (&gsi
, log
, GSI_SAME_STMT
);
1242 /* Go through the log and instrument address that must be instrumented
1243 with the logging functions. Leave the save/restore addresses for
1248 hash_table
<log_entry_hasher
>::iterator hi
;
1249 struct tm_log_entry
*lp
;
1251 FOR_EACH_HASH_TABLE_ELEMENT (*tm_log
, lp
, tm_log_entry_t
, hi
)
1258 fprintf (dump_file
, "TM thread private mem logging: ");
1259 print_generic_expr (dump_file
, lp
->addr
, 0);
1260 fprintf (dump_file
, "\n");
1266 fprintf (dump_file
, "DUMPING to variable\n");
1272 fprintf (dump_file
, "DUMPING with logging functions\n");
1273 for (i
= 0; lp
->stmts
.iterate (i
, &stmt
); ++i
)
1274 tm_log_emit_stmt (lp
->addr
, stmt
);
1279 /* Emit the save sequence for the corresponding addresses in the log.
1280 ENTRY_BLOCK is the entry block for the transaction.
1281 BB is the basic block to insert the code in. */
1283 tm_log_emit_saves (basic_block entry_block
, basic_block bb
)
1286 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1288 struct tm_log_entry l
, *lp
;
1290 for (i
= 0; i
< tm_log_save_addresses
.length (); ++i
)
1292 l
.addr
= tm_log_save_addresses
[i
];
1293 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1294 gcc_assert (lp
->save_var
!= NULL
);
1296 /* We only care about variables in the current transaction. */
1297 if (lp
->entry_block
!= entry_block
)
1300 stmt
= gimple_build_assign (lp
->save_var
, unshare_expr (lp
->addr
));
1302 /* Make sure we can create an SSA_NAME for this type. For
1303 instance, aggregates aren't allowed, in which case the system
1304 will create a VOP for us and everything will just work. */
1305 if (is_gimple_reg_type (TREE_TYPE (lp
->save_var
)))
1307 lp
->save_var
= make_ssa_name (lp
->save_var
, stmt
);
1308 gimple_assign_set_lhs (stmt
, lp
->save_var
);
1311 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
1315 /* Emit the restore sequence for the corresponding addresses in the log.
1316 ENTRY_BLOCK is the entry block for the transaction.
1317 BB is the basic block to insert the code in. */
1319 tm_log_emit_restores (basic_block entry_block
, basic_block bb
)
1322 struct tm_log_entry l
, *lp
;
1323 gimple_stmt_iterator gsi
;
1326 for (i
= tm_log_save_addresses
.length () - 1; i
>= 0; i
--)
1328 l
.addr
= tm_log_save_addresses
[i
];
1329 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1330 gcc_assert (lp
->save_var
!= NULL
);
1332 /* We only care about variables in the current transaction. */
1333 if (lp
->entry_block
!= entry_block
)
1336 /* Restores are in LIFO order from the saves in case we have
1338 gsi
= gsi_start_bb (bb
);
1340 stmt
= gimple_build_assign (unshare_expr (lp
->addr
), lp
->save_var
);
1341 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1346 static tree
lower_sequence_tm (gimple_stmt_iterator
*, bool *,
1347 struct walk_stmt_info
*);
1348 static tree
lower_sequence_no_tm (gimple_stmt_iterator
*, bool *,
1349 struct walk_stmt_info
*);
1351 /* Evaluate an address X being dereferenced and determine if it
1352 originally points to a non aliased new chunk of memory (malloc,
1355 Return MEM_THREAD_LOCAL if it points to a thread-local address.
1356 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address.
1357 Return MEM_NON_LOCAL otherwise.
1359 ENTRY_BLOCK is the entry block to the transaction containing the
1360 dereference of X. */
1361 static enum thread_memory_type
1362 thread_private_new_memory (basic_block entry_block
, tree x
)
1365 enum tree_code code
;
1366 tm_new_mem_map_t
**slot
;
1367 tm_new_mem_map_t elt
, *elt_p
;
1369 enum thread_memory_type retval
= mem_transaction_local
;
1372 || TREE_CODE (x
) != SSA_NAME
1373 /* Possible uninitialized use, or a function argument. In
1374 either case, we don't care. */
1375 || SSA_NAME_IS_DEFAULT_DEF (x
))
1376 return mem_non_local
;
1378 /* Look in cache first. */
1380 slot
= tm_new_mem_hash
->find_slot (&elt
, INSERT
);
1383 return elt_p
->local_new_memory
;
1385 /* Optimistically assume the memory is transaction local during
1386 processing. This catches recursion into this variable. */
1387 *slot
= elt_p
= XNEW (tm_new_mem_map_t
);
1389 elt_p
->local_new_memory
= mem_transaction_local
;
1391 /* Search DEF chain to find the original definition of this address. */
1394 if (ptr_deref_may_alias_global_p (x
))
1396 /* Address escapes. This is not thread-private. */
1397 retval
= mem_non_local
;
1398 goto new_memory_ret
;
1401 stmt
= SSA_NAME_DEF_STMT (x
);
1403 /* If the malloc call is outside the transaction, this is
1405 if (retval
!= mem_thread_local
1406 && !dominated_by_p (CDI_DOMINATORS
, gimple_bb (stmt
), entry_block
))
1407 retval
= mem_thread_local
;
1409 if (is_gimple_assign (stmt
))
1411 code
= gimple_assign_rhs_code (stmt
);
1412 /* x = foo ==> foo */
1413 if (code
== SSA_NAME
)
1414 x
= gimple_assign_rhs1 (stmt
);
1415 /* x = foo + n ==> foo */
1416 else if (code
== POINTER_PLUS_EXPR
)
1417 x
= gimple_assign_rhs1 (stmt
);
1418 /* x = (cast*) foo ==> foo */
1419 else if (code
== VIEW_CONVERT_EXPR
|| CONVERT_EXPR_CODE_P (code
))
1420 x
= gimple_assign_rhs1 (stmt
);
1421 /* x = c ? op1 : op2 == > op1 or op2 just like a PHI */
1422 else if (code
== COND_EXPR
)
1424 tree op1
= gimple_assign_rhs2 (stmt
);
1425 tree op2
= gimple_assign_rhs3 (stmt
);
1426 enum thread_memory_type mem
;
1427 retval
= thread_private_new_memory (entry_block
, op1
);
1428 if (retval
== mem_non_local
)
1429 goto new_memory_ret
;
1430 mem
= thread_private_new_memory (entry_block
, op2
);
1431 retval
= MIN (retval
, mem
);
1432 goto new_memory_ret
;
1436 retval
= mem_non_local
;
1437 goto new_memory_ret
;
1442 if (gimple_code (stmt
) == GIMPLE_PHI
)
1445 enum thread_memory_type mem
;
1446 tree phi_result
= gimple_phi_result (stmt
);
1448 /* If any of the ancestors are non-local, we are sure to
1449 be non-local. Otherwise we can avoid doing anything
1450 and inherit what has already been generated. */
1452 for (i
= 0; i
< gimple_phi_num_args (stmt
); ++i
)
1454 tree op
= PHI_ARG_DEF (stmt
, i
);
1456 /* Exclude self-assignment. */
1457 if (phi_result
== op
)
1460 mem
= thread_private_new_memory (entry_block
, op
);
1461 if (mem
== mem_non_local
)
1464 goto new_memory_ret
;
1466 retval
= MIN (retval
, mem
);
1468 goto new_memory_ret
;
1473 while (TREE_CODE (x
) == SSA_NAME
);
1475 if (stmt
&& is_gimple_call (stmt
) && gimple_call_flags (stmt
) & ECF_MALLOC
)
1476 /* Thread-local or transaction-local. */
1479 retval
= mem_non_local
;
1482 elt_p
->local_new_memory
= retval
;
1486 /* Determine whether X has to be instrumented using a read
1489 ENTRY_BLOCK is the entry block for the region where stmt resides
1490 in. NULL if unknown.
1492 STMT is the statement in which X occurs in. It is used for thread
1493 private memory instrumentation. If no TPM instrumentation is
1494 desired, STMT should be null. */
1496 requires_barrier (basic_block entry_block
, tree x
, gimple stmt
)
1499 while (handled_component_p (x
))
1500 x
= TREE_OPERAND (x
, 0);
1502 switch (TREE_CODE (x
))
1507 enum thread_memory_type ret
;
1509 ret
= thread_private_new_memory (entry_block
, TREE_OPERAND (x
, 0));
1510 if (ret
== mem_non_local
)
1512 if (stmt
&& ret
== mem_thread_local
)
1513 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */
1514 tm_log_add (entry_block
, orig
, stmt
);
1516 /* Transaction-locals require nothing at all. For malloc, a
1517 transaction restart frees the memory and we reallocate.
1518 For alloca, the stack pointer gets reset by the retry and
1523 case TARGET_MEM_REF
:
1524 if (TREE_CODE (TMR_BASE (x
)) != ADDR_EXPR
)
1526 x
= TREE_OPERAND (TMR_BASE (x
), 0);
1527 if (TREE_CODE (x
) == PARM_DECL
)
1529 gcc_assert (TREE_CODE (x
) == VAR_DECL
);
1535 if (DECL_BY_REFERENCE (x
))
1537 /* ??? This value is a pointer, but aggregate_value_p has been
1538 jigged to return true which confuses needs_to_live_in_memory.
1539 This ought to be cleaned up generically.
1541 FIXME: Verify this still happens after the next mainline
1542 merge. Testcase ie g++.dg/tm/pr47554.C.
1547 if (is_global_var (x
))
1548 return !TREE_READONLY (x
);
1549 if (/* FIXME: This condition should actually go below in the
1550 tm_log_add() call, however is_call_clobbered() depends on
1551 aliasing info which is not available during
1552 gimplification. Since requires_barrier() gets called
1553 during lower_sequence_tm/gimplification, leave the call
1554 to needs_to_live_in_memory until we eliminate
1555 lower_sequence_tm altogether. */
1556 needs_to_live_in_memory (x
))
1560 /* For local memory that doesn't escape (aka thread private
1561 memory), we can either save the value at the beginning of
1562 the transaction and restore on restart, or call a tm
1563 function to dynamically save and restore on restart
1566 tm_log_add (entry_block
, orig
, stmt
);
1575 /* Mark the GIMPLE_ASSIGN statement as appropriate for being inside
1576 a transaction region. */
1579 examine_assign_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1581 gimple stmt
= gsi_stmt (*gsi
);
1583 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_rhs1 (stmt
), NULL
))
1584 *state
|= GTMA_HAVE_LOAD
;
1585 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_lhs (stmt
), NULL
))
1586 *state
|= GTMA_HAVE_STORE
;
1589 /* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */
1592 examine_call_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1594 gimple stmt
= gsi_stmt (*gsi
);
1597 if (is_tm_pure_call (stmt
))
1600 /* Check if this call is a transaction abort. */
1601 fn
= gimple_call_fndecl (stmt
);
1602 if (is_tm_abort (fn
))
1603 *state
|= GTMA_HAVE_ABORT
;
1605 /* Note that something may happen. */
1606 *state
|= GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
;
1609 /* Lower a GIMPLE_TRANSACTION statement. */
1612 lower_transaction (gimple_stmt_iterator
*gsi
, struct walk_stmt_info
*wi
)
1615 gtransaction
*stmt
= as_a
<gtransaction
*> (gsi_stmt (*gsi
));
1616 unsigned int *outer_state
= (unsigned int *) wi
->info
;
1617 unsigned int this_state
= 0;
1618 struct walk_stmt_info this_wi
;
1620 /* First, lower the body. The scanning that we do inside gives
1621 us some idea of what we're dealing with. */
1622 memset (&this_wi
, 0, sizeof (this_wi
));
1623 this_wi
.info
= (void *) &this_state
;
1624 walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt
),
1625 lower_sequence_tm
, NULL
, &this_wi
);
1627 /* If there was absolutely nothing transaction related inside the
1628 transaction, we may elide it. Likewise if this is a nested
1629 transaction and does not contain an abort. */
1631 || (!(this_state
& GTMA_HAVE_ABORT
) && outer_state
!= NULL
))
1634 *outer_state
|= this_state
;
1636 gsi_insert_seq_before (gsi
, gimple_transaction_body (stmt
),
1638 gimple_transaction_set_body (stmt
, NULL
);
1640 gsi_remove (gsi
, true);
1641 wi
->removed_stmt
= true;
1645 /* Wrap the body of the transaction in a try-finally node so that
1646 the commit call is always properly called. */
1647 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT
), 0);
1648 if (flag_exceptions
)
1651 gimple_seq n_seq
, e_seq
;
1653 n_seq
= gimple_seq_alloc_with_stmt (g
);
1656 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER
),
1657 1, integer_zero_node
);
1658 ptr
= create_tmp_var (ptr_type_node
, NULL
);
1659 gimple_call_set_lhs (g
, ptr
);
1660 gimple_seq_add_stmt (&e_seq
, g
);
1662 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH
),
1664 gimple_seq_add_stmt (&e_seq
, g
);
1666 g
= gimple_build_eh_else (n_seq
, e_seq
);
1669 g
= gimple_build_try (gimple_transaction_body (stmt
),
1670 gimple_seq_alloc_with_stmt (g
), GIMPLE_TRY_FINALLY
);
1671 gsi_insert_after (gsi
, g
, GSI_CONTINUE_LINKING
);
1673 gimple_transaction_set_body (stmt
, NULL
);
1675 /* If the transaction calls abort or if this is an outer transaction,
1676 add an "over" label afterwards. */
1677 if ((this_state
& (GTMA_HAVE_ABORT
))
1678 || (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
))
1680 tree label
= create_artificial_label (UNKNOWN_LOCATION
);
1681 gimple_transaction_set_label (stmt
, label
);
1682 gsi_insert_after (gsi
, gimple_build_label (label
), GSI_CONTINUE_LINKING
);
1685 /* Record the set of operations found for use later. */
1686 this_state
|= gimple_transaction_subcode (stmt
) & GTMA_DECLARATION_MASK
;
1687 gimple_transaction_set_subcode (stmt
, this_state
);
1690 /* Iterate through the statements in the sequence, lowering them all
1691 as appropriate for being in a transaction. */
1694 lower_sequence_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1695 struct walk_stmt_info
*wi
)
1697 unsigned int *state
= (unsigned int *) wi
->info
;
1698 gimple stmt
= gsi_stmt (*gsi
);
1700 *handled_ops_p
= true;
1701 switch (gimple_code (stmt
))
1704 /* Only memory reads/writes need to be instrumented. */
1705 if (gimple_assign_single_p (stmt
))
1706 examine_assign_tm (state
, gsi
);
1710 examine_call_tm (state
, gsi
);
1714 *state
|= GTMA_MAY_ENTER_IRREVOCABLE
;
1717 case GIMPLE_TRANSACTION
:
1718 lower_transaction (gsi
, wi
);
1722 *handled_ops_p
= !gimple_has_substatements (stmt
);
1729 /* Iterate through the statements in the sequence, lowering them all
1730 as appropriate for being outside of a transaction. */
1733 lower_sequence_no_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1734 struct walk_stmt_info
* wi
)
1736 gimple stmt
= gsi_stmt (*gsi
);
1738 if (gimple_code (stmt
) == GIMPLE_TRANSACTION
)
1740 *handled_ops_p
= true;
1741 lower_transaction (gsi
, wi
);
1744 *handled_ops_p
= !gimple_has_substatements (stmt
);
1749 /* Main entry point for flattening GIMPLE_TRANSACTION constructs. After
1750 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has
1751 been moved out, and all the data required for constructing a proper
1752 CFG has been recorded. */
1755 execute_lower_tm (void)
1757 struct walk_stmt_info wi
;
1760 /* Transactional clones aren't created until a later pass. */
1761 gcc_assert (!decl_is_tm_clone (current_function_decl
));
1763 body
= gimple_body (current_function_decl
);
1764 memset (&wi
, 0, sizeof (wi
));
1765 walk_gimple_seq_mod (&body
, lower_sequence_no_tm
, NULL
, &wi
);
1766 gimple_set_body (current_function_decl
, body
);
1773 const pass_data pass_data_lower_tm
=
1775 GIMPLE_PASS
, /* type */
1776 "tmlower", /* name */
1777 OPTGROUP_NONE
, /* optinfo_flags */
1778 TV_TRANS_MEM
, /* tv_id */
1779 PROP_gimple_lcf
, /* properties_required */
1780 0, /* properties_provided */
1781 0, /* properties_destroyed */
1782 0, /* todo_flags_start */
1783 0, /* todo_flags_finish */
1786 class pass_lower_tm
: public gimple_opt_pass
1789 pass_lower_tm (gcc::context
*ctxt
)
1790 : gimple_opt_pass (pass_data_lower_tm
, ctxt
)
1793 /* opt_pass methods: */
1794 virtual bool gate (function
*) { return flag_tm
; }
1795 virtual unsigned int execute (function
*) { return execute_lower_tm (); }
1797 }; // class pass_lower_tm
1802 make_pass_lower_tm (gcc::context
*ctxt
)
1804 return new pass_lower_tm (ctxt
);
1807 /* Collect region information for each transaction. */
1813 /* The field "transaction_stmt" is initially a gtransaction *,
1814 but eventually gets lowered to a gcall *(to BUILT_IN_TM_START).
1816 Helper method to get it as a gtransaction *, with code-checking
1817 in a checked-build. */
1820 get_transaction_stmt () const
1822 return as_a
<gtransaction
*> (transaction_stmt
);
1827 /* Link to the next unnested transaction. */
1828 struct tm_region
*next
;
1830 /* Link to the next inner transaction. */
1831 struct tm_region
*inner
;
1833 /* Link to the next outer transaction. */
1834 struct tm_region
*outer
;
1836 /* The GIMPLE_TRANSACTION statement beginning this transaction.
1837 After TM_MARK, this gets replaced by a call to
1839 Hence this will be either a gtransaction *or a gcall *. */
1840 gimple transaction_stmt
;
1842 /* After TM_MARK expands the GIMPLE_TRANSACTION into a call to
1843 BUILT_IN_TM_START, this field is true if the transaction is an
1844 outer transaction. */
1845 bool original_transaction_was_outer
;
1847 /* Return value from BUILT_IN_TM_START. */
1850 /* The entry block to this region. This will always be the first
1851 block of the body of the transaction. */
1852 basic_block entry_block
;
1854 /* The first block after an expanded call to _ITM_beginTransaction. */
1855 basic_block restart_block
;
1857 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK.
1858 These blocks are still a part of the region (i.e., the border is
1859 inclusive). Note that this set is only complete for paths in the CFG
1860 starting at ENTRY_BLOCK, and that there is no exit block recorded for
1861 the edge to the "over" label. */
1864 /* The set of all blocks that have an TM_IRREVOCABLE call. */
1868 typedef struct tm_region
*tm_region_p
;
1870 /* True if there are pending edge statements to be committed for the
1871 current function being scanned in the tmmark pass. */
1872 bool pending_edge_inserts_p
;
1874 static struct tm_region
*all_tm_regions
;
1875 static bitmap_obstack tm_obstack
;
1878 /* A subroutine of tm_region_init. Record the existence of the
1879 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */
1881 static struct tm_region
*
1882 tm_region_init_0 (struct tm_region
*outer
, basic_block bb
,
1885 struct tm_region
*region
;
1887 region
= (struct tm_region
*)
1888 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1892 region
->next
= outer
->inner
;
1893 outer
->inner
= region
;
1897 region
->next
= all_tm_regions
;
1898 all_tm_regions
= region
;
1900 region
->inner
= NULL
;
1901 region
->outer
= outer
;
1903 region
->transaction_stmt
= stmt
;
1904 region
->original_transaction_was_outer
= false;
1905 region
->tm_state
= NULL
;
1907 /* There are either one or two edges out of the block containing
1908 the GIMPLE_TRANSACTION, one to the actual region and one to the
1909 "over" label if the region contains an abort. The former will
1910 always be the one marked FALLTHRU. */
1911 region
->entry_block
= FALLTHRU_EDGE (bb
)->dest
;
1913 region
->exit_blocks
= BITMAP_ALLOC (&tm_obstack
);
1914 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1919 /* A subroutine of tm_region_init. Record all the exit and
1920 irrevocable blocks in BB into the region's exit_blocks and
1921 irr_blocks bitmaps. Returns the new region being scanned. */
1923 static struct tm_region
*
1924 tm_region_init_1 (struct tm_region
*region
, basic_block bb
)
1926 gimple_stmt_iterator gsi
;
1930 || (!region
->irr_blocks
&& !region
->exit_blocks
))
1933 /* Check to see if this is the end of a region by seeing if it
1934 contains a call to __builtin_tm_commit{,_eh}. Note that the
1935 outermost region for DECL_IS_TM_CLONE need not collect this. */
1936 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
1939 if (gimple_code (g
) == GIMPLE_CALL
)
1941 tree fn
= gimple_call_fndecl (g
);
1942 if (fn
&& DECL_BUILT_IN_CLASS (fn
) == BUILT_IN_NORMAL
)
1944 if ((DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT
1945 || DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT_EH
)
1946 && region
->exit_blocks
)
1948 bitmap_set_bit (region
->exit_blocks
, bb
->index
);
1949 region
= region
->outer
;
1952 if (DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_IRREVOCABLE
)
1953 bitmap_set_bit (region
->irr_blocks
, bb
->index
);
1960 /* Collect all of the transaction regions within the current function
1961 and record them in ALL_TM_REGIONS. The REGION parameter may specify
1962 an "outermost" region for use by tm clones. */
1965 tm_region_init (struct tm_region
*region
)
1971 auto_vec
<basic_block
> queue
;
1972 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
1973 struct tm_region
*old_region
;
1974 auto_vec
<tm_region_p
> bb_regions
;
1976 all_tm_regions
= region
;
1977 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1979 /* We could store this information in bb->aux, but we may get called
1980 through get_all_tm_blocks() from another pass that may be already
1982 bb_regions
.safe_grow_cleared (last_basic_block_for_fn (cfun
));
1984 queue
.safe_push (bb
);
1985 bb_regions
[bb
->index
] = region
;
1989 region
= bb_regions
[bb
->index
];
1990 bb_regions
[bb
->index
] = NULL
;
1992 /* Record exit and irrevocable blocks. */
1993 region
= tm_region_init_1 (region
, bb
);
1995 /* Check for the last statement in the block beginning a new region. */
1997 old_region
= region
;
1999 if (gtransaction
*trans_stmt
= dyn_cast
<gtransaction
*> (g
))
2000 region
= tm_region_init_0 (region
, bb
, trans_stmt
);
2002 /* Process subsequent blocks. */
2003 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2004 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2006 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2007 queue
.safe_push (e
->dest
);
2009 /* If the current block started a new region, make sure that only
2010 the entry block of the new region is associated with this region.
2011 Other successors are still part of the old region. */
2012 if (old_region
!= region
&& e
->dest
!= region
->entry_block
)
2013 bb_regions
[e
->dest
->index
] = old_region
;
2015 bb_regions
[e
->dest
->index
] = region
;
2018 while (!queue
.is_empty ());
2019 BITMAP_FREE (visited_blocks
);
2022 /* The "gate" function for all transactional memory expansion and optimization
2023 passes. We collect region information for each top-level transaction, and
2024 if we don't find any, we skip all of the TM passes. Each region will have
2025 all of the exit blocks recorded, and the originating statement. */
2033 calculate_dominance_info (CDI_DOMINATORS
);
2034 bitmap_obstack_initialize (&tm_obstack
);
2036 /* If the function is a TM_CLONE, then the entire function is the region. */
2037 if (decl_is_tm_clone (current_function_decl
))
2039 struct tm_region
*region
= (struct tm_region
*)
2040 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
2041 memset (region
, 0, sizeof (*region
));
2042 region
->entry_block
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
2043 /* For a clone, the entire function is the region. But even if
2044 we don't need to record any exit blocks, we may need to
2045 record irrevocable blocks. */
2046 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
2048 tm_region_init (region
);
2052 tm_region_init (NULL
);
2054 /* If we didn't find any regions, cleanup and skip the whole tree
2055 of tm-related optimizations. */
2056 if (all_tm_regions
== NULL
)
2058 bitmap_obstack_release (&tm_obstack
);
2068 const pass_data pass_data_tm_init
=
2070 GIMPLE_PASS
, /* type */
2071 "*tminit", /* name */
2072 OPTGROUP_NONE
, /* optinfo_flags */
2073 TV_TRANS_MEM
, /* tv_id */
2074 ( PROP_ssa
| PROP_cfg
), /* properties_required */
2075 0, /* properties_provided */
2076 0, /* properties_destroyed */
2077 0, /* todo_flags_start */
2078 0, /* todo_flags_finish */
2081 class pass_tm_init
: public gimple_opt_pass
2084 pass_tm_init (gcc::context
*ctxt
)
2085 : gimple_opt_pass (pass_data_tm_init
, ctxt
)
2088 /* opt_pass methods: */
2089 virtual bool gate (function
*) { return gate_tm_init (); }
2091 }; // class pass_tm_init
2096 make_pass_tm_init (gcc::context
*ctxt
)
2098 return new pass_tm_init (ctxt
);
2101 /* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region
2102 represented by STATE. */
2105 transaction_subcode_ior (struct tm_region
*region
, unsigned flags
)
2107 if (region
&& region
->transaction_stmt
)
2109 gtransaction
*transaction_stmt
= region
->get_transaction_stmt ();
2110 flags
|= gimple_transaction_subcode (transaction_stmt
);
2111 gimple_transaction_set_subcode (transaction_stmt
, flags
);
2115 /* Construct a memory load in a transactional context. Return the
2116 gimple statement performing the load, or NULL if there is no
2117 TM_LOAD builtin of the appropriate size to do the load.
2119 LOC is the location to use for the new statement(s). */
2122 build_tm_load (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2124 enum built_in_function code
= END_BUILTINS
;
2125 tree t
, type
= TREE_TYPE (rhs
), decl
;
2128 if (type
== float_type_node
)
2129 code
= BUILT_IN_TM_LOAD_FLOAT
;
2130 else if (type
== double_type_node
)
2131 code
= BUILT_IN_TM_LOAD_DOUBLE
;
2132 else if (type
== long_double_type_node
)
2133 code
= BUILT_IN_TM_LOAD_LDOUBLE
;
2134 else if (TYPE_SIZE_UNIT (type
) != NULL
2135 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2137 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2140 code
= BUILT_IN_TM_LOAD_1
;
2143 code
= BUILT_IN_TM_LOAD_2
;
2146 code
= BUILT_IN_TM_LOAD_4
;
2149 code
= BUILT_IN_TM_LOAD_8
;
2154 if (code
== END_BUILTINS
)
2156 decl
= targetm
.vectorize
.builtin_tm_load (type
);
2161 decl
= builtin_decl_explicit (code
);
2163 t
= gimplify_addr (gsi
, rhs
);
2164 gcall
= gimple_build_call (decl
, 1, t
);
2165 gimple_set_location (gcall
, loc
);
2167 t
= TREE_TYPE (TREE_TYPE (decl
));
2168 if (useless_type_conversion_p (type
, t
))
2170 gimple_call_set_lhs (gcall
, lhs
);
2171 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2178 temp
= create_tmp_reg (t
, NULL
);
2179 gimple_call_set_lhs (gcall
, temp
);
2180 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2182 t
= fold_build1 (VIEW_CONVERT_EXPR
, type
, temp
);
2183 g
= gimple_build_assign (lhs
, t
);
2184 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2191 /* Similarly for storing TYPE in a transactional context. */
2194 build_tm_store (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2196 enum built_in_function code
= END_BUILTINS
;
2197 tree t
, fn
, type
= TREE_TYPE (rhs
), simple_type
;
2200 if (type
== float_type_node
)
2201 code
= BUILT_IN_TM_STORE_FLOAT
;
2202 else if (type
== double_type_node
)
2203 code
= BUILT_IN_TM_STORE_DOUBLE
;
2204 else if (type
== long_double_type_node
)
2205 code
= BUILT_IN_TM_STORE_LDOUBLE
;
2206 else if (TYPE_SIZE_UNIT (type
) != NULL
2207 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2209 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2212 code
= BUILT_IN_TM_STORE_1
;
2215 code
= BUILT_IN_TM_STORE_2
;
2218 code
= BUILT_IN_TM_STORE_4
;
2221 code
= BUILT_IN_TM_STORE_8
;
2226 if (code
== END_BUILTINS
)
2228 fn
= targetm
.vectorize
.builtin_tm_store (type
);
2233 fn
= builtin_decl_explicit (code
);
2235 simple_type
= TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn
))));
2237 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
2239 /* Handle the easy initialization to zero. */
2240 if (!CONSTRUCTOR_ELTS (rhs
))
2241 rhs
= build_int_cst (simple_type
, 0);
2244 /* ...otherwise punt to the caller and probably use
2245 BUILT_IN_TM_MEMMOVE, because we can't wrap a
2246 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce
2251 else if (!useless_type_conversion_p (simple_type
, type
))
2256 temp
= create_tmp_reg (simple_type
, NULL
);
2257 t
= fold_build1 (VIEW_CONVERT_EXPR
, simple_type
, rhs
);
2258 g
= gimple_build_assign (temp
, t
);
2259 gimple_set_location (g
, loc
);
2260 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2265 t
= gimplify_addr (gsi
, lhs
);
2266 gcall
= gimple_build_call (fn
, 2, t
, rhs
);
2267 gimple_set_location (gcall
, loc
);
2268 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2274 /* Expand an assignment statement into transactional builtins. */
2277 expand_assign_tm (struct tm_region
*region
, gimple_stmt_iterator
*gsi
)
2279 gimple stmt
= gsi_stmt (*gsi
);
2280 location_t loc
= gimple_location (stmt
);
2281 tree lhs
= gimple_assign_lhs (stmt
);
2282 tree rhs
= gimple_assign_rhs1 (stmt
);
2283 bool store_p
= requires_barrier (region
->entry_block
, lhs
, NULL
);
2284 bool load_p
= requires_barrier (region
->entry_block
, rhs
, NULL
);
2285 gimple gcall
= NULL
;
2287 if (!load_p
&& !store_p
)
2289 /* Add thread private addresses to log if applicable. */
2290 requires_barrier (region
->entry_block
, lhs
, stmt
);
2295 // Remove original load/store statement.
2296 gsi_remove (gsi
, true);
2298 if (load_p
&& !store_p
)
2300 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2301 gcall
= build_tm_load (loc
, lhs
, rhs
, gsi
);
2303 else if (store_p
&& !load_p
)
2305 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2306 gcall
= build_tm_store (loc
, lhs
, rhs
, gsi
);
2310 tree lhs_addr
, rhs_addr
, tmp
;
2313 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2315 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2317 /* ??? Figure out if there's any possible overlap between the LHS
2318 and the RHS and if not, use MEMCPY. */
2320 if (load_p
&& is_gimple_reg (lhs
))
2322 tmp
= create_tmp_var (TREE_TYPE (lhs
), NULL
);
2323 lhs_addr
= build_fold_addr_expr (tmp
);
2328 lhs_addr
= gimplify_addr (gsi
, lhs
);
2330 rhs_addr
= gimplify_addr (gsi
, rhs
);
2331 gcall
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
),
2332 3, lhs_addr
, rhs_addr
,
2333 TYPE_SIZE_UNIT (TREE_TYPE (lhs
)));
2334 gimple_set_location (gcall
, loc
);
2335 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2339 gcall
= gimple_build_assign (lhs
, tmp
);
2340 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2344 /* Now that we have the load/store in its instrumented form, add
2345 thread private addresses to the log if applicable. */
2347 requires_barrier (region
->entry_block
, lhs
, gcall
);
2349 // The calls to build_tm_{store,load} above inserted the instrumented
2350 // call into the stream.
2351 // gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2355 /* Expand a call statement as appropriate for a transaction. That is,
2356 either verify that the call does not affect the transaction, or
2357 redirect the call to a clone that handles transactions, or change
2358 the transaction state to IRREVOCABLE. Return true if the call is
2359 one of the builtins that end a transaction. */
2362 expand_call_tm (struct tm_region
*region
,
2363 gimple_stmt_iterator
*gsi
)
2365 gcall
*stmt
= as_a
<gcall
*> (gsi_stmt (*gsi
));
2366 tree lhs
= gimple_call_lhs (stmt
);
2368 struct cgraph_node
*node
;
2369 bool retval
= false;
2371 fn_decl
= gimple_call_fndecl (stmt
);
2373 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMCPY
)
2374 || fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
))
2375 transaction_subcode_ior (region
, GTMA_HAVE_STORE
| GTMA_HAVE_LOAD
);
2376 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMSET
))
2377 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2379 if (is_tm_pure_call (stmt
))
2383 retval
= is_tm_ending_fndecl (fn_decl
);
2386 /* Assume all non-const/pure calls write to memory, except
2387 transaction ending builtins. */
2388 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2391 /* For indirect calls, we already generated a call into the runtime. */
2394 tree fn
= gimple_call_fn (stmt
);
2396 /* We are guaranteed never to go irrevocable on a safe or pure
2397 call, and the pure call was handled above. */
2398 if (is_tm_safe (fn
))
2401 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2406 node
= cgraph_node::get (fn_decl
);
2407 /* All calls should have cgraph here. */
2410 /* We can have a nodeless call here if some pass after IPA-tm
2411 added uninstrumented calls. For example, loop distribution
2412 can transform certain loop constructs into __builtin_mem*
2413 calls. In this case, see if we have a suitable TM
2414 replacement and fill in the gaps. */
2415 gcc_assert (DECL_BUILT_IN_CLASS (fn_decl
) == BUILT_IN_NORMAL
);
2416 enum built_in_function code
= DECL_FUNCTION_CODE (fn_decl
);
2417 gcc_assert (code
== BUILT_IN_MEMCPY
2418 || code
== BUILT_IN_MEMMOVE
2419 || code
== BUILT_IN_MEMSET
);
2421 tree repl
= find_tm_replacement_function (fn_decl
);
2424 gimple_call_set_fndecl (stmt
, repl
);
2426 node
= cgraph_node::create (repl
);
2427 node
->local
.tm_may_enter_irr
= false;
2428 return expand_call_tm (region
, gsi
);
2432 if (node
->local
.tm_may_enter_irr
)
2433 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2435 if (is_tm_abort (fn_decl
))
2437 transaction_subcode_ior (region
, GTMA_HAVE_ABORT
);
2441 /* Instrument the store if needed.
2443 If the assignment happens inside the function call (return slot
2444 optimization), there is no instrumentation to be done, since
2445 the callee should have done the right thing. */
2446 if (lhs
&& requires_barrier (region
->entry_block
, lhs
, stmt
)
2447 && !gimple_call_return_slot_opt_p (stmt
))
2449 tree tmp
= create_tmp_reg (TREE_TYPE (lhs
), NULL
);
2450 location_t loc
= gimple_location (stmt
);
2451 edge fallthru_edge
= NULL
;
2452 gassign
*assign_stmt
;
2454 /* Remember if the call was going to throw. */
2455 if (stmt_can_throw_internal (stmt
))
2459 basic_block bb
= gimple_bb (stmt
);
2461 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2462 if (e
->flags
& EDGE_FALLTHRU
)
2469 gimple_call_set_lhs (stmt
, tmp
);
2471 assign_stmt
= gimple_build_assign (lhs
, tmp
);
2472 gimple_set_location (assign_stmt
, loc
);
2474 /* We cannot throw in the middle of a BB. If the call was going
2475 to throw, place the instrumentation on the fallthru edge, so
2476 the call remains the last statement in the block. */
2479 gimple_seq fallthru_seq
= gimple_seq_alloc_with_stmt (assign_stmt
);
2480 gimple_stmt_iterator fallthru_gsi
= gsi_start (fallthru_seq
);
2481 expand_assign_tm (region
, &fallthru_gsi
);
2482 gsi_insert_seq_on_edge (fallthru_edge
, fallthru_seq
);
2483 pending_edge_inserts_p
= true;
2487 gsi_insert_after (gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
2488 expand_assign_tm (region
, gsi
);
2491 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2498 /* Expand all statements in BB as appropriate for being inside
2502 expand_block_tm (struct tm_region
*region
, basic_block bb
)
2504 gimple_stmt_iterator gsi
;
2506 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2508 gimple stmt
= gsi_stmt (gsi
);
2509 switch (gimple_code (stmt
))
2512 /* Only memory reads/writes need to be instrumented. */
2513 if (gimple_assign_single_p (stmt
)
2514 && !gimple_clobber_p (stmt
))
2516 expand_assign_tm (region
, &gsi
);
2522 if (expand_call_tm (region
, &gsi
))
2532 if (!gsi_end_p (gsi
))
2537 /* Return the list of basic-blocks in REGION.
2539 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks
2540 following a TM_IRREVOCABLE call.
2542 INCLUDE_UNINSTRUMENTED_P is TRUE if we should include the
2543 uninstrumented code path blocks in the list of basic blocks
2544 returned, false otherwise. */
2546 static vec
<basic_block
>
2547 get_tm_region_blocks (basic_block entry_block
,
2550 bitmap all_region_blocks
,
2551 bool stop_at_irrevocable_p
,
2552 bool include_uninstrumented_p
= true)
2554 vec
<basic_block
> bbs
= vNULL
;
2558 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
2561 bbs
.safe_push (entry_block
);
2562 bitmap_set_bit (visited_blocks
, entry_block
->index
);
2566 basic_block bb
= bbs
[i
++];
2569 bitmap_bit_p (exit_blocks
, bb
->index
))
2572 if (stop_at_irrevocable_p
2574 && bitmap_bit_p (irr_blocks
, bb
->index
))
2577 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2578 if ((include_uninstrumented_p
2579 || !(e
->flags
& EDGE_TM_UNINSTRUMENTED
))
2580 && !bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2582 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2583 bbs
.safe_push (e
->dest
);
2586 while (i
< bbs
.length ());
2588 if (all_region_blocks
)
2589 bitmap_ior_into (all_region_blocks
, visited_blocks
);
2591 BITMAP_FREE (visited_blocks
);
2595 // Callback data for collect_bb2reg.
2598 vec
<tm_region_p
> *bb2reg
;
2599 bool include_uninstrumented_p
;
2602 // Callback for expand_regions, collect innermost region data for each bb.
2604 collect_bb2reg (struct tm_region
*region
, void *data
)
2606 struct bb2reg_stuff
*stuff
= (struct bb2reg_stuff
*)data
;
2607 vec
<tm_region_p
> *bb2reg
= stuff
->bb2reg
;
2608 vec
<basic_block
> queue
;
2612 queue
= get_tm_region_blocks (region
->entry_block
,
2613 region
->exit_blocks
,
2616 /*stop_at_irr_p=*/true,
2617 stuff
->include_uninstrumented_p
);
2619 // We expect expand_region to perform a post-order traversal of the region
2620 // tree. Therefore the last region seen for any bb is the innermost.
2621 FOR_EACH_VEC_ELT (queue
, i
, bb
)
2622 (*bb2reg
)[bb
->index
] = region
;
2628 // Returns a vector, indexed by BB->INDEX, of the innermost tm_region to
2629 // which a basic block belongs. Note that we only consider the instrumented
2630 // code paths for the region; the uninstrumented code paths are ignored if
2631 // INCLUDE_UNINSTRUMENTED_P is false.
2633 // ??? This data is very similar to the bb_regions array that is collected
2634 // during tm_region_init. Or, rather, this data is similar to what could
2635 // be used within tm_region_init. The actual computation in tm_region_init
2636 // begins and ends with bb_regions entirely full of NULL pointers, due to
2637 // the way in which pointers are swapped in and out of the array.
2639 // ??? Our callers expect that blocks are not shared between transactions.
2640 // When the optimizers get too smart, and blocks are shared, then during
2641 // the tm_mark phase we'll add log entries to only one of the two transactions,
2642 // and in the tm_edge phase we'll add edges to the CFG that create invalid
2643 // cycles. The symptom being SSA defs that do not dominate their uses.
2644 // Note that the optimizers were locally correct with their transformation,
2645 // as we have no info within the program that suggests that the blocks cannot
2648 // ??? There is currently a hack inside tree-ssa-pre.c to work around the
2649 // only known instance of this block sharing.
2651 static vec
<tm_region_p
>
2652 get_bb_regions_instrumented (bool traverse_clones
,
2653 bool include_uninstrumented_p
)
2655 unsigned n
= last_basic_block_for_fn (cfun
);
2656 struct bb2reg_stuff stuff
;
2657 vec
<tm_region_p
> ret
;
2660 ret
.safe_grow_cleared (n
);
2661 stuff
.bb2reg
= &ret
;
2662 stuff
.include_uninstrumented_p
= include_uninstrumented_p
;
2663 expand_regions (all_tm_regions
, collect_bb2reg
, &stuff
, traverse_clones
);
2668 /* Set the IN_TRANSACTION for all gimple statements that appear in a
2672 compute_transaction_bits (void)
2674 struct tm_region
*region
;
2675 vec
<basic_block
> queue
;
2679 /* ?? Perhaps we need to abstract gate_tm_init further, because we
2680 certainly don't need it to calculate CDI_DOMINATOR info. */
2683 FOR_EACH_BB_FN (bb
, cfun
)
2684 bb
->flags
&= ~BB_IN_TRANSACTION
;
2686 for (region
= all_tm_regions
; region
; region
= region
->next
)
2688 queue
= get_tm_region_blocks (region
->entry_block
,
2689 region
->exit_blocks
,
2692 /*stop_at_irr_p=*/true);
2693 for (i
= 0; queue
.iterate (i
, &bb
); ++i
)
2694 bb
->flags
|= BB_IN_TRANSACTION
;
2699 bitmap_obstack_release (&tm_obstack
);
2702 /* Replace the GIMPLE_TRANSACTION in this region with the corresponding
2703 call to BUILT_IN_TM_START. */
2706 expand_transaction (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2708 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2709 basic_block transaction_bb
= gimple_bb (region
->transaction_stmt
);
2710 tree tm_state
= region
->tm_state
;
2711 tree tm_state_type
= TREE_TYPE (tm_state
);
2712 edge abort_edge
= NULL
;
2713 edge inst_edge
= NULL
;
2714 edge uninst_edge
= NULL
;
2715 edge fallthru_edge
= NULL
;
2717 // Identify the various successors of the transaction start.
2721 FOR_EACH_EDGE (e
, i
, transaction_bb
->succs
)
2723 if (e
->flags
& EDGE_TM_ABORT
)
2725 else if (e
->flags
& EDGE_TM_UNINSTRUMENTED
)
2729 if (e
->flags
& EDGE_FALLTHRU
)
2734 /* ??? There are plenty of bits here we're not computing. */
2736 int subcode
= gimple_transaction_subcode (region
->get_transaction_stmt ());
2738 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2739 flags
|= PR_DOESGOIRREVOCABLE
;
2740 if ((subcode
& GTMA_MAY_ENTER_IRREVOCABLE
) == 0)
2741 flags
|= PR_HASNOIRREVOCABLE
;
2742 /* If the transaction does not have an abort in lexical scope and is not
2743 marked as an outer transaction, then it will never abort. */
2744 if ((subcode
& GTMA_HAVE_ABORT
) == 0 && (subcode
& GTMA_IS_OUTER
) == 0)
2745 flags
|= PR_HASNOABORT
;
2746 if ((subcode
& GTMA_HAVE_STORE
) == 0)
2747 flags
|= PR_READONLY
;
2748 if (inst_edge
&& !(subcode
& GTMA_HAS_NO_INSTRUMENTATION
))
2749 flags
|= PR_INSTRUMENTEDCODE
;
2751 flags
|= PR_UNINSTRUMENTEDCODE
;
2752 if (subcode
& GTMA_IS_OUTER
)
2753 region
->original_transaction_was_outer
= true;
2754 tree t
= build_int_cst (tm_state_type
, flags
);
2755 gcall
*call
= gimple_build_call (tm_start
, 1, t
);
2756 gimple_call_set_lhs (call
, tm_state
);
2757 gimple_set_location (call
, gimple_location (region
->transaction_stmt
));
2759 // Replace the GIMPLE_TRANSACTION with the call to BUILT_IN_TM_START.
2760 gimple_stmt_iterator gsi
= gsi_last_bb (transaction_bb
);
2761 gcc_assert (gsi_stmt (gsi
) == region
->transaction_stmt
);
2762 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
2763 gsi_remove (&gsi
, true);
2764 region
->transaction_stmt
= call
;
2767 // Generate log saves.
2768 if (!tm_log_save_addresses
.is_empty ())
2769 tm_log_emit_saves (region
->entry_block
, transaction_bb
);
2771 // In the beginning, we've no tests to perform on transaction restart.
2772 // Note that after this point, transaction_bb becomes the "most recent
2773 // block containing tests for the transaction".
2774 region
->restart_block
= region
->entry_block
;
2776 // Generate log restores.
2777 if (!tm_log_save_addresses
.is_empty ())
2779 basic_block test_bb
= create_empty_bb (transaction_bb
);
2780 basic_block code_bb
= create_empty_bb (test_bb
);
2781 basic_block join_bb
= create_empty_bb (code_bb
);
2782 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2783 add_bb_to_loop (code_bb
, transaction_bb
->loop_father
);
2784 add_bb_to_loop (join_bb
, transaction_bb
->loop_father
);
2785 if (region
->restart_block
== region
->entry_block
)
2786 region
->restart_block
= test_bb
;
2788 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2789 tree t2
= build_int_cst (tm_state_type
, A_RESTORELIVEVARIABLES
);
2790 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2792 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2793 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2795 t2
= build_int_cst (tm_state_type
, 0);
2796 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2797 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2799 tm_log_emit_restores (region
->entry_block
, code_bb
);
2801 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2802 edge et
= make_edge (test_bb
, code_bb
, EDGE_TRUE_VALUE
);
2803 edge ef
= make_edge (test_bb
, join_bb
, EDGE_FALSE_VALUE
);
2804 redirect_edge_pred (fallthru_edge
, join_bb
);
2806 join_bb
->frequency
= test_bb
->frequency
= transaction_bb
->frequency
;
2807 join_bb
->count
= test_bb
->count
= transaction_bb
->count
;
2809 ei
->probability
= PROB_ALWAYS
;
2810 et
->probability
= PROB_LIKELY
;
2811 ef
->probability
= PROB_UNLIKELY
;
2812 et
->count
= apply_probability (test_bb
->count
, et
->probability
);
2813 ef
->count
= apply_probability (test_bb
->count
, ef
->probability
);
2815 code_bb
->count
= et
->count
;
2816 code_bb
->frequency
= EDGE_FREQUENCY (et
);
2818 transaction_bb
= join_bb
;
2821 // If we have an ABORT edge, create a test to perform the abort.
2824 basic_block test_bb
= create_empty_bb (transaction_bb
);
2825 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2826 if (region
->restart_block
== region
->entry_block
)
2827 region
->restart_block
= test_bb
;
2829 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2830 tree t2
= build_int_cst (tm_state_type
, A_ABORTTRANSACTION
);
2831 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2833 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2834 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2836 t2
= build_int_cst (tm_state_type
, 0);
2837 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2838 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2840 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2841 test_bb
->frequency
= transaction_bb
->frequency
;
2842 test_bb
->count
= transaction_bb
->count
;
2843 ei
->probability
= PROB_ALWAYS
;
2845 // Not abort edge. If both are live, chose one at random as we'll
2846 // we'll be fixing that up below.
2847 redirect_edge_pred (fallthru_edge
, test_bb
);
2848 fallthru_edge
->flags
= EDGE_FALSE_VALUE
;
2849 fallthru_edge
->probability
= PROB_VERY_LIKELY
;
2850 fallthru_edge
->count
2851 = apply_probability (test_bb
->count
, fallthru_edge
->probability
);
2854 redirect_edge_pred (abort_edge
, test_bb
);
2855 abort_edge
->flags
= EDGE_TRUE_VALUE
;
2856 abort_edge
->probability
= PROB_VERY_UNLIKELY
;
2858 = apply_probability (test_bb
->count
, abort_edge
->probability
);
2860 transaction_bb
= test_bb
;
2863 // If we have both instrumented and uninstrumented code paths, select one.
2864 if (inst_edge
&& uninst_edge
)
2866 basic_block test_bb
= create_empty_bb (transaction_bb
);
2867 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2868 if (region
->restart_block
== region
->entry_block
)
2869 region
->restart_block
= test_bb
;
2871 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2872 tree t2
= build_int_cst (tm_state_type
, A_RUNUNINSTRUMENTEDCODE
);
2874 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2876 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2877 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2879 t2
= build_int_cst (tm_state_type
, 0);
2880 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2881 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2883 // Create the edge into test_bb first, as we want to copy values
2884 // out of the fallthru edge.
2885 edge e
= make_edge (transaction_bb
, test_bb
, fallthru_edge
->flags
);
2886 e
->probability
= fallthru_edge
->probability
;
2887 test_bb
->count
= e
->count
= fallthru_edge
->count
;
2888 test_bb
->frequency
= EDGE_FREQUENCY (e
);
2890 // Now update the edges to the inst/uninist implementations.
2891 // For now assume that the paths are equally likely. When using HTM,
2892 // we'll try the uninst path first and fallback to inst path if htm
2893 // buffers are exceeded. Without HTM we start with the inst path and
2894 // use the uninst path when falling back to serial mode.
2895 redirect_edge_pred (inst_edge
, test_bb
);
2896 inst_edge
->flags
= EDGE_FALSE_VALUE
;
2897 inst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2899 = apply_probability (test_bb
->count
, inst_edge
->probability
);
2901 redirect_edge_pred (uninst_edge
, test_bb
);
2902 uninst_edge
->flags
= EDGE_TRUE_VALUE
;
2903 uninst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2905 = apply_probability (test_bb
->count
, uninst_edge
->probability
);
2908 // If we have no previous special cases, and we have PHIs at the beginning
2909 // of the atomic region, this means we have a loop at the beginning of the
2910 // atomic region that shares the first block. This can cause problems with
2911 // the transaction restart abnormal edges to be added in the tm_edges pass.
2912 // Solve this by adding a new empty block to receive the abnormal edges.
2913 if (region
->restart_block
== region
->entry_block
2914 && phi_nodes (region
->entry_block
))
2916 basic_block empty_bb
= create_empty_bb (transaction_bb
);
2917 region
->restart_block
= empty_bb
;
2918 add_bb_to_loop (empty_bb
, transaction_bb
->loop_father
);
2920 redirect_edge_pred (fallthru_edge
, empty_bb
);
2921 make_edge (transaction_bb
, empty_bb
, EDGE_FALLTHRU
);
2927 /* Generate the temporary to be used for the return value of
2928 BUILT_IN_TM_START. */
2931 generate_tm_state (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2933 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2935 create_tmp_reg (TREE_TYPE (TREE_TYPE (tm_start
)), "tm_state");
2937 // Reset the subcode, post optimizations. We'll fill this in
2938 // again as we process blocks.
2939 if (region
->exit_blocks
)
2941 gtransaction
*transaction_stmt
= region
->get_transaction_stmt ();
2942 unsigned int subcode
= gimple_transaction_subcode (transaction_stmt
);
2944 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2945 subcode
&= (GTMA_DECLARATION_MASK
| GTMA_DOES_GO_IRREVOCABLE
2946 | GTMA_MAY_ENTER_IRREVOCABLE
2947 | GTMA_HAS_NO_INSTRUMENTATION
);
2949 subcode
&= GTMA_DECLARATION_MASK
;
2950 gimple_transaction_set_subcode (transaction_stmt
, subcode
);
2956 // Propagate flags from inner transactions outwards.
2958 propagate_tm_flags_out (struct tm_region
*region
)
2962 propagate_tm_flags_out (region
->inner
);
2964 if (region
->outer
&& region
->outer
->transaction_stmt
)
2967 = gimple_transaction_subcode (region
->get_transaction_stmt ());
2968 s
&= (GTMA_HAVE_ABORT
| GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
2969 | GTMA_MAY_ENTER_IRREVOCABLE
);
2970 s
|= gimple_transaction_subcode (region
->outer
->get_transaction_stmt ());
2971 gimple_transaction_set_subcode (region
->outer
->get_transaction_stmt (),
2975 propagate_tm_flags_out (region
->next
);
2978 /* Entry point to the MARK phase of TM expansion. Here we replace
2979 transactional memory statements with calls to builtins, and function
2980 calls with their transactional clones (if available). But we don't
2981 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */
2984 execute_tm_mark (void)
2986 pending_edge_inserts_p
= false;
2988 expand_regions (all_tm_regions
, generate_tm_state
, NULL
,
2989 /*traverse_clones=*/true);
2993 vec
<tm_region_p
> bb_regions
2994 = get_bb_regions_instrumented (/*traverse_clones=*/true,
2995 /*include_uninstrumented_p=*/false);
2996 struct tm_region
*r
;
2999 // Expand memory operations into calls into the runtime.
3000 // This collects log entries as well.
3001 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3005 if (r
->transaction_stmt
)
3008 = gimple_transaction_subcode (r
->get_transaction_stmt ());
3010 /* If we're sure to go irrevocable, there won't be
3011 anything to expand, since the run-time will go
3012 irrevocable right away. */
3013 if (sub
& GTMA_DOES_GO_IRREVOCABLE
3014 && sub
& GTMA_MAY_ENTER_IRREVOCABLE
)
3017 expand_block_tm (r
, BASIC_BLOCK_FOR_FN (cfun
, i
));
3021 bb_regions
.release ();
3023 // Propagate flags from inner transactions outwards.
3024 propagate_tm_flags_out (all_tm_regions
);
3026 // Expand GIMPLE_TRANSACTIONs into calls into the runtime.
3027 expand_regions (all_tm_regions
, expand_transaction
, NULL
,
3028 /*traverse_clones=*/false);
3033 if (pending_edge_inserts_p
)
3034 gsi_commit_edge_inserts ();
3035 free_dominance_info (CDI_DOMINATORS
);
3041 const pass_data pass_data_tm_mark
=
3043 GIMPLE_PASS
, /* type */
3044 "tmmark", /* name */
3045 OPTGROUP_NONE
, /* optinfo_flags */
3046 TV_TRANS_MEM
, /* tv_id */
3047 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3048 0, /* properties_provided */
3049 0, /* properties_destroyed */
3050 0, /* todo_flags_start */
3051 TODO_update_ssa
, /* todo_flags_finish */
3054 class pass_tm_mark
: public gimple_opt_pass
3057 pass_tm_mark (gcc::context
*ctxt
)
3058 : gimple_opt_pass (pass_data_tm_mark
, ctxt
)
3061 /* opt_pass methods: */
3062 virtual unsigned int execute (function
*) { return execute_tm_mark (); }
3064 }; // class pass_tm_mark
3069 make_pass_tm_mark (gcc::context
*ctxt
)
3071 return new pass_tm_mark (ctxt
);
3075 /* Create an abnormal edge from STMT at iter, splitting the block
3076 as necessary. Adjust *PNEXT as needed for the split block. */
3079 split_bb_make_tm_edge (gimple stmt
, basic_block dest_bb
,
3080 gimple_stmt_iterator iter
, gimple_stmt_iterator
*pnext
)
3082 basic_block bb
= gimple_bb (stmt
);
3083 if (!gsi_one_before_end_p (iter
))
3085 edge e
= split_block (bb
, stmt
);
3086 *pnext
= gsi_start_bb (e
->dest
);
3088 make_edge (bb
, dest_bb
, EDGE_ABNORMAL
);
3090 // Record the need for the edge for the benefit of the rtl passes.
3091 if (cfun
->gimple_df
->tm_restart
== NULL
)
3092 cfun
->gimple_df
->tm_restart
= htab_create_ggc (31, struct_ptr_hash
,
3093 struct_ptr_eq
, ggc_free
);
3095 struct tm_restart_node dummy
;
3097 dummy
.label_or_list
= gimple_block_label (dest_bb
);
3099 void **slot
= htab_find_slot (cfun
->gimple_df
->tm_restart
, &dummy
, INSERT
);
3100 struct tm_restart_node
*n
= (struct tm_restart_node
*) *slot
;
3103 n
= ggc_alloc
<tm_restart_node
> ();
3108 tree old
= n
->label_or_list
;
3109 if (TREE_CODE (old
) == LABEL_DECL
)
3110 old
= tree_cons (NULL
, old
, NULL
);
3111 n
->label_or_list
= tree_cons (NULL
, dummy
.label_or_list
, old
);
3115 /* Split block BB as necessary for every builtin function we added, and
3116 wire up the abnormal back edges implied by the transaction restart. */
3119 expand_block_edges (struct tm_region
*const region
, basic_block bb
)
3121 gimple_stmt_iterator gsi
, next_gsi
;
3123 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi
= next_gsi
)
3125 gimple stmt
= gsi_stmt (gsi
);
3129 gsi_next (&next_gsi
);
3131 // ??? Shouldn't we split for any non-pure, non-irrevocable function?
3132 call_stmt
= dyn_cast
<gcall
*> (stmt
);
3134 || (gimple_call_flags (call_stmt
) & ECF_TM_BUILTIN
) == 0)
3137 if (DECL_FUNCTION_CODE (gimple_call_fndecl (call_stmt
))
3138 == BUILT_IN_TM_ABORT
)
3140 // If we have a ``_transaction_cancel [[outer]]'', there is only
3141 // one abnormal edge: to the transaction marked OUTER.
3142 // All compiler-generated instances of BUILT_IN_TM_ABORT have a
3143 // constant argument, which we can examine here. Users invoking
3144 // TM_ABORT directly get what they deserve.
3145 tree arg
= gimple_call_arg (call_stmt
, 0);
3146 if (TREE_CODE (arg
) == INTEGER_CST
3147 && (TREE_INT_CST_LOW (arg
) & AR_OUTERABORT
) != 0
3148 && !decl_is_tm_clone (current_function_decl
))
3150 // Find the GTMA_IS_OUTER transaction.
3151 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3152 if (o
->original_transaction_was_outer
)
3154 split_bb_make_tm_edge (call_stmt
, o
->restart_block
,
3159 // Otherwise, the front-end should have semantically checked
3160 // outer aborts, but in either case the target region is not
3161 // within this function.
3165 // Non-outer, TM aborts have an abnormal edge to the inner-most
3166 // transaction, the one being aborted;
3167 split_bb_make_tm_edge (call_stmt
, region
->restart_block
, gsi
,
3171 // All TM builtins have an abnormal edge to the outer-most transaction.
3172 // We never restart inner transactions. For tm clones, we know a-priori
3173 // that the outer-most transaction is outside the function.
3174 if (decl_is_tm_clone (current_function_decl
))
3177 if (cfun
->gimple_df
->tm_restart
== NULL
)
3178 cfun
->gimple_df
->tm_restart
3179 = htab_create_ggc (31, struct_ptr_hash
, struct_ptr_eq
, ggc_free
);
3181 // All TM builtins have an abnormal edge to the outer-most transaction.
3182 // We never restart inner transactions.
3183 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3186 split_bb_make_tm_edge (call_stmt
, o
->restart_block
, gsi
, &next_gsi
);
3190 // Delete any tail-call annotation that may have been added.
3191 // The tail-call pass may have mis-identified the commit as being
3192 // a candidate because we had not yet added this restart edge.
3193 gimple_call_set_tail (call_stmt
, false);
3197 /* Entry point to the final expansion of transactional nodes. */
3201 const pass_data pass_data_tm_edges
=
3203 GIMPLE_PASS
, /* type */
3204 "tmedge", /* name */
3205 OPTGROUP_NONE
, /* optinfo_flags */
3206 TV_TRANS_MEM
, /* tv_id */
3207 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3208 0, /* properties_provided */
3209 0, /* properties_destroyed */
3210 0, /* todo_flags_start */
3211 TODO_update_ssa
, /* todo_flags_finish */
3214 class pass_tm_edges
: public gimple_opt_pass
3217 pass_tm_edges (gcc::context
*ctxt
)
3218 : gimple_opt_pass (pass_data_tm_edges
, ctxt
)
3221 /* opt_pass methods: */
3222 virtual unsigned int execute (function
*);
3224 }; // class pass_tm_edges
3227 pass_tm_edges::execute (function
*fun
)
3229 vec
<tm_region_p
> bb_regions
3230 = get_bb_regions_instrumented (/*traverse_clones=*/false,
3231 /*include_uninstrumented_p=*/true);
3232 struct tm_region
*r
;
3235 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3237 expand_block_edges (r
, BASIC_BLOCK_FOR_FN (fun
, i
));
3239 bb_regions
.release ();
3241 /* We've got to release the dominance info now, to indicate that it
3242 must be rebuilt completely. Otherwise we'll crash trying to update
3243 the SSA web in the TODO section following this pass. */
3244 free_dominance_info (CDI_DOMINATORS
);
3245 bitmap_obstack_release (&tm_obstack
);
3246 all_tm_regions
= NULL
;
3254 make_pass_tm_edges (gcc::context
*ctxt
)
3256 return new pass_tm_edges (ctxt
);
3259 /* Helper function for expand_regions. Expand REGION and recurse to
3260 the inner region. Call CALLBACK on each region. CALLBACK returns
3261 NULL to continue the traversal, otherwise a non-null value which
3262 this function will return as well. TRAVERSE_CLONES is true if we
3263 should traverse transactional clones. */
3266 expand_regions_1 (struct tm_region
*region
,
3267 void *(*callback
)(struct tm_region
*, void *),
3269 bool traverse_clones
)
3271 void *retval
= NULL
;
3272 if (region
->exit_blocks
3273 || (traverse_clones
&& decl_is_tm_clone (current_function_decl
)))
3275 retval
= callback (region
, data
);
3281 retval
= expand_regions (region
->inner
, callback
, data
, traverse_clones
);
3288 /* Traverse the regions enclosed and including REGION. Execute
3289 CALLBACK for each region, passing DATA. CALLBACK returns NULL to
3290 continue the traversal, otherwise a non-null value which this
3291 function will return as well. TRAVERSE_CLONES is true if we should
3292 traverse transactional clones. */
3295 expand_regions (struct tm_region
*region
,
3296 void *(*callback
)(struct tm_region
*, void *),
3298 bool traverse_clones
)
3300 void *retval
= NULL
;
3303 retval
= expand_regions_1 (region
, callback
, data
, traverse_clones
);
3306 region
= region
->next
;
3312 /* A unique TM memory operation. */
3313 typedef struct tm_memop
3315 /* Unique ID that all memory operations to the same location have. */
3316 unsigned int value_id
;
3317 /* Address of load/store. */
3321 /* TM memory operation hashtable helpers. */
3323 struct tm_memop_hasher
: typed_free_remove
<tm_memop
>
3325 typedef tm_memop value_type
;
3326 typedef tm_memop compare_type
;
3327 static inline hashval_t
hash (const value_type
*);
3328 static inline bool equal (const value_type
*, const compare_type
*);
3331 /* Htab support. Return a hash value for a `tm_memop'. */
3333 tm_memop_hasher::hash (const value_type
*mem
)
3335 tree addr
= mem
->addr
;
3336 /* We drill down to the SSA_NAME/DECL for the hash, but equality is
3337 actually done with operand_equal_p (see tm_memop_eq). */
3338 if (TREE_CODE (addr
) == ADDR_EXPR
)
3339 addr
= TREE_OPERAND (addr
, 0);
3340 return iterative_hash_expr (addr
, 0);
3343 /* Htab support. Return true if two tm_memop's are the same. */
3345 tm_memop_hasher::equal (const value_type
*mem1
, const compare_type
*mem2
)
3347 return operand_equal_p (mem1
->addr
, mem2
->addr
, 0);
3350 /* Sets for solving data flow equations in the memory optimization pass. */
3351 struct tm_memopt_bitmaps
3353 /* Stores available to this BB upon entry. Basically, stores that
3354 dominate this BB. */
3355 bitmap store_avail_in
;
3356 /* Stores available at the end of this BB. */
3357 bitmap store_avail_out
;
3358 bitmap store_antic_in
;
3359 bitmap store_antic_out
;
3360 /* Reads available to this BB upon entry. Basically, reads that
3361 dominate this BB. */
3362 bitmap read_avail_in
;
3363 /* Reads available at the end of this BB. */
3364 bitmap read_avail_out
;
3365 /* Reads performed in this BB. */
3367 /* Writes performed in this BB. */
3370 /* Temporary storage for pass. */
3371 /* Is the current BB in the worklist? */
3372 bool avail_in_worklist_p
;
3373 /* Have we visited this BB? */
3377 static bitmap_obstack tm_memopt_obstack
;
3379 /* Unique counter for TM loads and stores. Loads and stores of the
3380 same address get the same ID. */
3381 static unsigned int tm_memopt_value_id
;
3382 static hash_table
<tm_memop_hasher
> *tm_memopt_value_numbers
;
3384 #define STORE_AVAIL_IN(BB) \
3385 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in
3386 #define STORE_AVAIL_OUT(BB) \
3387 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out
3388 #define STORE_ANTIC_IN(BB) \
3389 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in
3390 #define STORE_ANTIC_OUT(BB) \
3391 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out
3392 #define READ_AVAIL_IN(BB) \
3393 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in
3394 #define READ_AVAIL_OUT(BB) \
3395 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out
3396 #define READ_LOCAL(BB) \
3397 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local
3398 #define STORE_LOCAL(BB) \
3399 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local
3400 #define AVAIL_IN_WORKLIST_P(BB) \
3401 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p
3402 #define BB_VISITED_P(BB) \
3403 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p
3405 /* Given a TM load/store in STMT, return the value number for the address
3409 tm_memopt_value_number (gimple stmt
, enum insert_option op
)
3411 struct tm_memop tmpmem
, *mem
;
3414 gcc_assert (is_tm_load (stmt
) || is_tm_store (stmt
));
3415 tmpmem
.addr
= gimple_call_arg (stmt
, 0);
3416 slot
= tm_memopt_value_numbers
->find_slot (&tmpmem
, op
);
3419 else if (op
== INSERT
)
3421 mem
= XNEW (struct tm_memop
);
3423 mem
->value_id
= tm_memopt_value_id
++;
3424 mem
->addr
= tmpmem
.addr
;
3428 return mem
->value_id
;
3431 /* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */
3434 tm_memopt_accumulate_memops (basic_block bb
)
3436 gimple_stmt_iterator gsi
;
3438 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3440 gimple stmt
= gsi_stmt (gsi
);
3444 if (is_tm_store (stmt
))
3445 bits
= STORE_LOCAL (bb
);
3446 else if (is_tm_load (stmt
))
3447 bits
= READ_LOCAL (bb
);
3451 loc
= tm_memopt_value_number (stmt
, INSERT
);
3452 bitmap_set_bit (bits
, loc
);
3455 fprintf (dump_file
, "TM memopt (%s): value num=%d, BB=%d, addr=",
3456 is_tm_load (stmt
) ? "LOAD" : "STORE", loc
,
3457 gimple_bb (stmt
)->index
);
3458 print_generic_expr (dump_file
, gimple_call_arg (stmt
, 0), 0);
3459 fprintf (dump_file
, "\n");
3464 /* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */
3467 dump_tm_memopt_set (const char *set_name
, bitmap bits
)
3471 const char *comma
= "";
3473 fprintf (dump_file
, "TM memopt: %s: [", set_name
);
3474 EXECUTE_IF_SET_IN_BITMAP (bits
, 0, i
, bi
)
3476 hash_table
<tm_memop_hasher
>::iterator hi
;
3477 struct tm_memop
*mem
= NULL
;
3479 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */
3480 FOR_EACH_HASH_TABLE_ELEMENT (*tm_memopt_value_numbers
, mem
, tm_memop_t
, hi
)
3481 if (mem
->value_id
== i
)
3483 gcc_assert (mem
->value_id
== i
);
3484 fprintf (dump_file
, "%s", comma
);
3486 print_generic_expr (dump_file
, mem
->addr
, 0);
3488 fprintf (dump_file
, "]\n");
3491 /* Prettily dump all of the memopt sets in BLOCKS. */
3494 dump_tm_memopt_sets (vec
<basic_block
> blocks
)
3499 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3501 fprintf (dump_file
, "------------BB %d---------\n", bb
->index
);
3502 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb
));
3503 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb
));
3504 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb
));
3505 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb
));
3506 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb
));
3507 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb
));
3511 /* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */
3514 tm_memopt_compute_avin (basic_block bb
)
3519 /* Seed with the AVOUT of any predecessor. */
3520 for (ix
= 0; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3522 e
= EDGE_PRED (bb
, ix
);
3523 /* Make sure we have already visited this BB, and is thus
3526 If e->src->aux is NULL, this predecessor is actually on an
3527 enclosing transaction. We only care about the current
3528 transaction, so ignore it. */
3529 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3531 bitmap_copy (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3532 bitmap_copy (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3537 for (; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3539 e
= EDGE_PRED (bb
, ix
);
3540 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3542 bitmap_and_into (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3543 bitmap_and_into (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3547 BB_VISITED_P (bb
) = true;
3550 /* Compute the STORE_ANTIC_IN for the basic block BB. */
3553 tm_memopt_compute_antin (basic_block bb
)
3558 /* Seed with the ANTIC_OUT of any successor. */
3559 for (ix
= 0; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3561 e
= EDGE_SUCC (bb
, ix
);
3562 /* Make sure we have already visited this BB, and is thus
3564 if (BB_VISITED_P (e
->dest
))
3566 bitmap_copy (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3571 for (; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3573 e
= EDGE_SUCC (bb
, ix
);
3574 if (BB_VISITED_P (e
->dest
))
3575 bitmap_and_into (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3578 BB_VISITED_P (bb
) = true;
3581 /* Compute the AVAIL sets for every basic block in BLOCKS.
3583 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows:
3585 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb])
3586 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors])
3588 This is basically what we do in lcm's compute_available(), but here
3589 we calculate two sets of sets (one for STOREs and one for READs),
3590 and we work on a region instead of the entire CFG.
3592 REGION is the TM region.
3593 BLOCKS are the basic blocks in the region. */
3596 tm_memopt_compute_available (struct tm_region
*region
,
3597 vec
<basic_block
> blocks
)
3600 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3601 unsigned int qlen
, i
;
3605 /* Allocate a worklist array/queue. Entries are only added to the
3606 list if they were not already on the list. So the size is
3607 bounded by the number of basic blocks in the region. */
3608 qlen
= blocks
.length () - 1;
3609 qin
= qout
= worklist
=
3610 XNEWVEC (basic_block
, qlen
);
3612 /* Put every block in the region on the worklist. */
3613 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3615 /* Seed AVAIL_OUT with the LOCAL set. */
3616 bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_LOCAL (bb
));
3617 bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_LOCAL (bb
));
3619 AVAIL_IN_WORKLIST_P (bb
) = true;
3620 /* No need to insert the entry block, since it has an AVIN of
3621 null, and an AVOUT that has already been seeded in. */
3622 if (bb
!= region
->entry_block
)
3626 /* The entry block has been initialized with the local sets. */
3627 BB_VISITED_P (region
->entry_block
) = true;
3630 qend
= &worklist
[qlen
];
3632 /* Iterate until the worklist is empty. */
3635 /* Take the first entry off the worklist. */
3642 /* This block can be added to the worklist again if necessary. */
3643 AVAIL_IN_WORKLIST_P (bb
) = false;
3644 tm_memopt_compute_avin (bb
);
3646 /* Note: We do not add the LOCAL sets here because we already
3647 seeded the AVAIL_OUT sets with them. */
3648 changed
= bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_AVAIL_IN (bb
));
3649 changed
|= bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_AVAIL_IN (bb
));
3651 && (region
->exit_blocks
== NULL
3652 || !bitmap_bit_p (region
->exit_blocks
, bb
->index
)))
3653 /* If the out state of this block changed, then we need to add
3654 its successors to the worklist if they are not already in. */
3655 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3656 if (!AVAIL_IN_WORKLIST_P (e
->dest
)
3657 && e
->dest
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
3660 AVAIL_IN_WORKLIST_P (e
->dest
) = true;
3671 dump_tm_memopt_sets (blocks
);
3674 /* Compute ANTIC sets for every basic block in BLOCKS.
3676 We compute STORE_ANTIC_OUT as follows:
3678 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb])
3679 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors])
3681 REGION is the TM region.
3682 BLOCKS are the basic blocks in the region. */
3685 tm_memopt_compute_antic (struct tm_region
*region
,
3686 vec
<basic_block
> blocks
)
3689 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3694 /* Allocate a worklist array/queue. Entries are only added to the
3695 list if they were not already on the list. So the size is
3696 bounded by the number of basic blocks in the region. */
3697 qin
= qout
= worklist
= XNEWVEC (basic_block
, blocks
.length ());
3699 for (qlen
= 0, i
= blocks
.length () - 1; i
>= 0; --i
)
3703 /* Seed ANTIC_OUT with the LOCAL set. */
3704 bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_LOCAL (bb
));
3706 /* Put every block in the region on the worklist. */
3707 AVAIL_IN_WORKLIST_P (bb
) = true;
3708 /* No need to insert exit blocks, since their ANTIC_IN is NULL,
3709 and their ANTIC_OUT has already been seeded in. */
3710 if (region
->exit_blocks
3711 && !bitmap_bit_p (region
->exit_blocks
, bb
->index
))
3718 /* The exit blocks have been initialized with the local sets. */
3719 if (region
->exit_blocks
)
3723 EXECUTE_IF_SET_IN_BITMAP (region
->exit_blocks
, 0, i
, bi
)
3724 BB_VISITED_P (BASIC_BLOCK_FOR_FN (cfun
, i
)) = true;
3728 qend
= &worklist
[qlen
];
3730 /* Iterate until the worklist is empty. */
3733 /* Take the first entry off the worklist. */
3740 /* This block can be added to the worklist again if necessary. */
3741 AVAIL_IN_WORKLIST_P (bb
) = false;
3742 tm_memopt_compute_antin (bb
);
3744 /* Note: We do not add the LOCAL sets here because we already
3745 seeded the ANTIC_OUT sets with them. */
3746 if (bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_ANTIC_IN (bb
))
3747 && bb
!= region
->entry_block
)
3748 /* If the out state of this block changed, then we need to add
3749 its predecessors to the worklist if they are not already in. */
3750 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
3751 if (!AVAIL_IN_WORKLIST_P (e
->src
))
3754 AVAIL_IN_WORKLIST_P (e
->src
) = true;
3765 dump_tm_memopt_sets (blocks
);
3768 /* Offsets of load variants from TM_LOAD. For example,
3769 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*.
3770 See gtm-builtins.def. */
3771 #define TRANSFORM_RAR 1
3772 #define TRANSFORM_RAW 2
3773 #define TRANSFORM_RFW 3
3774 /* Offsets of store variants from TM_STORE. */
3775 #define TRANSFORM_WAR 1
3776 #define TRANSFORM_WAW 2
3778 /* Inform about a load/store optimization. */
3781 dump_tm_memopt_transform (gimple stmt
)
3785 fprintf (dump_file
, "TM memopt: transforming: ");
3786 print_gimple_stmt (dump_file
, stmt
, 0, 0);
3787 fprintf (dump_file
, "\n");
3791 /* Perform a read/write optimization. Replaces the TM builtin in STMT
3792 by a builtin that is OFFSET entries down in the builtins table in
3793 gtm-builtins.def. */
3796 tm_memopt_transform_stmt (unsigned int offset
,
3798 gimple_stmt_iterator
*gsi
)
3800 tree fn
= gimple_call_fn (stmt
);
3801 gcc_assert (TREE_CODE (fn
) == ADDR_EXPR
);
3802 TREE_OPERAND (fn
, 0)
3803 = builtin_decl_explicit ((enum built_in_function
)
3804 (DECL_FUNCTION_CODE (TREE_OPERAND (fn
, 0))
3806 gimple_call_set_fn (stmt
, fn
);
3807 gsi_replace (gsi
, stmt
, true);
3808 dump_tm_memopt_transform (stmt
);
3811 /* Perform the actual TM memory optimization transformations in the
3812 basic blocks in BLOCKS. */
3815 tm_memopt_transform_blocks (vec
<basic_block
> blocks
)
3819 gimple_stmt_iterator gsi
;
3821 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3823 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3825 gimple stmt
= gsi_stmt (gsi
);
3826 bitmap read_avail
= READ_AVAIL_IN (bb
);
3827 bitmap store_avail
= STORE_AVAIL_IN (bb
);
3828 bitmap store_antic
= STORE_ANTIC_OUT (bb
);
3831 if (is_tm_simple_load (stmt
))
3833 gcall
*call_stmt
= as_a
<gcall
*> (stmt
);
3834 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3835 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3836 tm_memopt_transform_stmt (TRANSFORM_RAW
, call_stmt
, &gsi
);
3837 else if (store_antic
&& bitmap_bit_p (store_antic
, loc
))
3839 tm_memopt_transform_stmt (TRANSFORM_RFW
, call_stmt
, &gsi
);
3840 bitmap_set_bit (store_avail
, loc
);
3842 else if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3843 tm_memopt_transform_stmt (TRANSFORM_RAR
, call_stmt
, &gsi
);
3845 bitmap_set_bit (read_avail
, loc
);
3847 else if (is_tm_simple_store (stmt
))
3849 gcall
*call_stmt
= as_a
<gcall
*> (stmt
);
3850 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3851 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3852 tm_memopt_transform_stmt (TRANSFORM_WAW
, call_stmt
, &gsi
);
3855 if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3856 tm_memopt_transform_stmt (TRANSFORM_WAR
, call_stmt
, &gsi
);
3857 bitmap_set_bit (store_avail
, loc
);
3864 /* Return a new set of bitmaps for a BB. */
3866 static struct tm_memopt_bitmaps
*
3867 tm_memopt_init_sets (void)
3869 struct tm_memopt_bitmaps
*b
3870 = XOBNEW (&tm_memopt_obstack
.obstack
, struct tm_memopt_bitmaps
);
3871 b
->store_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3872 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3873 b
->store_antic_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3874 b
->store_antic_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3875 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3876 b
->read_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3877 b
->read_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3878 b
->read_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3879 b
->store_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3883 /* Free sets computed for each BB. */
3886 tm_memopt_free_sets (vec
<basic_block
> blocks
)
3891 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3895 /* Clear the visited bit for every basic block in BLOCKS. */
3898 tm_memopt_clear_visited (vec
<basic_block
> blocks
)
3903 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3904 BB_VISITED_P (bb
) = false;
3907 /* Replace TM load/stores with hints for the runtime. We handle
3908 things like read-after-write, write-after-read, read-after-read,
3909 read-for-write, etc. */
3912 execute_tm_memopt (void)
3914 struct tm_region
*region
;
3915 vec
<basic_block
> bbs
;
3917 tm_memopt_value_id
= 0;
3918 tm_memopt_value_numbers
= new hash_table
<tm_memop_hasher
> (10);
3920 for (region
= all_tm_regions
; region
; region
= region
->next
)
3922 /* All the TM stores/loads in the current region. */
3926 bitmap_obstack_initialize (&tm_memopt_obstack
);
3928 /* Save all BBs for the current region. */
3929 bbs
= get_tm_region_blocks (region
->entry_block
,
3930 region
->exit_blocks
,
3935 /* Collect all the memory operations. */
3936 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
3938 bb
->aux
= tm_memopt_init_sets ();
3939 tm_memopt_accumulate_memops (bb
);
3942 /* Solve data flow equations and transform each block accordingly. */
3943 tm_memopt_clear_visited (bbs
);
3944 tm_memopt_compute_available (region
, bbs
);
3945 tm_memopt_clear_visited (bbs
);
3946 tm_memopt_compute_antic (region
, bbs
);
3947 tm_memopt_transform_blocks (bbs
);
3949 tm_memopt_free_sets (bbs
);
3951 bitmap_obstack_release (&tm_memopt_obstack
);
3952 tm_memopt_value_numbers
->empty ();
3955 delete tm_memopt_value_numbers
;
3956 tm_memopt_value_numbers
= NULL
;
3962 const pass_data pass_data_tm_memopt
=
3964 GIMPLE_PASS
, /* type */
3965 "tmmemopt", /* name */
3966 OPTGROUP_NONE
, /* optinfo_flags */
3967 TV_TRANS_MEM
, /* tv_id */
3968 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3969 0, /* properties_provided */
3970 0, /* properties_destroyed */
3971 0, /* todo_flags_start */
3972 0, /* todo_flags_finish */
3975 class pass_tm_memopt
: public gimple_opt_pass
3978 pass_tm_memopt (gcc::context
*ctxt
)
3979 : gimple_opt_pass (pass_data_tm_memopt
, ctxt
)
3982 /* opt_pass methods: */
3983 virtual bool gate (function
*) { return flag_tm
&& optimize
> 0; }
3984 virtual unsigned int execute (function
*) { return execute_tm_memopt (); }
3986 }; // class pass_tm_memopt
3991 make_pass_tm_memopt (gcc::context
*ctxt
)
3993 return new pass_tm_memopt (ctxt
);
3997 /* Interprocedual analysis for the creation of transactional clones.
3998 The aim of this pass is to find which functions are referenced in
3999 a non-irrevocable transaction context, and for those over which
4000 we have control (or user directive), create a version of the
4001 function which uses only the transactional interface to reference
4002 protected memories. This analysis proceeds in several steps:
4004 (1) Collect the set of all possible transactional clones:
4006 (a) For all local public functions marked tm_callable, push
4007 it onto the tm_callee queue.
4009 (b) For all local functions, scan for calls in transaction blocks.
4010 Push the caller and callee onto the tm_caller and tm_callee
4011 queues. Count the number of callers for each callee.
4013 (c) For each local function on the callee list, assume we will
4014 create a transactional clone. Push *all* calls onto the
4015 callee queues; count the number of clone callers separately
4016 to the number of original callers.
4018 (2) Propagate irrevocable status up the dominator tree:
4020 (a) Any external function on the callee list that is not marked
4021 tm_callable is irrevocable. Push all callers of such onto
4024 (b) For each function on the worklist, mark each block that
4025 contains an irrevocable call. Use the AND operator to
4026 propagate that mark up the dominator tree.
4028 (c) If we reach the entry block for a possible transactional
4029 clone, then the transactional clone is irrevocable, and
4030 we should not create the clone after all. Push all
4031 callers onto the worklist.
4033 (d) Place tm_irrevocable calls at the beginning of the relevant
4034 blocks. Special case here is the entry block for the entire
4035 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for
4036 the library to begin the region in serial mode. Decrement
4037 the call count for all callees in the irrevocable region.
4039 (3) Create the transactional clones:
4041 Any tm_callee that still has a non-zero call count is cloned.
4044 /* This structure is stored in the AUX field of each cgraph_node. */
4045 struct tm_ipa_cg_data
4047 /* The clone of the function that got created. */
4048 struct cgraph_node
*clone
;
4050 /* The tm regions in the normal function. */
4051 struct tm_region
*all_tm_regions
;
4053 /* The blocks of the normal/clone functions that contain irrevocable
4054 calls, or blocks that are post-dominated by irrevocable calls. */
4055 bitmap irrevocable_blocks_normal
;
4056 bitmap irrevocable_blocks_clone
;
4058 /* The blocks of the normal function that are involved in transactions. */
4059 bitmap transaction_blocks_normal
;
4061 /* The number of callers to the transactional clone of this function
4062 from normal and transactional clones respectively. */
4063 unsigned tm_callers_normal
;
4064 unsigned tm_callers_clone
;
4066 /* True if all calls to this function's transactional clone
4067 are irrevocable. Also automatically true if the function
4068 has no transactional clone. */
4069 bool is_irrevocable
;
4071 /* Flags indicating the presence of this function in various queues. */
4072 bool in_callee_queue
;
4075 /* Flags indicating the kind of scan desired while in the worklist. */
4076 bool want_irr_scan_normal
;
4079 typedef vec
<cgraph_node
*> cgraph_node_queue
;
4081 /* Return the ipa data associated with NODE, allocating zeroed memory
4082 if necessary. TRAVERSE_ALIASES is true if we must traverse aliases
4083 and set *NODE accordingly. */
4085 static struct tm_ipa_cg_data
*
4086 get_cg_data (struct cgraph_node
**node
, bool traverse_aliases
)
4088 struct tm_ipa_cg_data
*d
;
4090 if (traverse_aliases
&& (*node
)->alias
)
4091 *node
= (*node
)->get_alias_target ();
4093 d
= (struct tm_ipa_cg_data
*) (*node
)->aux
;
4097 d
= (struct tm_ipa_cg_data
*)
4098 obstack_alloc (&tm_obstack
.obstack
, sizeof (*d
));
4099 (*node
)->aux
= (void *) d
;
4100 memset (d
, 0, sizeof (*d
));
4106 /* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that
4107 it is already present. */
4110 maybe_push_queue (struct cgraph_node
*node
,
4111 cgraph_node_queue
*queue_p
, bool *in_queue_p
)
4116 queue_p
->safe_push (node
);
4120 /* Duplicate the basic blocks in QUEUE for use in the uninstrumented
4121 code path. QUEUE are the basic blocks inside the transaction
4122 represented in REGION.
4124 Later in split_code_paths() we will add the conditional to choose
4125 between the two alternatives. */
4128 ipa_uninstrument_transaction (struct tm_region
*region
,
4129 vec
<basic_block
> queue
)
4131 gimple transaction
= region
->transaction_stmt
;
4132 basic_block transaction_bb
= gimple_bb (transaction
);
4133 int n
= queue
.length ();
4134 basic_block
*new_bbs
= XNEWVEC (basic_block
, n
);
4136 copy_bbs (queue
.address (), n
, new_bbs
, NULL
, 0, NULL
, NULL
, transaction_bb
,
4138 edge e
= make_edge (transaction_bb
, new_bbs
[0], EDGE_TM_UNINSTRUMENTED
);
4139 add_phi_args_after_copy (new_bbs
, n
, e
);
4141 // Now we will have a GIMPLE_ATOMIC with 3 possible edges out of it.
4142 // a) EDGE_FALLTHRU into the transaction
4143 // b) EDGE_TM_ABORT out of the transaction
4144 // c) EDGE_TM_UNINSTRUMENTED into the uninstrumented blocks.
4149 /* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone.
4150 Queue all callees within block BB. */
4153 ipa_tm_scan_calls_block (cgraph_node_queue
*callees_p
,
4154 basic_block bb
, bool for_clone
)
4156 gimple_stmt_iterator gsi
;
4158 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4160 gimple stmt
= gsi_stmt (gsi
);
4161 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4163 tree fndecl
= gimple_call_fndecl (stmt
);
4166 struct tm_ipa_cg_data
*d
;
4168 struct cgraph_node
*node
;
4170 if (is_tm_ending_fndecl (fndecl
))
4172 if (find_tm_replacement_function (fndecl
))
4175 node
= cgraph_node::get (fndecl
);
4176 gcc_assert (node
!= NULL
);
4177 d
= get_cg_data (&node
, true);
4179 pcallers
= (for_clone
? &d
->tm_callers_clone
4180 : &d
->tm_callers_normal
);
4183 maybe_push_queue (node
, callees_p
, &d
->in_callee_queue
);
4189 /* Scan all calls in NODE that are within a transaction region,
4190 and push the resulting nodes into the callee queue. */
4193 ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data
*d
,
4194 cgraph_node_queue
*callees_p
)
4196 struct tm_region
*r
;
4198 d
->transaction_blocks_normal
= BITMAP_ALLOC (&tm_obstack
);
4199 d
->all_tm_regions
= all_tm_regions
;
4201 for (r
= all_tm_regions
; r
; r
= r
->next
)
4203 vec
<basic_block
> bbs
;
4207 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
, NULL
,
4208 d
->transaction_blocks_normal
, false);
4210 // Generate the uninstrumented code path for this transaction.
4211 ipa_uninstrument_transaction (r
, bbs
);
4213 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
4214 ipa_tm_scan_calls_block (callees_p
, bb
, false);
4219 // ??? copy_bbs should maintain cgraph edges for the blocks as it is
4220 // copying them, rather than forcing us to do this externally.
4221 cgraph_edge::rebuild_edges ();
4223 // ??? In ipa_uninstrument_transaction we don't try to update dominators
4224 // because copy_bbs doesn't return a VEC like iterate_fix_dominators expects.
4225 // Instead, just release dominators here so update_ssa recomputes them.
4226 free_dominance_info (CDI_DOMINATORS
);
4228 // When building the uninstrumented code path, copy_bbs will have invoked
4229 // create_new_def_for starting an "ssa update context". There is only one
4230 // instance of this context, so resolve ssa updates before moving on to
4231 // the next function.
4232 update_ssa (TODO_update_ssa
);
4235 /* Scan all calls in NODE as if this is the transactional clone,
4236 and push the destinations into the callee queue. */
4239 ipa_tm_scan_calls_clone (struct cgraph_node
*node
,
4240 cgraph_node_queue
*callees_p
)
4242 struct function
*fn
= DECL_STRUCT_FUNCTION (node
->decl
);
4245 FOR_EACH_BB_FN (bb
, fn
)
4246 ipa_tm_scan_calls_block (callees_p
, bb
, true);
4249 /* The function NODE has been detected to be irrevocable. Push all
4250 of its callers onto WORKLIST for the purpose of re-scanning them. */
4253 ipa_tm_note_irrevocable (struct cgraph_node
*node
,
4254 cgraph_node_queue
*worklist_p
)
4256 struct tm_ipa_cg_data
*d
= get_cg_data (&node
, true);
4257 struct cgraph_edge
*e
;
4259 d
->is_irrevocable
= true;
4261 for (e
= node
->callers
; e
; e
= e
->next_caller
)
4264 struct cgraph_node
*caller
;
4266 /* Don't examine recursive calls. */
4267 if (e
->caller
== node
)
4269 /* Even if we think we can go irrevocable, believe the user
4271 if (is_tm_safe_or_pure (e
->caller
->decl
))
4275 d
= get_cg_data (&caller
, true);
4277 /* Check if the callee is in a transactional region. If so,
4278 schedule the function for normal re-scan as well. */
4279 bb
= gimple_bb (e
->call_stmt
);
4280 gcc_assert (bb
!= NULL
);
4281 if (d
->transaction_blocks_normal
4282 && bitmap_bit_p (d
->transaction_blocks_normal
, bb
->index
))
4283 d
->want_irr_scan_normal
= true;
4285 maybe_push_queue (caller
, worklist_p
, &d
->in_worklist
);
4289 /* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement
4290 within the block is irrevocable. */
4293 ipa_tm_scan_irr_block (basic_block bb
)
4295 gimple_stmt_iterator gsi
;
4298 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4300 gimple stmt
= gsi_stmt (gsi
);
4301 switch (gimple_code (stmt
))
4304 if (gimple_assign_single_p (stmt
))
4306 tree lhs
= gimple_assign_lhs (stmt
);
4307 tree rhs
= gimple_assign_rhs1 (stmt
);
4308 if (volatile_var_p (lhs
) || volatile_var_p (rhs
))
4315 tree lhs
= gimple_call_lhs (stmt
);
4316 if (lhs
&& volatile_var_p (lhs
))
4319 if (is_tm_pure_call (stmt
))
4322 fn
= gimple_call_fn (stmt
);
4324 /* Functions with the attribute are by definition irrevocable. */
4325 if (is_tm_irrevocable (fn
))
4328 /* For direct function calls, go ahead and check for replacement
4329 functions, or transitive irrevocable functions. For indirect
4330 functions, we'll ask the runtime. */
4331 if (TREE_CODE (fn
) == ADDR_EXPR
)
4333 struct tm_ipa_cg_data
*d
;
4334 struct cgraph_node
*node
;
4336 fn
= TREE_OPERAND (fn
, 0);
4337 if (is_tm_ending_fndecl (fn
))
4339 if (find_tm_replacement_function (fn
))
4342 node
= cgraph_node::get (fn
);
4343 d
= get_cg_data (&node
, true);
4345 /* Return true if irrevocable, but above all, believe
4347 if (d
->is_irrevocable
4348 && !is_tm_safe_or_pure (fn
))
4355 /* ??? The Approved Method of indicating that an inline
4356 assembly statement is not relevant to the transaction
4357 is to wrap it in a __tm_waiver block. This is not
4358 yet implemented, so we can't check for it. */
4359 if (is_tm_safe (current_function_decl
))
4361 tree t
= build1 (NOP_EXPR
, void_type_node
, size_zero_node
);
4362 SET_EXPR_LOCATION (t
, gimple_location (stmt
));
4363 error ("%Kasm not allowed in %<transaction_safe%> function", t
);
4375 /* For each of the blocks seeded witin PQUEUE, walk the CFG looking
4376 for new irrevocable blocks, marking them in NEW_IRR. Don't bother
4377 scanning past OLD_IRR or EXIT_BLOCKS. */
4380 ipa_tm_scan_irr_blocks (vec
<basic_block
> *pqueue
, bitmap new_irr
,
4381 bitmap old_irr
, bitmap exit_blocks
)
4383 bool any_new_irr
= false;
4386 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4390 basic_block bb
= pqueue
->pop ();
4392 /* Don't re-scan blocks we know already are irrevocable. */
4393 if (old_irr
&& bitmap_bit_p (old_irr
, bb
->index
))
4396 if (ipa_tm_scan_irr_block (bb
))
4398 bitmap_set_bit (new_irr
, bb
->index
);
4401 else if (exit_blocks
== NULL
|| !bitmap_bit_p (exit_blocks
, bb
->index
))
4403 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4404 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
4406 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
4407 pqueue
->safe_push (e
->dest
);
4411 while (!pqueue
->is_empty ());
4413 BITMAP_FREE (visited_blocks
);
4418 /* Propagate the irrevocable property both up and down the dominator tree.
4419 BB is the current block being scanned; EXIT_BLOCKS are the edges of the
4420 TM regions; OLD_IRR are the results of a previous scan of the dominator
4421 tree which has been fully propagated; NEW_IRR is the set of new blocks
4422 which are gaining the irrevocable property during the current scan. */
4425 ipa_tm_propagate_irr (basic_block entry_block
, bitmap new_irr
,
4426 bitmap old_irr
, bitmap exit_blocks
)
4428 vec
<basic_block
> bbs
;
4429 bitmap all_region_blocks
;
4431 /* If this block is in the old set, no need to rescan. */
4432 if (old_irr
&& bitmap_bit_p (old_irr
, entry_block
->index
))
4435 all_region_blocks
= BITMAP_ALLOC (&tm_obstack
);
4436 bbs
= get_tm_region_blocks (entry_block
, exit_blocks
, NULL
,
4437 all_region_blocks
, false);
4440 basic_block bb
= bbs
.pop ();
4441 bool this_irr
= bitmap_bit_p (new_irr
, bb
->index
);
4442 bool all_son_irr
= false;
4446 /* Propagate up. If my children are, I am too, but we must have
4447 at least one child that is. */
4450 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4452 if (!bitmap_bit_p (new_irr
, e
->dest
->index
))
4454 all_son_irr
= false;
4462 /* Add block to new_irr if it hasn't already been processed. */
4463 if (!old_irr
|| !bitmap_bit_p (old_irr
, bb
->index
))
4465 bitmap_set_bit (new_irr
, bb
->index
);
4471 /* Propagate down to everyone we immediately dominate. */
4475 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
4477 son
= next_dom_son (CDI_DOMINATORS
, son
))
4479 /* Make sure block is actually in a TM region, and it
4480 isn't already in old_irr. */
4481 if ((!old_irr
|| !bitmap_bit_p (old_irr
, son
->index
))
4482 && bitmap_bit_p (all_region_blocks
, son
->index
))
4483 bitmap_set_bit (new_irr
, son
->index
);
4487 while (!bbs
.is_empty ());
4489 BITMAP_FREE (all_region_blocks
);
4494 ipa_tm_decrement_clone_counts (basic_block bb
, bool for_clone
)
4496 gimple_stmt_iterator gsi
;
4498 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4500 gimple stmt
= gsi_stmt (gsi
);
4501 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4503 tree fndecl
= gimple_call_fndecl (stmt
);
4506 struct tm_ipa_cg_data
*d
;
4508 struct cgraph_node
*tnode
;
4510 if (is_tm_ending_fndecl (fndecl
))
4512 if (find_tm_replacement_function (fndecl
))
4515 tnode
= cgraph_node::get (fndecl
);
4516 d
= get_cg_data (&tnode
, true);
4518 pcallers
= (for_clone
? &d
->tm_callers_clone
4519 : &d
->tm_callers_normal
);
4521 gcc_assert (*pcallers
> 0);
4528 /* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions,
4529 as well as other irrevocable actions such as inline assembly. Mark all
4530 such blocks as irrevocable and decrement the number of calls to
4531 transactional clones. Return true if, for the transactional clone, the
4532 entire function is irrevocable. */
4535 ipa_tm_scan_irr_function (struct cgraph_node
*node
, bool for_clone
)
4537 struct tm_ipa_cg_data
*d
;
4538 bitmap new_irr
, old_irr
;
4541 /* Builtin operators (operator new, and such). */
4542 if (DECL_STRUCT_FUNCTION (node
->decl
) == NULL
4543 || DECL_STRUCT_FUNCTION (node
->decl
)->cfg
== NULL
)
4546 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
4547 calculate_dominance_info (CDI_DOMINATORS
);
4549 d
= get_cg_data (&node
, true);
4550 auto_vec
<basic_block
, 10> queue
;
4551 new_irr
= BITMAP_ALLOC (&tm_obstack
);
4553 /* Scan each tm region, propagating irrevocable status through the tree. */
4556 old_irr
= d
->irrevocable_blocks_clone
;
4557 queue
.quick_push (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)));
4558 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
, NULL
))
4560 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
4563 ret
= bitmap_bit_p (new_irr
,
4564 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
))->index
);
4569 struct tm_region
*region
;
4571 old_irr
= d
->irrevocable_blocks_normal
;
4572 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
4574 queue
.quick_push (region
->entry_block
);
4575 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
,
4576 region
->exit_blocks
))
4577 ipa_tm_propagate_irr (region
->entry_block
, new_irr
, old_irr
,
4578 region
->exit_blocks
);
4582 /* If we found any new irrevocable blocks, reduce the call count for
4583 transactional clones within the irrevocable blocks. Save the new
4584 set of irrevocable blocks for next time. */
4585 if (!bitmap_empty_p (new_irr
))
4587 bitmap_iterator bmi
;
4590 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4591 ipa_tm_decrement_clone_counts (BASIC_BLOCK_FOR_FN (cfun
, i
),
4596 bitmap_ior_into (old_irr
, new_irr
);
4597 BITMAP_FREE (new_irr
);
4600 d
->irrevocable_blocks_clone
= new_irr
;
4602 d
->irrevocable_blocks_normal
= new_irr
;
4604 if (dump_file
&& new_irr
)
4607 bitmap_iterator bmi
;
4610 dname
= lang_hooks
.decl_printable_name (current_function_decl
, 2);
4611 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4612 fprintf (dump_file
, "%s: bb %d goes irrevocable\n", dname
, i
);
4616 BITMAP_FREE (new_irr
);
4623 /* Return true if, for the transactional clone of NODE, any call
4624 may enter irrevocable mode. */
4627 ipa_tm_mayenterirr_function (struct cgraph_node
*node
)
4629 struct tm_ipa_cg_data
*d
;
4633 d
= get_cg_data (&node
, true);
4635 flags
= flags_from_decl_or_type (decl
);
4637 /* Handle some TM builtins. Ordinarily these aren't actually generated
4638 at this point, but handling these functions when written in by the
4639 user makes it easier to build unit tests. */
4640 if (flags
& ECF_TM_BUILTIN
)
4643 /* Filter out all functions that are marked. */
4644 if (flags
& ECF_TM_PURE
)
4646 if (is_tm_safe (decl
))
4648 if (is_tm_irrevocable (decl
))
4650 if (is_tm_callable (decl
))
4652 if (find_tm_replacement_function (decl
))
4655 /* If we aren't seeing the final version of the function we don't
4656 know what it will contain at runtime. */
4657 if (node
->get_availability () < AVAIL_AVAILABLE
)
4660 /* If the function must go irrevocable, then of course true. */
4661 if (d
->is_irrevocable
)
4664 /* If there are any blocks marked irrevocable, then the function
4665 as a whole may enter irrevocable. */
4666 if (d
->irrevocable_blocks_clone
)
4669 /* We may have previously marked this function as tm_may_enter_irr;
4670 see pass_diagnose_tm_blocks. */
4671 if (node
->local
.tm_may_enter_irr
)
4674 /* Recurse on the main body for aliases. In general, this will
4675 result in one of the bits above being set so that we will not
4676 have to recurse next time. */
4678 return ipa_tm_mayenterirr_function (cgraph_node::get (node
->thunk
.alias
));
4680 /* What remains is unmarked local functions without items that force
4681 the function to go irrevocable. */
4685 /* Diagnose calls from transaction_safe functions to unmarked
4686 functions that are determined to not be safe. */
4689 ipa_tm_diagnose_tm_safe (struct cgraph_node
*node
)
4691 struct cgraph_edge
*e
;
4693 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4694 if (!is_tm_callable (e
->callee
->decl
)
4695 && e
->callee
->local
.tm_may_enter_irr
)
4696 error_at (gimple_location (e
->call_stmt
),
4697 "unsafe function call %qD within "
4698 "%<transaction_safe%> function", e
->callee
->decl
);
4701 /* Diagnose call from atomic transactions to unmarked functions
4702 that are determined to not be safe. */
4705 ipa_tm_diagnose_transaction (struct cgraph_node
*node
,
4706 struct tm_region
*all_tm_regions
)
4708 struct tm_region
*r
;
4710 for (r
= all_tm_regions
; r
; r
= r
->next
)
4711 if (gimple_transaction_subcode (r
->get_transaction_stmt ())
4714 /* Atomic transactions can be nested inside relaxed. */
4716 ipa_tm_diagnose_transaction (node
, r
->inner
);
4720 vec
<basic_block
> bbs
;
4721 gimple_stmt_iterator gsi
;
4725 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
,
4726 r
->irr_blocks
, NULL
, false);
4728 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
4729 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4731 gimple stmt
= gsi_stmt (gsi
);
4734 if (gimple_code (stmt
) == GIMPLE_ASM
)
4736 error_at (gimple_location (stmt
),
4737 "asm not allowed in atomic transaction");
4741 if (!is_gimple_call (stmt
))
4743 fndecl
= gimple_call_fndecl (stmt
);
4745 /* Indirect function calls have been diagnosed already. */
4749 /* Stop at the end of the transaction. */
4750 if (is_tm_ending_fndecl (fndecl
))
4752 if (bitmap_bit_p (r
->exit_blocks
, bb
->index
))
4757 /* Marked functions have been diagnosed already. */
4758 if (is_tm_pure_call (stmt
))
4760 if (is_tm_callable (fndecl
))
4763 if (cgraph_node::local_info (fndecl
)->tm_may_enter_irr
)
4764 error_at (gimple_location (stmt
),
4765 "unsafe function call %qD within "
4766 "atomic transaction", fndecl
);
4773 /* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in
4774 OLD_DECL. The returned value is a freshly malloced pointer that
4775 should be freed by the caller. */
4778 tm_mangle (tree old_asm_id
)
4780 const char *old_asm_name
;
4783 struct demangle_component
*dc
;
4786 /* Determine if the symbol is already a valid C++ mangled name. Do this
4787 even for C, which might be interfacing with C++ code via appropriately
4788 ugly identifiers. */
4789 /* ??? We could probably do just as well checking for "_Z" and be done. */
4790 old_asm_name
= IDENTIFIER_POINTER (old_asm_id
);
4791 dc
= cplus_demangle_v3_components (old_asm_name
, DMGL_NO_OPTS
, &alloc
);
4798 sprintf (length
, "%u", IDENTIFIER_LENGTH (old_asm_id
));
4799 tm_name
= concat ("_ZGTt", length
, old_asm_name
, NULL
);
4803 old_asm_name
+= 2; /* Skip _Z */
4807 case DEMANGLE_COMPONENT_TRANSACTION_CLONE
:
4808 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE
:
4809 /* Don't play silly games, you! */
4812 case DEMANGLE_COMPONENT_HIDDEN_ALIAS
:
4813 /* I'd really like to know if we can ever be passed one of
4814 these from the C++ front end. The Logical Thing would
4815 seem that hidden-alias should be outer-most, so that we
4816 get hidden-alias of a transaction-clone and not vice-versa. */
4824 tm_name
= concat ("_ZGTt", old_asm_name
, NULL
);
4828 new_asm_id
= get_identifier (tm_name
);
4835 ipa_tm_mark_force_output_node (struct cgraph_node
*node
)
4837 node
->mark_force_output ();
4838 node
->analyzed
= true;
4842 ipa_tm_mark_forced_by_abi_node (struct cgraph_node
*node
)
4844 node
->forced_by_abi
= true;
4845 node
->analyzed
= true;
4848 /* Callback data for ipa_tm_create_version_alias. */
4849 struct create_version_alias_info
4851 struct cgraph_node
*old_node
;
4855 /* A subroutine of ipa_tm_create_version, called via
4856 cgraph_for_node_and_aliases. Create new tm clones for each of
4857 the existing aliases. */
4859 ipa_tm_create_version_alias (struct cgraph_node
*node
, void *data
)
4861 struct create_version_alias_info
*info
4862 = (struct create_version_alias_info
*)data
;
4863 tree old_decl
, new_decl
, tm_name
;
4864 struct cgraph_node
*new_node
;
4866 if (!node
->cpp_implicit_alias
)
4869 old_decl
= node
->decl
;
4870 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4871 new_decl
= build_decl (DECL_SOURCE_LOCATION (old_decl
),
4872 TREE_CODE (old_decl
), tm_name
,
4873 TREE_TYPE (old_decl
));
4875 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4876 SET_DECL_RTL (new_decl
, NULL
);
4878 /* Based loosely on C++'s make_alias_for(). */
4879 TREE_PUBLIC (new_decl
) = TREE_PUBLIC (old_decl
);
4880 DECL_CONTEXT (new_decl
) = DECL_CONTEXT (old_decl
);
4881 DECL_LANG_SPECIFIC (new_decl
) = DECL_LANG_SPECIFIC (old_decl
);
4882 TREE_READONLY (new_decl
) = TREE_READONLY (old_decl
);
4883 DECL_EXTERNAL (new_decl
) = 0;
4884 DECL_ARTIFICIAL (new_decl
) = 1;
4885 TREE_ADDRESSABLE (new_decl
) = 1;
4886 TREE_USED (new_decl
) = 1;
4887 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4889 /* Perform the same remapping to the comdat group. */
4890 if (DECL_ONE_ONLY (new_decl
))
4891 varpool_node::get (new_decl
)->set_comdat_group
4892 (tm_mangle (decl_comdat_group_id (old_decl
)));
4894 new_node
= cgraph_node::create_same_body_alias (new_decl
, info
->new_decl
);
4895 new_node
->tm_clone
= true;
4896 new_node
->externally_visible
= info
->old_node
->externally_visible
;
4897 new_node
->no_reorder
= info
->old_node
->no_reorder
;
4898 /* ?? Do not traverse aliases here. */
4899 get_cg_data (&node
, false)->clone
= new_node
;
4901 record_tm_clone_pair (old_decl
, new_decl
);
4903 if (info
->old_node
->force_output
4904 || info
->old_node
->ref_list
.first_referring ())
4905 ipa_tm_mark_force_output_node (new_node
);
4906 if (info
->old_node
->forced_by_abi
)
4907 ipa_tm_mark_forced_by_abi_node (new_node
);
4911 /* Create a copy of the function (possibly declaration only) of OLD_NODE,
4912 appropriate for the transactional clone. */
4915 ipa_tm_create_version (struct cgraph_node
*old_node
)
4917 tree new_decl
, old_decl
, tm_name
;
4918 struct cgraph_node
*new_node
;
4920 old_decl
= old_node
->decl
;
4921 new_decl
= copy_node (old_decl
);
4923 /* DECL_ASSEMBLER_NAME needs to be set before we call
4924 cgraph_copy_node_for_versioning below, because cgraph_node will
4925 fill the assembler_name_hash. */
4926 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4927 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4928 SET_DECL_RTL (new_decl
, NULL
);
4929 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4931 /* Perform the same remapping to the comdat group. */
4932 if (DECL_ONE_ONLY (new_decl
))
4933 varpool_node::get (new_decl
)->set_comdat_group
4934 (tm_mangle (DECL_COMDAT_GROUP (old_decl
)));
4936 gcc_assert (!old_node
->ipa_transforms_to_apply
.exists ());
4937 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
4938 new_node
->local
.local
= false;
4939 new_node
->externally_visible
= old_node
->externally_visible
;
4940 new_node
->lowered
= true;
4941 new_node
->tm_clone
= 1;
4942 get_cg_data (&old_node
, true)->clone
= new_node
;
4944 if (old_node
->get_availability () >= AVAIL_INTERPOSABLE
)
4946 /* Remap extern inline to static inline. */
4947 /* ??? Is it worth trying to use make_decl_one_only? */
4948 if (DECL_DECLARED_INLINE_P (new_decl
) && DECL_EXTERNAL (new_decl
))
4950 DECL_EXTERNAL (new_decl
) = 0;
4951 TREE_PUBLIC (new_decl
) = 0;
4952 DECL_WEAK (new_decl
) = 0;
4955 tree_function_versioning (old_decl
, new_decl
,
4960 record_tm_clone_pair (old_decl
, new_decl
);
4962 symtab
->call_cgraph_insertion_hooks (new_node
);
4963 if (old_node
->force_output
4964 || old_node
->ref_list
.first_referring ())
4965 ipa_tm_mark_force_output_node (new_node
);
4966 if (old_node
->forced_by_abi
)
4967 ipa_tm_mark_forced_by_abi_node (new_node
);
4969 /* Do the same thing, but for any aliases of the original node. */
4971 struct create_version_alias_info data
;
4972 data
.old_node
= old_node
;
4973 data
.new_decl
= new_decl
;
4974 old_node
->call_for_symbol_thunks_and_aliases (ipa_tm_create_version_alias
,
4979 /* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */
4982 ipa_tm_insert_irr_call (struct cgraph_node
*node
, struct tm_region
*region
,
4985 gimple_stmt_iterator gsi
;
4988 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4990 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
),
4991 1, build_int_cst (NULL_TREE
, MODE_SERIALIRREVOCABLE
));
4993 split_block_after_labels (bb
);
4994 gsi
= gsi_after_labels (bb
);
4995 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
4997 node
->create_edge (cgraph_node::get_create
4998 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
)),
5000 compute_call_stmt_bb_frequency (node
->decl
,
5004 /* Construct a call to TM_GETTMCLONE and insert it before GSI. */
5007 ipa_tm_insert_gettmclone_call (struct cgraph_node
*node
,
5008 struct tm_region
*region
,
5009 gimple_stmt_iterator
*gsi
, gcall
*stmt
)
5011 tree gettm_fn
, ret
, old_fn
, callfn
;
5016 old_fn
= gimple_call_fn (stmt
);
5018 if (TREE_CODE (old_fn
) == ADDR_EXPR
)
5020 tree fndecl
= TREE_OPERAND (old_fn
, 0);
5021 tree clone
= get_tm_clone_pair (fndecl
);
5023 /* By transforming the call into a TM_GETTMCLONE, we are
5024 technically taking the address of the original function and
5025 its clone. Explain this so inlining will know this function
5027 cgraph_node::get (fndecl
)->mark_address_taken () ;
5029 cgraph_node::get (clone
)->mark_address_taken ();
5032 safe
= is_tm_safe (TREE_TYPE (old_fn
));
5033 gettm_fn
= builtin_decl_explicit (safe
? BUILT_IN_TM_GETTMCLONE_SAFE
5034 : BUILT_IN_TM_GETTMCLONE_IRR
);
5035 ret
= create_tmp_var (ptr_type_node
, NULL
);
5038 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
5040 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */
5041 if (TREE_CODE (old_fn
) == OBJ_TYPE_REF
)
5042 old_fn
= OBJ_TYPE_REF_EXPR (old_fn
);
5044 g
= gimple_build_call (gettm_fn
, 1, old_fn
);
5045 ret
= make_ssa_name (ret
, g
);
5046 gimple_call_set_lhs (g
, ret
);
5048 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
5050 node
->create_edge (cgraph_node::get_create (gettm_fn
), g
, 0,
5051 compute_call_stmt_bb_frequency (node
->decl
,
5054 /* Cast return value from tm_gettmclone* into appropriate function
5056 callfn
= create_tmp_var (TREE_TYPE (old_fn
), NULL
);
5057 g2
= gimple_build_assign (callfn
,
5058 fold_build1 (NOP_EXPR
, TREE_TYPE (callfn
), ret
));
5059 callfn
= make_ssa_name (callfn
, g2
);
5060 gimple_assign_set_lhs (g2
, callfn
);
5061 gsi_insert_before (gsi
, g2
, GSI_SAME_STMT
);
5063 /* ??? This is a hack to preserve the NOTHROW bit on the call,
5064 which we would have derived from the decl. Failure to save
5065 this bit means we might have to split the basic block. */
5066 if (gimple_call_nothrow_p (stmt
))
5067 gimple_call_set_nothrow (stmt
, true);
5069 gimple_call_set_fn (stmt
, callfn
);
5071 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS
5072 for a call statement. Fix it. */
5074 tree lhs
= gimple_call_lhs (stmt
);
5075 tree rettype
= TREE_TYPE (gimple_call_fntype (stmt
));
5077 && !useless_type_conversion_p (TREE_TYPE (lhs
), rettype
))
5081 temp
= create_tmp_reg (rettype
, 0);
5082 gimple_call_set_lhs (stmt
, temp
);
5084 g2
= gimple_build_assign (lhs
,
5085 fold_build1 (VIEW_CONVERT_EXPR
,
5086 TREE_TYPE (lhs
), temp
));
5087 gsi_insert_after (gsi
, g2
, GSI_SAME_STMT
);
5092 cgraph_edge
*e
= cgraph_node::get (current_function_decl
)->get_edge (stmt
);
5093 if (e
&& e
->indirect_info
)
5094 e
->indirect_info
->polymorphic
= false;
5099 /* Helper function for ipa_tm_transform_calls*. Given a call
5100 statement in GSI which resides inside transaction REGION, redirect
5101 the call to either its wrapper function, or its clone. */
5104 ipa_tm_transform_calls_redirect (struct cgraph_node
*node
,
5105 struct tm_region
*region
,
5106 gimple_stmt_iterator
*gsi
,
5107 bool *need_ssa_rename_p
)
5109 gcall
*stmt
= as_a
<gcall
*> (gsi_stmt (*gsi
));
5110 struct cgraph_node
*new_node
;
5111 struct cgraph_edge
*e
= node
->get_edge (stmt
);
5112 tree fndecl
= gimple_call_fndecl (stmt
);
5114 /* For indirect calls, pass the address through the runtime. */
5117 *need_ssa_rename_p
|=
5118 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5122 /* Handle some TM builtins. Ordinarily these aren't actually generated
5123 at this point, but handling these functions when written in by the
5124 user makes it easier to build unit tests. */
5125 if (flags_from_decl_or_type (fndecl
) & ECF_TM_BUILTIN
)
5128 /* Fixup recursive calls inside clones. */
5129 /* ??? Why did cgraph_copy_node_for_versioning update the call edges
5130 for recursion but not update the call statements themselves? */
5131 if (e
->caller
== e
->callee
&& decl_is_tm_clone (current_function_decl
))
5133 gimple_call_set_fndecl (stmt
, current_function_decl
);
5137 /* If there is a replacement, use it. */
5138 fndecl
= find_tm_replacement_function (fndecl
);
5141 new_node
= cgraph_node::get_create (fndecl
);
5143 /* ??? Mark all transaction_wrap functions tm_may_enter_irr.
5145 We can't do this earlier in record_tm_replacement because
5146 cgraph_remove_unreachable_nodes is called before we inject
5147 references to the node. Further, we can't do this in some
5148 nice central place in ipa_tm_execute because we don't have
5149 the exact list of wrapper functions that would be used.
5150 Marking more wrappers than necessary results in the creation
5151 of unnecessary cgraph_nodes, which can cause some of the
5152 other IPA passes to crash.
5154 We do need to mark these nodes so that we get the proper
5155 result in expand_call_tm. */
5156 /* ??? This seems broken. How is it that we're marking the
5157 CALLEE as may_enter_irr? Surely we should be marking the
5158 CALLER. Also note that find_tm_replacement_function also
5159 contains mappings into the TM runtime, e.g. memcpy. These
5160 we know won't go irrevocable. */
5161 new_node
->local
.tm_may_enter_irr
= 1;
5165 struct tm_ipa_cg_data
*d
;
5166 struct cgraph_node
*tnode
= e
->callee
;
5168 d
= get_cg_data (&tnode
, true);
5169 new_node
= d
->clone
;
5171 /* As we've already skipped pure calls and appropriate builtins,
5172 and we've already marked irrevocable blocks, if we can't come
5173 up with a static replacement, then ask the runtime. */
5174 if (new_node
== NULL
)
5176 *need_ssa_rename_p
|=
5177 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5181 fndecl
= new_node
->decl
;
5184 e
->redirect_callee (new_node
);
5185 gimple_call_set_fndecl (stmt
, fndecl
);
5188 /* Helper function for ipa_tm_transform_calls. For a given BB,
5189 install calls to tm_irrevocable when IRR_BLOCKS are reached,
5190 redirect other calls to the generated transactional clone. */
5193 ipa_tm_transform_calls_1 (struct cgraph_node
*node
, struct tm_region
*region
,
5194 basic_block bb
, bitmap irr_blocks
)
5196 gimple_stmt_iterator gsi
;
5197 bool need_ssa_rename
= false;
5199 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5201 ipa_tm_insert_irr_call (node
, region
, bb
);
5205 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5207 gimple stmt
= gsi_stmt (gsi
);
5209 if (!is_gimple_call (stmt
))
5211 if (is_tm_pure_call (stmt
))
5214 /* Redirect edges to the appropriate replacement or clone. */
5215 ipa_tm_transform_calls_redirect (node
, region
, &gsi
, &need_ssa_rename
);
5218 return need_ssa_rename
;
5221 /* Walk the CFG for REGION, beginning at BB. Install calls to
5222 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to
5223 the generated transactional clone. */
5226 ipa_tm_transform_calls (struct cgraph_node
*node
, struct tm_region
*region
,
5227 basic_block bb
, bitmap irr_blocks
)
5229 bool need_ssa_rename
= false;
5232 auto_vec
<basic_block
> queue
;
5233 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
5235 queue
.safe_push (bb
);
5241 ipa_tm_transform_calls_1 (node
, region
, bb
, irr_blocks
);
5243 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5246 if (region
&& bitmap_bit_p (region
->exit_blocks
, bb
->index
))
5249 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
5250 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
5252 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
5253 queue
.safe_push (e
->dest
);
5256 while (!queue
.is_empty ());
5258 BITMAP_FREE (visited_blocks
);
5260 return need_ssa_rename
;
5263 /* Transform the calls within the TM regions within NODE. */
5266 ipa_tm_transform_transaction (struct cgraph_node
*node
)
5268 struct tm_ipa_cg_data
*d
;
5269 struct tm_region
*region
;
5270 bool need_ssa_rename
= false;
5272 d
= get_cg_data (&node
, true);
5274 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5275 calculate_dominance_info (CDI_DOMINATORS
);
5277 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
5279 /* If we're sure to go irrevocable, don't transform anything. */
5280 if (d
->irrevocable_blocks_normal
5281 && bitmap_bit_p (d
->irrevocable_blocks_normal
,
5282 region
->entry_block
->index
))
5284 transaction_subcode_ior (region
, GTMA_DOES_GO_IRREVOCABLE
5285 | GTMA_MAY_ENTER_IRREVOCABLE
5286 | GTMA_HAS_NO_INSTRUMENTATION
);
5291 ipa_tm_transform_calls (node
, region
, region
->entry_block
,
5292 d
->irrevocable_blocks_normal
);
5295 if (need_ssa_rename
)
5296 update_ssa (TODO_update_ssa_only_virtuals
);
5301 /* Transform the calls within the transactional clone of NODE. */
5304 ipa_tm_transform_clone (struct cgraph_node
*node
)
5306 struct tm_ipa_cg_data
*d
;
5307 bool need_ssa_rename
;
5309 d
= get_cg_data (&node
, true);
5311 /* If this function makes no calls and has no irrevocable blocks,
5312 then there's nothing to do. */
5313 /* ??? Remove non-aborting top-level transactions. */
5314 if (!node
->callees
&& !node
->indirect_calls
&& !d
->irrevocable_blocks_clone
)
5317 push_cfun (DECL_STRUCT_FUNCTION (d
->clone
->decl
));
5318 calculate_dominance_info (CDI_DOMINATORS
);
5321 ipa_tm_transform_calls (d
->clone
, NULL
,
5322 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
5323 d
->irrevocable_blocks_clone
);
5325 if (need_ssa_rename
)
5326 update_ssa (TODO_update_ssa_only_virtuals
);
5331 /* Main entry point for the transactional memory IPA pass. */
5334 ipa_tm_execute (void)
5336 cgraph_node_queue tm_callees
= cgraph_node_queue ();
5337 /* List of functions that will go irrevocable. */
5338 cgraph_node_queue irr_worklist
= cgraph_node_queue ();
5340 struct cgraph_node
*node
;
5341 struct tm_ipa_cg_data
*d
;
5342 enum availability a
;
5345 #ifdef ENABLE_CHECKING
5346 cgraph_node::verify_cgraph_nodes ();
5349 bitmap_obstack_initialize (&tm_obstack
);
5350 initialize_original_copy_tables ();
5352 /* For all local functions marked tm_callable, queue them. */
5353 FOR_EACH_DEFINED_FUNCTION (node
)
5354 if (is_tm_callable (node
->decl
)
5355 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5357 d
= get_cg_data (&node
, true);
5358 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5361 /* For all local reachable functions... */
5362 FOR_EACH_DEFINED_FUNCTION (node
)
5364 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5366 /* ... marked tm_pure, record that fact for the runtime by
5367 indicating that the pure function is its own tm_callable.
5368 No need to do this if the function's address can't be taken. */
5369 if (is_tm_pure (node
->decl
))
5371 if (!node
->local
.local
)
5372 record_tm_clone_pair (node
->decl
, node
->decl
);
5376 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5377 calculate_dominance_info (CDI_DOMINATORS
);
5379 tm_region_init (NULL
);
5382 d
= get_cg_data (&node
, true);
5384 /* Scan for calls that are in each transaction, and
5385 generate the uninstrumented code path. */
5386 ipa_tm_scan_calls_transaction (d
, &tm_callees
);
5388 /* Put it in the worklist so we can scan the function
5389 later (ipa_tm_scan_irr_function) and mark the
5390 irrevocable blocks. */
5391 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5392 d
->want_irr_scan_normal
= true;
5398 /* For every local function on the callee list, scan as if we will be
5399 creating a transactional clone, queueing all new functions we find
5401 for (i
= 0; i
< tm_callees
.length (); ++i
)
5403 node
= tm_callees
[i
];
5404 a
= node
->get_availability ();
5405 d
= get_cg_data (&node
, true);
5407 /* Put it in the worklist so we can scan the function later
5408 (ipa_tm_scan_irr_function) and mark the irrevocable
5410 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5412 /* Some callees cannot be arbitrarily cloned. These will always be
5413 irrevocable. Mark these now, so that we need not scan them. */
5414 if (is_tm_irrevocable (node
->decl
))
5415 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5416 else if (a
<= AVAIL_NOT_AVAILABLE
5417 && !is_tm_safe_or_pure (node
->decl
))
5418 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5419 else if (a
>= AVAIL_INTERPOSABLE
)
5421 if (!tree_versionable_function_p (node
->decl
))
5422 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5423 else if (!d
->is_irrevocable
)
5425 /* If this is an alias, make sure its base is queued as well.
5426 we need not scan the callees now, as the base will do. */
5429 node
= cgraph_node::get (node
->thunk
.alias
);
5430 d
= get_cg_data (&node
, true);
5431 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5435 /* Add all nodes called by this function into
5436 tm_callees as well. */
5437 ipa_tm_scan_calls_clone (node
, &tm_callees
);
5442 /* Iterate scans until no more work to be done. Prefer not to use
5443 vec::pop because the worklist tends to follow a breadth-first
5444 search of the callgraph, which should allow convergance with a
5445 minimum number of scans. But we also don't want the worklist
5446 array to grow without bound, so we shift the array up periodically. */
5447 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5449 if (i
> 256 && i
== irr_worklist
.length () / 8)
5451 irr_worklist
.block_remove (0, i
);
5455 node
= irr_worklist
[i
];
5456 d
= get_cg_data (&node
, true);
5457 d
->in_worklist
= false;
5459 if (d
->want_irr_scan_normal
)
5461 d
->want_irr_scan_normal
= false;
5462 ipa_tm_scan_irr_function (node
, false);
5464 if (d
->in_callee_queue
&& ipa_tm_scan_irr_function (node
, true))
5465 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5468 /* For every function on the callee list, collect the tm_may_enter_irr
5470 irr_worklist
.truncate (0);
5471 for (i
= 0; i
< tm_callees
.length (); ++i
)
5473 node
= tm_callees
[i
];
5474 if (ipa_tm_mayenterirr_function (node
))
5476 d
= get_cg_data (&node
, true);
5477 gcc_assert (d
->in_worklist
== false);
5478 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5482 /* Propagate the tm_may_enter_irr bit to callers until stable. */
5483 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5485 struct cgraph_node
*caller
;
5486 struct cgraph_edge
*e
;
5487 struct ipa_ref
*ref
;
5489 if (i
> 256 && i
== irr_worklist
.length () / 8)
5491 irr_worklist
.block_remove (0, i
);
5495 node
= irr_worklist
[i
];
5496 d
= get_cg_data (&node
, true);
5497 d
->in_worklist
= false;
5498 node
->local
.tm_may_enter_irr
= true;
5500 /* Propagate back to normal callers. */
5501 for (e
= node
->callers
; e
; e
= e
->next_caller
)
5504 if (!is_tm_safe_or_pure (caller
->decl
)
5505 && !caller
->local
.tm_may_enter_irr
)
5507 d
= get_cg_data (&caller
, true);
5508 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5512 /* Propagate back to referring aliases as well. */
5513 FOR_EACH_ALIAS (node
, ref
)
5515 caller
= dyn_cast
<cgraph_node
*> (ref
->referring
);
5516 if (!caller
->local
.tm_may_enter_irr
)
5518 /* ?? Do not traverse aliases here. */
5519 d
= get_cg_data (&caller
, false);
5520 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5525 /* Now validate all tm_safe functions, and all atomic regions in
5527 FOR_EACH_DEFINED_FUNCTION (node
)
5529 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5531 d
= get_cg_data (&node
, true);
5532 if (is_tm_safe (node
->decl
))
5533 ipa_tm_diagnose_tm_safe (node
);
5534 else if (d
->all_tm_regions
)
5535 ipa_tm_diagnose_transaction (node
, d
->all_tm_regions
);
5538 /* Create clones. Do those that are not irrevocable and have a
5539 positive call count. Do those publicly visible functions that
5540 the user directed us to clone. */
5541 for (i
= 0; i
< tm_callees
.length (); ++i
)
5545 node
= tm_callees
[i
];
5546 if (node
->cpp_implicit_alias
)
5549 a
= node
->get_availability ();
5550 d
= get_cg_data (&node
, true);
5552 if (a
<= AVAIL_NOT_AVAILABLE
)
5553 doit
= is_tm_callable (node
->decl
);
5554 else if (a
<= AVAIL_AVAILABLE
&& is_tm_callable (node
->decl
))
5556 else if (!d
->is_irrevocable
5557 && d
->tm_callers_normal
+ d
->tm_callers_clone
> 0)
5561 ipa_tm_create_version (node
);
5564 /* Redirect calls to the new clones, and insert irrevocable marks. */
5565 for (i
= 0; i
< tm_callees
.length (); ++i
)
5567 node
= tm_callees
[i
];
5570 d
= get_cg_data (&node
, true);
5572 ipa_tm_transform_clone (node
);
5575 FOR_EACH_DEFINED_FUNCTION (node
)
5577 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5579 d
= get_cg_data (&node
, true);
5580 if (d
->all_tm_regions
)
5581 ipa_tm_transform_transaction (node
);
5584 /* Free and clear all data structures. */
5585 tm_callees
.release ();
5586 irr_worklist
.release ();
5587 bitmap_obstack_release (&tm_obstack
);
5588 free_original_copy_tables ();
5590 FOR_EACH_FUNCTION (node
)
5593 #ifdef ENABLE_CHECKING
5594 cgraph_node::verify_cgraph_nodes ();
5602 const pass_data pass_data_ipa_tm
=
5604 SIMPLE_IPA_PASS
, /* type */
5606 OPTGROUP_NONE
, /* optinfo_flags */
5607 TV_TRANS_MEM
, /* tv_id */
5608 ( PROP_ssa
| PROP_cfg
), /* properties_required */
5609 0, /* properties_provided */
5610 0, /* properties_destroyed */
5611 0, /* todo_flags_start */
5612 0, /* todo_flags_finish */
5615 class pass_ipa_tm
: public simple_ipa_opt_pass
5618 pass_ipa_tm (gcc::context
*ctxt
)
5619 : simple_ipa_opt_pass (pass_data_ipa_tm
, ctxt
)
5622 /* opt_pass methods: */
5623 virtual bool gate (function
*) { return flag_tm
; }
5624 virtual unsigned int execute (function
*) { return ipa_tm_execute (); }
5626 }; // class pass_ipa_tm
5630 simple_ipa_opt_pass
*
5631 make_pass_ipa_tm (gcc::context
*ctxt
)
5633 return new pass_ipa_tm (ctxt
);
5636 #include "gt-trans-mem.h"