1 /* Passes for transactional memory support.
2 Copyright (C) 2008-2015 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
27 #include "fold-const.h"
30 #include "hard-reg-set.h"
32 #include "dominance.h"
34 #include "basic-block.h"
35 #include "tree-ssa-alias.h"
36 #include "internal-fn.h"
38 #include "gimple-expr.h"
44 #include "gimple-iterator.h"
45 #include "gimplify-me.h"
46 #include "gimple-walk.h"
47 #include "gimple-ssa.h"
48 #include "plugin-api.h"
52 #include "stringpool.h"
53 #include "tree-ssanames.h"
54 #include "tree-into-ssa.h"
55 #include "tree-pass.h"
56 #include "tree-inline.h"
57 #include "diagnostic-core.h"
60 #include "trans-mem.h"
63 #include "langhooks.h"
64 #include "gimple-pretty-print.h"
66 #include "tree-ssa-address.h"
69 #define A_RUNINSTRUMENTEDCODE 0x0001
70 #define A_RUNUNINSTRUMENTEDCODE 0x0002
71 #define A_SAVELIVEVARIABLES 0x0004
72 #define A_RESTORELIVEVARIABLES 0x0008
73 #define A_ABORTTRANSACTION 0x0010
75 #define AR_USERABORT 0x0001
76 #define AR_USERRETRY 0x0002
77 #define AR_TMCONFLICT 0x0004
78 #define AR_EXCEPTIONBLOCKABORT 0x0008
79 #define AR_OUTERABORT 0x0010
81 #define MODE_SERIALIRREVOCABLE 0x0000
84 /* The representation of a transaction changes several times during the
85 lowering process. In the beginning, in the front-end we have the
86 GENERIC tree TRANSACTION_EXPR. For example,
94 During initial gimplification (gimplify.c) the TRANSACTION_EXPR node is
95 trivially replaced with a GIMPLE_TRANSACTION node.
97 During pass_lower_tm, we examine the body of transactions looking
98 for aborts. Transactions that do not contain an abort may be
99 merged into an outer transaction. We also add a TRY-FINALLY node
100 to arrange for the transaction to be committed on any exit.
102 [??? Think about how this arrangement affects throw-with-commit
103 and throw-with-abort operations. In this case we want the TRY to
104 handle gotos, but not to catch any exceptions because the transaction
105 will already be closed.]
107 GIMPLE_TRANSACTION [label=NULL] {
114 __builtin___tm_abort ();
116 __builtin___tm_commit ();
120 During pass_lower_eh, we create EH regions for the transactions,
121 intermixed with the regular EH stuff. This gives us a nice persistent
122 mapping (all the way through rtl) from transactional memory operation
123 back to the transaction, which allows us to get the abnormal edges
124 correct to model transaction aborts and restarts:
126 GIMPLE_TRANSACTION [label=over]
132 __builtin___tm_abort ();
133 __builtin___tm_commit ();
136 This is the end of all_lowering_passes, and so is what is present
137 during the IPA passes, and through all of the optimization passes.
139 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all
140 functions and mark functions for cloning.
142 At the end of gimple optimization, before exiting SSA form,
143 pass_tm_edges replaces statements that perform transactional
144 memory operations with the appropriate TM builtins, and swap
145 out function calls with their transactional clones. At this
146 point we introduce the abnormal transaction restart edges and
147 complete lowering of the GIMPLE_TRANSACTION node.
149 x = __builtin___tm_start (MAY_ABORT);
151 if (x & abort_transaction)
154 t0 = __builtin___tm_load (global);
156 __builtin___tm_store (&global, t1);
158 __builtin___tm_abort ();
159 __builtin___tm_commit ();
163 static void *expand_regions (struct tm_region
*,
164 void *(*callback
)(struct tm_region
*, void *),
168 /* Return the attributes we want to examine for X, or NULL if it's not
169 something we examine. We look at function types, but allow pointers
170 to function types and function decls and peek through. */
173 get_attrs_for (const_tree x
)
178 switch (TREE_CODE (x
))
181 return TYPE_ATTRIBUTES (TREE_TYPE (x
));
188 if (TREE_CODE (x
) != POINTER_TYPE
)
194 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
200 return TYPE_ATTRIBUTES (x
);
204 /* Return true if X has been marked TM_PURE. */
207 is_tm_pure (const_tree x
)
211 switch (TREE_CODE (x
))
222 if (TREE_CODE (x
) != POINTER_TYPE
)
228 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
233 flags
= flags_from_decl_or_type (x
);
234 return (flags
& ECF_TM_PURE
) != 0;
237 /* Return true if X has been marked TM_IRREVOCABLE. */
240 is_tm_irrevocable (tree x
)
242 tree attrs
= get_attrs_for (x
);
244 if (attrs
&& lookup_attribute ("transaction_unsafe", attrs
))
247 /* A call to the irrevocable builtin is by definition,
249 if (TREE_CODE (x
) == ADDR_EXPR
)
250 x
= TREE_OPERAND (x
, 0);
251 if (TREE_CODE (x
) == FUNCTION_DECL
252 && DECL_BUILT_IN_CLASS (x
) == BUILT_IN_NORMAL
253 && DECL_FUNCTION_CODE (x
) == BUILT_IN_TM_IRREVOCABLE
)
259 /* Return true if X has been marked TM_SAFE. */
262 is_tm_safe (const_tree x
)
266 tree attrs
= get_attrs_for (x
);
269 if (lookup_attribute ("transaction_safe", attrs
))
271 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
278 /* Return true if CALL is const, or tm_pure. */
281 is_tm_pure_call (gimple call
)
283 tree fn
= gimple_call_fn (call
);
285 if (TREE_CODE (fn
) == ADDR_EXPR
)
287 fn
= TREE_OPERAND (fn
, 0);
288 gcc_assert (TREE_CODE (fn
) == FUNCTION_DECL
);
293 return is_tm_pure (fn
);
296 /* Return true if X has been marked TM_CALLABLE. */
299 is_tm_callable (tree x
)
301 tree attrs
= get_attrs_for (x
);
304 if (lookup_attribute ("transaction_callable", attrs
))
306 if (lookup_attribute ("transaction_safe", attrs
))
308 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
314 /* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */
317 is_tm_may_cancel_outer (tree x
)
319 tree attrs
= get_attrs_for (x
);
321 return lookup_attribute ("transaction_may_cancel_outer", attrs
) != NULL
;
325 /* Return true for built in functions that "end" a transaction. */
328 is_tm_ending_fndecl (tree fndecl
)
330 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
331 switch (DECL_FUNCTION_CODE (fndecl
))
333 case BUILT_IN_TM_COMMIT
:
334 case BUILT_IN_TM_COMMIT_EH
:
335 case BUILT_IN_TM_ABORT
:
336 case BUILT_IN_TM_IRREVOCABLE
:
345 /* Return true if STMT is a built in function call that "ends" a
349 is_tm_ending (gimple stmt
)
353 if (gimple_code (stmt
) != GIMPLE_CALL
)
356 fndecl
= gimple_call_fndecl (stmt
);
357 return (fndecl
!= NULL_TREE
358 && is_tm_ending_fndecl (fndecl
));
361 /* Return true if STMT is a TM load. */
364 is_tm_load (gimple stmt
)
368 if (gimple_code (stmt
) != GIMPLE_CALL
)
371 fndecl
= gimple_call_fndecl (stmt
);
372 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
373 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl
)));
376 /* Same as above, but for simple TM loads, that is, not the
377 after-write, after-read, etc optimized variants. */
380 is_tm_simple_load (gimple stmt
)
384 if (gimple_code (stmt
) != GIMPLE_CALL
)
387 fndecl
= gimple_call_fndecl (stmt
);
388 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
390 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
391 return (fcode
== BUILT_IN_TM_LOAD_1
392 || fcode
== BUILT_IN_TM_LOAD_2
393 || fcode
== BUILT_IN_TM_LOAD_4
394 || fcode
== BUILT_IN_TM_LOAD_8
395 || fcode
== BUILT_IN_TM_LOAD_FLOAT
396 || fcode
== BUILT_IN_TM_LOAD_DOUBLE
397 || fcode
== BUILT_IN_TM_LOAD_LDOUBLE
398 || fcode
== BUILT_IN_TM_LOAD_M64
399 || fcode
== BUILT_IN_TM_LOAD_M128
400 || fcode
== BUILT_IN_TM_LOAD_M256
);
405 /* Return true if STMT is a TM store. */
408 is_tm_store (gimple stmt
)
412 if (gimple_code (stmt
) != GIMPLE_CALL
)
415 fndecl
= gimple_call_fndecl (stmt
);
416 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
417 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl
)));
420 /* Same as above, but for simple TM stores, that is, not the
421 after-write, after-read, etc optimized variants. */
424 is_tm_simple_store (gimple stmt
)
428 if (gimple_code (stmt
) != GIMPLE_CALL
)
431 fndecl
= gimple_call_fndecl (stmt
);
432 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
434 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
435 return (fcode
== BUILT_IN_TM_STORE_1
436 || fcode
== BUILT_IN_TM_STORE_2
437 || fcode
== BUILT_IN_TM_STORE_4
438 || fcode
== BUILT_IN_TM_STORE_8
439 || fcode
== BUILT_IN_TM_STORE_FLOAT
440 || fcode
== BUILT_IN_TM_STORE_DOUBLE
441 || fcode
== BUILT_IN_TM_STORE_LDOUBLE
442 || fcode
== BUILT_IN_TM_STORE_M64
443 || fcode
== BUILT_IN_TM_STORE_M128
444 || fcode
== BUILT_IN_TM_STORE_M256
);
449 /* Return true if FNDECL is BUILT_IN_TM_ABORT. */
452 is_tm_abort (tree fndecl
)
455 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
456 && DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_TM_ABORT
);
459 /* Build a GENERIC tree for a user abort. This is called by front ends
460 while transforming the __tm_abort statement. */
463 build_tm_abort_call (location_t loc
, bool is_outer
)
465 return build_call_expr_loc (loc
, builtin_decl_explicit (BUILT_IN_TM_ABORT
), 1,
466 build_int_cst (integer_type_node
,
468 | (is_outer
? AR_OUTERABORT
: 0)));
471 /* Map for aribtrary function replacement under TM, as created
472 by the tm_wrap attribute. */
474 struct tm_wrapper_hasher
: ggc_cache_hasher
<tree_map
*>
476 static inline hashval_t
hash (tree_map
*m
) { return m
->hash
; }
478 equal (tree_map
*a
, tree_map
*b
)
480 return a
->base
.from
== b
->base
.from
;
484 keep_cache_entry (tree_map
*&m
)
486 return ggc_marked_p (m
->base
.from
);
490 static GTY((cache
)) hash_table
<tm_wrapper_hasher
> *tm_wrap_map
;
493 record_tm_replacement (tree from
, tree to
)
495 struct tree_map
**slot
, *h
;
497 /* Do not inline wrapper functions that will get replaced in the TM
500 Suppose you have foo() that will get replaced into tmfoo(). Make
501 sure the inliner doesn't try to outsmart us and inline foo()
502 before we get a chance to do the TM replacement. */
503 DECL_UNINLINABLE (from
) = 1;
505 if (tm_wrap_map
== NULL
)
506 tm_wrap_map
= hash_table
<tm_wrapper_hasher
>::create_ggc (32);
508 h
= ggc_alloc
<tree_map
> ();
509 h
->hash
= htab_hash_pointer (from
);
513 slot
= tm_wrap_map
->find_slot_with_hash (h
, h
->hash
, INSERT
);
517 /* Return a TM-aware replacement function for DECL. */
520 find_tm_replacement_function (tree fndecl
)
524 struct tree_map
*h
, in
;
526 in
.base
.from
= fndecl
;
527 in
.hash
= htab_hash_pointer (fndecl
);
528 h
= tm_wrap_map
->find_with_hash (&in
, in
.hash
);
533 /* ??? We may well want TM versions of most of the common <string.h>
534 functions. For now, we've already these two defined. */
535 /* Adjust expand_call_tm() attributes as necessary for the cases
537 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
538 switch (DECL_FUNCTION_CODE (fndecl
))
540 case BUILT_IN_MEMCPY
:
541 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY
);
542 case BUILT_IN_MEMMOVE
:
543 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
);
544 case BUILT_IN_MEMSET
:
545 return builtin_decl_explicit (BUILT_IN_TM_MEMSET
);
553 /* When appropriate, record TM replacement for memory allocation functions.
555 FROM is the FNDECL to wrap. */
557 tm_malloc_replacement (tree from
)
562 if (TREE_CODE (from
) != FUNCTION_DECL
)
565 /* If we have a previous replacement, the user must be explicitly
566 wrapping malloc/calloc/free. They better know what they're
568 if (find_tm_replacement_function (from
))
571 str
= IDENTIFIER_POINTER (DECL_NAME (from
));
573 if (!strcmp (str
, "malloc"))
574 to
= builtin_decl_explicit (BUILT_IN_TM_MALLOC
);
575 else if (!strcmp (str
, "calloc"))
576 to
= builtin_decl_explicit (BUILT_IN_TM_CALLOC
);
577 else if (!strcmp (str
, "free"))
578 to
= builtin_decl_explicit (BUILT_IN_TM_FREE
);
582 TREE_NOTHROW (to
) = 0;
584 record_tm_replacement (from
, to
);
587 /* Diagnostics for tm_safe functions/regions. Called by the front end
588 once we've lowered the function to high-gimple. */
590 /* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq.
591 Process exactly one statement. WI->INFO is set to non-null when in
592 the context of a tm_safe function, and null for a __transaction block. */
594 #define DIAG_TM_OUTER 1
595 #define DIAG_TM_SAFE 2
596 #define DIAG_TM_RELAXED 4
600 unsigned int summary_flags
: 8;
601 unsigned int block_flags
: 8;
602 unsigned int func_flags
: 8;
603 unsigned int saw_volatile
: 1;
607 /* Return true if T is a volatile variable of some kind. */
610 volatile_var_p (tree t
)
612 return (SSA_VAR_P (t
)
613 && TREE_THIS_VOLATILE (TREE_TYPE (t
)));
616 /* Tree callback function for diagnose_tm pass. */
619 diagnose_tm_1_op (tree
*tp
, int *walk_subtrees ATTRIBUTE_UNUSED
,
622 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
623 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
625 if (volatile_var_p (*tp
)
626 && d
->block_flags
& DIAG_TM_SAFE
630 error_at (gimple_location (d
->stmt
),
631 "invalid volatile use of %qD inside transaction",
639 is_tm_safe_or_pure (const_tree x
)
641 return is_tm_safe (x
) || is_tm_pure (x
);
645 diagnose_tm_1 (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
646 struct walk_stmt_info
*wi
)
648 gimple stmt
= gsi_stmt (*gsi
);
649 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
651 /* Save stmt for use in leaf analysis. */
654 switch (gimple_code (stmt
))
658 tree fn
= gimple_call_fn (stmt
);
660 if ((d
->summary_flags
& DIAG_TM_OUTER
) == 0
661 && is_tm_may_cancel_outer (fn
))
662 error_at (gimple_location (stmt
),
663 "%<transaction_may_cancel_outer%> function call not within"
664 " outer transaction or %<transaction_may_cancel_outer%>");
666 if (d
->summary_flags
& DIAG_TM_SAFE
)
668 bool is_safe
, direct_call_p
;
671 if (TREE_CODE (fn
) == ADDR_EXPR
672 && TREE_CODE (TREE_OPERAND (fn
, 0)) == FUNCTION_DECL
)
674 direct_call_p
= true;
675 replacement
= TREE_OPERAND (fn
, 0);
676 replacement
= find_tm_replacement_function (replacement
);
682 direct_call_p
= false;
683 replacement
= NULL_TREE
;
686 if (is_tm_safe_or_pure (fn
))
688 else if (is_tm_callable (fn
) || is_tm_irrevocable (fn
))
690 /* A function explicitly marked transaction_callable as
691 opposed to transaction_safe is being defined to be
692 unsafe as part of its ABI, regardless of its contents. */
695 else if (direct_call_p
)
697 if (IS_TYPE_OR_DECL_P (fn
)
698 && flags_from_decl_or_type (fn
) & ECF_TM_BUILTIN
)
700 else if (replacement
)
702 /* ??? At present we've been considering replacements
703 merely transaction_callable, and therefore might
704 enter irrevocable. The tm_wrap attribute has not
705 yet made it into the new language spec. */
710 /* ??? Diagnostics for unmarked direct calls moved into
711 the IPA pass. Section 3.2 of the spec details how
712 functions not marked should be considered "implicitly
713 safe" based on having examined the function body. */
719 /* An unmarked indirect call. Consider it unsafe even
720 though optimization may yet figure out how to inline. */
726 if (TREE_CODE (fn
) == ADDR_EXPR
)
727 fn
= TREE_OPERAND (fn
, 0);
728 if (d
->block_flags
& DIAG_TM_SAFE
)
731 error_at (gimple_location (stmt
),
732 "unsafe function call %qD within "
733 "atomic transaction", fn
);
736 if (!DECL_P (fn
) || DECL_NAME (fn
))
737 error_at (gimple_location (stmt
),
738 "unsafe function call %qE within "
739 "atomic transaction", fn
);
741 error_at (gimple_location (stmt
),
742 "unsafe indirect function call within "
743 "atomic transaction");
749 error_at (gimple_location (stmt
),
750 "unsafe function call %qD within "
751 "%<transaction_safe%> function", fn
);
754 if (!DECL_P (fn
) || DECL_NAME (fn
))
755 error_at (gimple_location (stmt
),
756 "unsafe function call %qE within "
757 "%<transaction_safe%> function", fn
);
759 error_at (gimple_location (stmt
),
760 "unsafe indirect function call within "
761 "%<transaction_safe%> function");
770 /* ??? We ought to come up with a way to add attributes to
771 asm statements, and then add "transaction_safe" to it.
772 Either that or get the language spec to resurrect __tm_waiver. */
773 if (d
->block_flags
& DIAG_TM_SAFE
)
774 error_at (gimple_location (stmt
),
775 "asm not allowed in atomic transaction");
776 else if (d
->func_flags
& DIAG_TM_SAFE
)
777 error_at (gimple_location (stmt
),
778 "asm not allowed in %<transaction_safe%> function");
781 case GIMPLE_TRANSACTION
:
783 gtransaction
*trans_stmt
= as_a
<gtransaction
*> (stmt
);
784 unsigned char inner_flags
= DIAG_TM_SAFE
;
786 if (gimple_transaction_subcode (trans_stmt
) & GTMA_IS_RELAXED
)
788 if (d
->block_flags
& DIAG_TM_SAFE
)
789 error_at (gimple_location (stmt
),
790 "relaxed transaction in atomic transaction");
791 else if (d
->func_flags
& DIAG_TM_SAFE
)
792 error_at (gimple_location (stmt
),
793 "relaxed transaction in %<transaction_safe%> function");
794 inner_flags
= DIAG_TM_RELAXED
;
796 else if (gimple_transaction_subcode (trans_stmt
) & GTMA_IS_OUTER
)
799 error_at (gimple_location (stmt
),
800 "outer transaction in transaction");
801 else if (d
->func_flags
& DIAG_TM_OUTER
)
802 error_at (gimple_location (stmt
),
803 "outer transaction in "
804 "%<transaction_may_cancel_outer%> function");
805 else if (d
->func_flags
& DIAG_TM_SAFE
)
806 error_at (gimple_location (stmt
),
807 "outer transaction in %<transaction_safe%> function");
808 inner_flags
|= DIAG_TM_OUTER
;
811 *handled_ops_p
= true;
812 if (gimple_transaction_body (trans_stmt
))
814 struct walk_stmt_info wi_inner
;
815 struct diagnose_tm d_inner
;
817 memset (&d_inner
, 0, sizeof (d_inner
));
818 d_inner
.func_flags
= d
->func_flags
;
819 d_inner
.block_flags
= d
->block_flags
| inner_flags
;
820 d_inner
.summary_flags
= d_inner
.func_flags
| d_inner
.block_flags
;
822 memset (&wi_inner
, 0, sizeof (wi_inner
));
823 wi_inner
.info
= &d_inner
;
825 walk_gimple_seq (gimple_transaction_body (trans_stmt
),
826 diagnose_tm_1
, diagnose_tm_1_op
, &wi_inner
);
839 diagnose_tm_blocks (void)
841 struct walk_stmt_info wi
;
842 struct diagnose_tm d
;
844 memset (&d
, 0, sizeof (d
));
845 if (is_tm_may_cancel_outer (current_function_decl
))
846 d
.func_flags
= DIAG_TM_OUTER
| DIAG_TM_SAFE
;
847 else if (is_tm_safe (current_function_decl
))
848 d
.func_flags
= DIAG_TM_SAFE
;
849 d
.summary_flags
= d
.func_flags
;
851 memset (&wi
, 0, sizeof (wi
));
854 walk_gimple_seq (gimple_body (current_function_decl
),
855 diagnose_tm_1
, diagnose_tm_1_op
, &wi
);
862 const pass_data pass_data_diagnose_tm_blocks
=
864 GIMPLE_PASS
, /* type */
865 "*diagnose_tm_blocks", /* name */
866 OPTGROUP_NONE
, /* optinfo_flags */
867 TV_TRANS_MEM
, /* tv_id */
868 PROP_gimple_any
, /* properties_required */
869 0, /* properties_provided */
870 0, /* properties_destroyed */
871 0, /* todo_flags_start */
872 0, /* todo_flags_finish */
875 class pass_diagnose_tm_blocks
: public gimple_opt_pass
878 pass_diagnose_tm_blocks (gcc::context
*ctxt
)
879 : gimple_opt_pass (pass_data_diagnose_tm_blocks
, ctxt
)
882 /* opt_pass methods: */
883 virtual bool gate (function
*) { return flag_tm
; }
884 virtual unsigned int execute (function
*) { return diagnose_tm_blocks (); }
886 }; // class pass_diagnose_tm_blocks
891 make_pass_diagnose_tm_blocks (gcc::context
*ctxt
)
893 return new pass_diagnose_tm_blocks (ctxt
);
896 /* Instead of instrumenting thread private memory, we save the
897 addresses in a log which we later use to save/restore the addresses
898 upon transaction start/restart.
900 The log is keyed by address, where each element contains individual
901 statements among different code paths that perform the store.
903 This log is later used to generate either plain save/restore of the
904 addresses upon transaction start/restart, or calls to the ITM_L*
907 So for something like:
909 struct large { int x[1000]; };
910 struct large lala = { 0 };
916 We can either save/restore:
919 trxn = _ITM_startTransaction ();
920 if (trxn & a_saveLiveVariables)
921 tmp_lala1 = lala.x[i];
922 else if (a & a_restoreLiveVariables)
923 lala.x[i] = tmp_lala1;
925 or use the logging functions:
928 trxn = _ITM_startTransaction ();
929 _ITM_LU4 (&lala.x[i]);
931 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as
932 far up the dominator tree to shadow all of the writes to a given
933 location (thus reducing the total number of logging calls), but not
934 so high as to be called on a path that does not perform a
937 /* One individual log entry. We may have multiple statements for the
938 same location if neither dominate each other (on different
940 typedef struct tm_log_entry
942 /* Address to save. */
944 /* Entry block for the transaction this address occurs in. */
945 basic_block entry_block
;
946 /* Dominating statements the store occurs in. */
948 /* Initially, while we are building the log, we place a nonzero
949 value here to mean that this address *will* be saved with a
950 save/restore sequence. Later, when generating the save sequence
951 we place the SSA temp generated here. */
956 /* Log entry hashtable helpers. */
958 struct log_entry_hasher
960 typedef tm_log_entry
*value_type
;
961 typedef tm_log_entry
*compare_type
;
962 static inline hashval_t
hash (const tm_log_entry
*);
963 static inline bool equal (const tm_log_entry
*, const tm_log_entry
*);
964 static inline void remove (tm_log_entry
*);
967 /* Htab support. Return hash value for a `tm_log_entry'. */
969 log_entry_hasher::hash (const tm_log_entry
*log
)
971 return iterative_hash_expr (log
->addr
, 0);
974 /* Htab support. Return true if two log entries are the same. */
976 log_entry_hasher::equal (const tm_log_entry
*log1
, const tm_log_entry
*log2
)
980 rth: I suggest that we get rid of the component refs etc.
981 I.e. resolve the reference to base + offset.
983 We may need to actually finish a merge with mainline for this,
984 since we'd like to be presented with Richi's MEM_REF_EXPRs more
985 often than not. But in the meantime your tm_log_entry could save
986 the results of get_inner_reference.
988 See: g++.dg/tm/pr46653.C
991 /* Special case plain equality because operand_equal_p() below will
992 return FALSE if the addresses are equal but they have
993 side-effects (e.g. a volatile address). */
994 if (log1
->addr
== log2
->addr
)
997 return operand_equal_p (log1
->addr
, log2
->addr
, 0);
1000 /* Htab support. Free one tm_log_entry. */
1002 log_entry_hasher::remove (tm_log_entry
*lp
)
1004 lp
->stmts
.release ();
1009 /* The actual log. */
1010 static hash_table
<log_entry_hasher
> *tm_log
;
1012 /* Addresses to log with a save/restore sequence. These should be in
1014 static vec
<tree
> tm_log_save_addresses
;
1016 enum thread_memory_type
1020 mem_transaction_local
,
1024 typedef struct tm_new_mem_map
1026 /* SSA_NAME being dereferenced. */
1028 enum thread_memory_type local_new_memory
;
1031 /* Hashtable helpers. */
1033 struct tm_mem_map_hasher
: free_ptr_hash
<tm_new_mem_map_t
>
1035 static inline hashval_t
hash (const tm_new_mem_map_t
*);
1036 static inline bool equal (const tm_new_mem_map_t
*, const tm_new_mem_map_t
*);
1040 tm_mem_map_hasher::hash (const tm_new_mem_map_t
*v
)
1042 return (intptr_t)v
->val
>> 4;
1046 tm_mem_map_hasher::equal (const tm_new_mem_map_t
*v
, const tm_new_mem_map_t
*c
)
1048 return v
->val
== c
->val
;
1051 /* Map for an SSA_NAME originally pointing to a non aliased new piece
1052 of memory (malloc, alloc, etc). */
1053 static hash_table
<tm_mem_map_hasher
> *tm_new_mem_hash
;
1055 /* Initialize logging data structures. */
1059 tm_log
= new hash_table
<log_entry_hasher
> (10);
1060 tm_new_mem_hash
= new hash_table
<tm_mem_map_hasher
> (5);
1061 tm_log_save_addresses
.create (5);
1064 /* Free logging data structures. */
1066 tm_log_delete (void)
1070 delete tm_new_mem_hash
;
1071 tm_new_mem_hash
= NULL
;
1072 tm_log_save_addresses
.release ();
1075 /* Return true if MEM is a transaction invariant memory for the TM
1076 region starting at REGION_ENTRY_BLOCK. */
1078 transaction_invariant_address_p (const_tree mem
, basic_block region_entry_block
)
1080 if ((TREE_CODE (mem
) == INDIRECT_REF
|| TREE_CODE (mem
) == MEM_REF
)
1081 && TREE_CODE (TREE_OPERAND (mem
, 0)) == SSA_NAME
)
1085 def_bb
= gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem
, 0)));
1086 return def_bb
!= region_entry_block
1087 && dominated_by_p (CDI_DOMINATORS
, region_entry_block
, def_bb
);
1090 mem
= strip_invariant_refs (mem
);
1091 return mem
&& (CONSTANT_CLASS_P (mem
) || decl_address_invariant_p (mem
));
1094 /* Given an address ADDR in STMT, find it in the memory log or add it,
1095 making sure to keep only the addresses highest in the dominator
1098 ENTRY_BLOCK is the entry_block for the transaction.
1100 If we find the address in the log, make sure it's either the same
1101 address, or an equivalent one that dominates ADDR.
1103 If we find the address, but neither ADDR dominates the found
1104 address, nor the found one dominates ADDR, we're on different
1105 execution paths. Add it.
1107 If known, ENTRY_BLOCK is the entry block for the region, otherwise
1110 tm_log_add (basic_block entry_block
, tree addr
, gimple stmt
)
1112 tm_log_entry
**slot
;
1113 struct tm_log_entry l
, *lp
;
1116 slot
= tm_log
->find_slot (&l
, INSERT
);
1119 tree type
= TREE_TYPE (addr
);
1121 lp
= XNEW (struct tm_log_entry
);
1125 /* Small invariant addresses can be handled as save/restores. */
1127 && transaction_invariant_address_p (lp
->addr
, entry_block
)
1128 && TYPE_SIZE_UNIT (type
) != NULL
1129 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
))
1130 && ((HOST_WIDE_INT
) tree_to_uhwi (TYPE_SIZE_UNIT (type
))
1131 < PARAM_VALUE (PARAM_TM_MAX_AGGREGATE_SIZE
))
1132 /* We must be able to copy this type normally. I.e., no
1133 special constructors and the like. */
1134 && !TREE_ADDRESSABLE (type
))
1136 lp
->save_var
= create_tmp_reg (TREE_TYPE (lp
->addr
), "tm_save");
1137 lp
->stmts
.create (0);
1138 lp
->entry_block
= entry_block
;
1139 /* Save addresses separately in dominator order so we don't
1140 get confused by overlapping addresses in the save/restore
1142 tm_log_save_addresses
.safe_push (lp
->addr
);
1146 /* Use the logging functions. */
1147 lp
->stmts
.create (5);
1148 lp
->stmts
.quick_push (stmt
);
1149 lp
->save_var
= NULL
;
1159 /* If we're generating a save/restore sequence, we don't care
1160 about statements. */
1164 for (i
= 0; lp
->stmts
.iterate (i
, &oldstmt
); ++i
)
1166 if (stmt
== oldstmt
)
1168 /* We already have a store to the same address, higher up the
1169 dominator tree. Nothing to do. */
1170 if (dominated_by_p (CDI_DOMINATORS
,
1171 gimple_bb (stmt
), gimple_bb (oldstmt
)))
1173 /* We should be processing blocks in dominator tree order. */
1174 gcc_assert (!dominated_by_p (CDI_DOMINATORS
,
1175 gimple_bb (oldstmt
), gimple_bb (stmt
)));
1177 /* Store is on a different code path. */
1178 lp
->stmts
.safe_push (stmt
);
1182 /* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME
1183 result, insert the new statements before GSI. */
1186 gimplify_addr (gimple_stmt_iterator
*gsi
, tree x
)
1188 if (TREE_CODE (x
) == TARGET_MEM_REF
)
1189 x
= tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x
)), x
);
1191 x
= build_fold_addr_expr (x
);
1192 return force_gimple_operand_gsi (gsi
, x
, true, NULL
, true, GSI_SAME_STMT
);
1195 /* Instrument one address with the logging functions.
1196 ADDR is the address to save.
1197 STMT is the statement before which to place it. */
1199 tm_log_emit_stmt (tree addr
, gimple stmt
)
1201 tree type
= TREE_TYPE (addr
);
1202 tree size
= TYPE_SIZE_UNIT (type
);
1203 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
1205 enum built_in_function code
= BUILT_IN_TM_LOG
;
1207 if (type
== float_type_node
)
1208 code
= BUILT_IN_TM_LOG_FLOAT
;
1209 else if (type
== double_type_node
)
1210 code
= BUILT_IN_TM_LOG_DOUBLE
;
1211 else if (type
== long_double_type_node
)
1212 code
= BUILT_IN_TM_LOG_LDOUBLE
;
1213 else if (tree_fits_uhwi_p (size
))
1215 unsigned int n
= tree_to_uhwi (size
);
1219 code
= BUILT_IN_TM_LOG_1
;
1222 code
= BUILT_IN_TM_LOG_2
;
1225 code
= BUILT_IN_TM_LOG_4
;
1228 code
= BUILT_IN_TM_LOG_8
;
1231 code
= BUILT_IN_TM_LOG
;
1232 if (TREE_CODE (type
) == VECTOR_TYPE
)
1234 if (n
== 8 && builtin_decl_explicit (BUILT_IN_TM_LOG_M64
))
1235 code
= BUILT_IN_TM_LOG_M64
;
1236 else if (n
== 16 && builtin_decl_explicit (BUILT_IN_TM_LOG_M128
))
1237 code
= BUILT_IN_TM_LOG_M128
;
1238 else if (n
== 32 && builtin_decl_explicit (BUILT_IN_TM_LOG_M256
))
1239 code
= BUILT_IN_TM_LOG_M256
;
1245 addr
= gimplify_addr (&gsi
, addr
);
1246 if (code
== BUILT_IN_TM_LOG
)
1247 log
= gimple_build_call (builtin_decl_explicit (code
), 2, addr
, size
);
1249 log
= gimple_build_call (builtin_decl_explicit (code
), 1, addr
);
1250 gsi_insert_before (&gsi
, log
, GSI_SAME_STMT
);
1253 /* Go through the log and instrument address that must be instrumented
1254 with the logging functions. Leave the save/restore addresses for
1259 hash_table
<log_entry_hasher
>::iterator hi
;
1260 struct tm_log_entry
*lp
;
1262 FOR_EACH_HASH_TABLE_ELEMENT (*tm_log
, lp
, tm_log_entry_t
, hi
)
1269 fprintf (dump_file
, "TM thread private mem logging: ");
1270 print_generic_expr (dump_file
, lp
->addr
, 0);
1271 fprintf (dump_file
, "\n");
1277 fprintf (dump_file
, "DUMPING to variable\n");
1283 fprintf (dump_file
, "DUMPING with logging functions\n");
1284 for (i
= 0; lp
->stmts
.iterate (i
, &stmt
); ++i
)
1285 tm_log_emit_stmt (lp
->addr
, stmt
);
1290 /* Emit the save sequence for the corresponding addresses in the log.
1291 ENTRY_BLOCK is the entry block for the transaction.
1292 BB is the basic block to insert the code in. */
1294 tm_log_emit_saves (basic_block entry_block
, basic_block bb
)
1297 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1299 struct tm_log_entry l
, *lp
;
1301 for (i
= 0; i
< tm_log_save_addresses
.length (); ++i
)
1303 l
.addr
= tm_log_save_addresses
[i
];
1304 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1305 gcc_assert (lp
->save_var
!= NULL
);
1307 /* We only care about variables in the current transaction. */
1308 if (lp
->entry_block
!= entry_block
)
1311 stmt
= gimple_build_assign (lp
->save_var
, unshare_expr (lp
->addr
));
1313 /* Make sure we can create an SSA_NAME for this type. For
1314 instance, aggregates aren't allowed, in which case the system
1315 will create a VOP for us and everything will just work. */
1316 if (is_gimple_reg_type (TREE_TYPE (lp
->save_var
)))
1318 lp
->save_var
= make_ssa_name (lp
->save_var
, stmt
);
1319 gimple_assign_set_lhs (stmt
, lp
->save_var
);
1322 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
1326 /* Emit the restore sequence for the corresponding addresses in the log.
1327 ENTRY_BLOCK is the entry block for the transaction.
1328 BB is the basic block to insert the code in. */
1330 tm_log_emit_restores (basic_block entry_block
, basic_block bb
)
1333 struct tm_log_entry l
, *lp
;
1334 gimple_stmt_iterator gsi
;
1337 for (i
= tm_log_save_addresses
.length () - 1; i
>= 0; i
--)
1339 l
.addr
= tm_log_save_addresses
[i
];
1340 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1341 gcc_assert (lp
->save_var
!= NULL
);
1343 /* We only care about variables in the current transaction. */
1344 if (lp
->entry_block
!= entry_block
)
1347 /* Restores are in LIFO order from the saves in case we have
1349 gsi
= gsi_start_bb (bb
);
1351 stmt
= gimple_build_assign (unshare_expr (lp
->addr
), lp
->save_var
);
1352 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1357 static tree
lower_sequence_tm (gimple_stmt_iterator
*, bool *,
1358 struct walk_stmt_info
*);
1359 static tree
lower_sequence_no_tm (gimple_stmt_iterator
*, bool *,
1360 struct walk_stmt_info
*);
1362 /* Evaluate an address X being dereferenced and determine if it
1363 originally points to a non aliased new chunk of memory (malloc,
1366 Return MEM_THREAD_LOCAL if it points to a thread-local address.
1367 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address.
1368 Return MEM_NON_LOCAL otherwise.
1370 ENTRY_BLOCK is the entry block to the transaction containing the
1371 dereference of X. */
1372 static enum thread_memory_type
1373 thread_private_new_memory (basic_block entry_block
, tree x
)
1376 enum tree_code code
;
1377 tm_new_mem_map_t
**slot
;
1378 tm_new_mem_map_t elt
, *elt_p
;
1380 enum thread_memory_type retval
= mem_transaction_local
;
1383 || TREE_CODE (x
) != SSA_NAME
1384 /* Possible uninitialized use, or a function argument. In
1385 either case, we don't care. */
1386 || SSA_NAME_IS_DEFAULT_DEF (x
))
1387 return mem_non_local
;
1389 /* Look in cache first. */
1391 slot
= tm_new_mem_hash
->find_slot (&elt
, INSERT
);
1394 return elt_p
->local_new_memory
;
1396 /* Optimistically assume the memory is transaction local during
1397 processing. This catches recursion into this variable. */
1398 *slot
= elt_p
= XNEW (tm_new_mem_map_t
);
1400 elt_p
->local_new_memory
= mem_transaction_local
;
1402 /* Search DEF chain to find the original definition of this address. */
1405 if (ptr_deref_may_alias_global_p (x
))
1407 /* Address escapes. This is not thread-private. */
1408 retval
= mem_non_local
;
1409 goto new_memory_ret
;
1412 stmt
= SSA_NAME_DEF_STMT (x
);
1414 /* If the malloc call is outside the transaction, this is
1416 if (retval
!= mem_thread_local
1417 && !dominated_by_p (CDI_DOMINATORS
, gimple_bb (stmt
), entry_block
))
1418 retval
= mem_thread_local
;
1420 if (is_gimple_assign (stmt
))
1422 code
= gimple_assign_rhs_code (stmt
);
1423 /* x = foo ==> foo */
1424 if (code
== SSA_NAME
)
1425 x
= gimple_assign_rhs1 (stmt
);
1426 /* x = foo + n ==> foo */
1427 else if (code
== POINTER_PLUS_EXPR
)
1428 x
= gimple_assign_rhs1 (stmt
);
1429 /* x = (cast*) foo ==> foo */
1430 else if (code
== VIEW_CONVERT_EXPR
|| CONVERT_EXPR_CODE_P (code
))
1431 x
= gimple_assign_rhs1 (stmt
);
1432 /* x = c ? op1 : op2 == > op1 or op2 just like a PHI */
1433 else if (code
== COND_EXPR
)
1435 tree op1
= gimple_assign_rhs2 (stmt
);
1436 tree op2
= gimple_assign_rhs3 (stmt
);
1437 enum thread_memory_type mem
;
1438 retval
= thread_private_new_memory (entry_block
, op1
);
1439 if (retval
== mem_non_local
)
1440 goto new_memory_ret
;
1441 mem
= thread_private_new_memory (entry_block
, op2
);
1442 retval
= MIN (retval
, mem
);
1443 goto new_memory_ret
;
1447 retval
= mem_non_local
;
1448 goto new_memory_ret
;
1453 if (gimple_code (stmt
) == GIMPLE_PHI
)
1456 enum thread_memory_type mem
;
1457 tree phi_result
= gimple_phi_result (stmt
);
1459 /* If any of the ancestors are non-local, we are sure to
1460 be non-local. Otherwise we can avoid doing anything
1461 and inherit what has already been generated. */
1463 for (i
= 0; i
< gimple_phi_num_args (stmt
); ++i
)
1465 tree op
= PHI_ARG_DEF (stmt
, i
);
1467 /* Exclude self-assignment. */
1468 if (phi_result
== op
)
1471 mem
= thread_private_new_memory (entry_block
, op
);
1472 if (mem
== mem_non_local
)
1475 goto new_memory_ret
;
1477 retval
= MIN (retval
, mem
);
1479 goto new_memory_ret
;
1484 while (TREE_CODE (x
) == SSA_NAME
);
1486 if (stmt
&& is_gimple_call (stmt
) && gimple_call_flags (stmt
) & ECF_MALLOC
)
1487 /* Thread-local or transaction-local. */
1490 retval
= mem_non_local
;
1493 elt_p
->local_new_memory
= retval
;
1497 /* Determine whether X has to be instrumented using a read
1500 ENTRY_BLOCK is the entry block for the region where stmt resides
1501 in. NULL if unknown.
1503 STMT is the statement in which X occurs in. It is used for thread
1504 private memory instrumentation. If no TPM instrumentation is
1505 desired, STMT should be null. */
1507 requires_barrier (basic_block entry_block
, tree x
, gimple stmt
)
1510 while (handled_component_p (x
))
1511 x
= TREE_OPERAND (x
, 0);
1513 switch (TREE_CODE (x
))
1518 enum thread_memory_type ret
;
1520 ret
= thread_private_new_memory (entry_block
, TREE_OPERAND (x
, 0));
1521 if (ret
== mem_non_local
)
1523 if (stmt
&& ret
== mem_thread_local
)
1524 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */
1525 tm_log_add (entry_block
, orig
, stmt
);
1527 /* Transaction-locals require nothing at all. For malloc, a
1528 transaction restart frees the memory and we reallocate.
1529 For alloca, the stack pointer gets reset by the retry and
1534 case TARGET_MEM_REF
:
1535 if (TREE_CODE (TMR_BASE (x
)) != ADDR_EXPR
)
1537 x
= TREE_OPERAND (TMR_BASE (x
), 0);
1538 if (TREE_CODE (x
) == PARM_DECL
)
1540 gcc_assert (TREE_CODE (x
) == VAR_DECL
);
1546 if (DECL_BY_REFERENCE (x
))
1548 /* ??? This value is a pointer, but aggregate_value_p has been
1549 jigged to return true which confuses needs_to_live_in_memory.
1550 This ought to be cleaned up generically.
1552 FIXME: Verify this still happens after the next mainline
1553 merge. Testcase ie g++.dg/tm/pr47554.C.
1558 if (is_global_var (x
))
1559 return !TREE_READONLY (x
);
1560 if (/* FIXME: This condition should actually go below in the
1561 tm_log_add() call, however is_call_clobbered() depends on
1562 aliasing info which is not available during
1563 gimplification. Since requires_barrier() gets called
1564 during lower_sequence_tm/gimplification, leave the call
1565 to needs_to_live_in_memory until we eliminate
1566 lower_sequence_tm altogether. */
1567 needs_to_live_in_memory (x
))
1571 /* For local memory that doesn't escape (aka thread private
1572 memory), we can either save the value at the beginning of
1573 the transaction and restore on restart, or call a tm
1574 function to dynamically save and restore on restart
1577 tm_log_add (entry_block
, orig
, stmt
);
1586 /* Mark the GIMPLE_ASSIGN statement as appropriate for being inside
1587 a transaction region. */
1590 examine_assign_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1592 gimple stmt
= gsi_stmt (*gsi
);
1594 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_rhs1 (stmt
), NULL
))
1595 *state
|= GTMA_HAVE_LOAD
;
1596 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_lhs (stmt
), NULL
))
1597 *state
|= GTMA_HAVE_STORE
;
1600 /* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */
1603 examine_call_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1605 gimple stmt
= gsi_stmt (*gsi
);
1608 if (is_tm_pure_call (stmt
))
1611 /* Check if this call is a transaction abort. */
1612 fn
= gimple_call_fndecl (stmt
);
1613 if (is_tm_abort (fn
))
1614 *state
|= GTMA_HAVE_ABORT
;
1616 /* Note that something may happen. */
1617 *state
|= GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
;
1620 /* Lower a GIMPLE_TRANSACTION statement. */
1623 lower_transaction (gimple_stmt_iterator
*gsi
, struct walk_stmt_info
*wi
)
1626 gtransaction
*stmt
= as_a
<gtransaction
*> (gsi_stmt (*gsi
));
1627 unsigned int *outer_state
= (unsigned int *) wi
->info
;
1628 unsigned int this_state
= 0;
1629 struct walk_stmt_info this_wi
;
1631 /* First, lower the body. The scanning that we do inside gives
1632 us some idea of what we're dealing with. */
1633 memset (&this_wi
, 0, sizeof (this_wi
));
1634 this_wi
.info
= (void *) &this_state
;
1635 walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt
),
1636 lower_sequence_tm
, NULL
, &this_wi
);
1638 /* If there was absolutely nothing transaction related inside the
1639 transaction, we may elide it. Likewise if this is a nested
1640 transaction and does not contain an abort. */
1642 || (!(this_state
& GTMA_HAVE_ABORT
) && outer_state
!= NULL
))
1645 *outer_state
|= this_state
;
1647 gsi_insert_seq_before (gsi
, gimple_transaction_body (stmt
),
1649 gimple_transaction_set_body (stmt
, NULL
);
1651 gsi_remove (gsi
, true);
1652 wi
->removed_stmt
= true;
1656 /* Wrap the body of the transaction in a try-finally node so that
1657 the commit call is always properly called. */
1658 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT
), 0);
1659 if (flag_exceptions
)
1662 gimple_seq n_seq
, e_seq
;
1664 n_seq
= gimple_seq_alloc_with_stmt (g
);
1667 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER
),
1668 1, integer_zero_node
);
1669 ptr
= create_tmp_var (ptr_type_node
);
1670 gimple_call_set_lhs (g
, ptr
);
1671 gimple_seq_add_stmt (&e_seq
, g
);
1673 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH
),
1675 gimple_seq_add_stmt (&e_seq
, g
);
1677 g
= gimple_build_eh_else (n_seq
, e_seq
);
1680 g
= gimple_build_try (gimple_transaction_body (stmt
),
1681 gimple_seq_alloc_with_stmt (g
), GIMPLE_TRY_FINALLY
);
1682 gsi_insert_after (gsi
, g
, GSI_CONTINUE_LINKING
);
1684 gimple_transaction_set_body (stmt
, NULL
);
1686 /* If the transaction calls abort or if this is an outer transaction,
1687 add an "over" label afterwards. */
1688 if ((this_state
& (GTMA_HAVE_ABORT
))
1689 || (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
))
1691 tree label
= create_artificial_label (UNKNOWN_LOCATION
);
1692 gimple_transaction_set_label (stmt
, label
);
1693 gsi_insert_after (gsi
, gimple_build_label (label
), GSI_CONTINUE_LINKING
);
1696 /* Record the set of operations found for use later. */
1697 this_state
|= gimple_transaction_subcode (stmt
) & GTMA_DECLARATION_MASK
;
1698 gimple_transaction_set_subcode (stmt
, this_state
);
1701 /* Iterate through the statements in the sequence, lowering them all
1702 as appropriate for being in a transaction. */
1705 lower_sequence_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1706 struct walk_stmt_info
*wi
)
1708 unsigned int *state
= (unsigned int *) wi
->info
;
1709 gimple stmt
= gsi_stmt (*gsi
);
1711 *handled_ops_p
= true;
1712 switch (gimple_code (stmt
))
1715 /* Only memory reads/writes need to be instrumented. */
1716 if (gimple_assign_single_p (stmt
))
1717 examine_assign_tm (state
, gsi
);
1721 examine_call_tm (state
, gsi
);
1725 *state
|= GTMA_MAY_ENTER_IRREVOCABLE
;
1728 case GIMPLE_TRANSACTION
:
1729 lower_transaction (gsi
, wi
);
1733 *handled_ops_p
= !gimple_has_substatements (stmt
);
1740 /* Iterate through the statements in the sequence, lowering them all
1741 as appropriate for being outside of a transaction. */
1744 lower_sequence_no_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1745 struct walk_stmt_info
* wi
)
1747 gimple stmt
= gsi_stmt (*gsi
);
1749 if (gimple_code (stmt
) == GIMPLE_TRANSACTION
)
1751 *handled_ops_p
= true;
1752 lower_transaction (gsi
, wi
);
1755 *handled_ops_p
= !gimple_has_substatements (stmt
);
1760 /* Main entry point for flattening GIMPLE_TRANSACTION constructs. After
1761 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has
1762 been moved out, and all the data required for constructing a proper
1763 CFG has been recorded. */
1766 execute_lower_tm (void)
1768 struct walk_stmt_info wi
;
1771 /* Transactional clones aren't created until a later pass. */
1772 gcc_assert (!decl_is_tm_clone (current_function_decl
));
1774 body
= gimple_body (current_function_decl
);
1775 memset (&wi
, 0, sizeof (wi
));
1776 walk_gimple_seq_mod (&body
, lower_sequence_no_tm
, NULL
, &wi
);
1777 gimple_set_body (current_function_decl
, body
);
1784 const pass_data pass_data_lower_tm
=
1786 GIMPLE_PASS
, /* type */
1787 "tmlower", /* name */
1788 OPTGROUP_NONE
, /* optinfo_flags */
1789 TV_TRANS_MEM
, /* tv_id */
1790 PROP_gimple_lcf
, /* properties_required */
1791 0, /* properties_provided */
1792 0, /* properties_destroyed */
1793 0, /* todo_flags_start */
1794 0, /* todo_flags_finish */
1797 class pass_lower_tm
: public gimple_opt_pass
1800 pass_lower_tm (gcc::context
*ctxt
)
1801 : gimple_opt_pass (pass_data_lower_tm
, ctxt
)
1804 /* opt_pass methods: */
1805 virtual bool gate (function
*) { return flag_tm
; }
1806 virtual unsigned int execute (function
*) { return execute_lower_tm (); }
1808 }; // class pass_lower_tm
1813 make_pass_lower_tm (gcc::context
*ctxt
)
1815 return new pass_lower_tm (ctxt
);
1818 /* Collect region information for each transaction. */
1824 /* The field "transaction_stmt" is initially a gtransaction *,
1825 but eventually gets lowered to a gcall *(to BUILT_IN_TM_START).
1827 Helper method to get it as a gtransaction *, with code-checking
1828 in a checked-build. */
1831 get_transaction_stmt () const
1833 return as_a
<gtransaction
*> (transaction_stmt
);
1838 /* Link to the next unnested transaction. */
1839 struct tm_region
*next
;
1841 /* Link to the next inner transaction. */
1842 struct tm_region
*inner
;
1844 /* Link to the next outer transaction. */
1845 struct tm_region
*outer
;
1847 /* The GIMPLE_TRANSACTION statement beginning this transaction.
1848 After TM_MARK, this gets replaced by a call to
1850 Hence this will be either a gtransaction *or a gcall *. */
1851 gimple transaction_stmt
;
1853 /* After TM_MARK expands the GIMPLE_TRANSACTION into a call to
1854 BUILT_IN_TM_START, this field is true if the transaction is an
1855 outer transaction. */
1856 bool original_transaction_was_outer
;
1858 /* Return value from BUILT_IN_TM_START. */
1861 /* The entry block to this region. This will always be the first
1862 block of the body of the transaction. */
1863 basic_block entry_block
;
1865 /* The first block after an expanded call to _ITM_beginTransaction. */
1866 basic_block restart_block
;
1868 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK.
1869 These blocks are still a part of the region (i.e., the border is
1870 inclusive). Note that this set is only complete for paths in the CFG
1871 starting at ENTRY_BLOCK, and that there is no exit block recorded for
1872 the edge to the "over" label. */
1875 /* The set of all blocks that have an TM_IRREVOCABLE call. */
1879 typedef struct tm_region
*tm_region_p
;
1881 /* True if there are pending edge statements to be committed for the
1882 current function being scanned in the tmmark pass. */
1883 bool pending_edge_inserts_p
;
1885 static struct tm_region
*all_tm_regions
;
1886 static bitmap_obstack tm_obstack
;
1889 /* A subroutine of tm_region_init. Record the existence of the
1890 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */
1892 static struct tm_region
*
1893 tm_region_init_0 (struct tm_region
*outer
, basic_block bb
,
1896 struct tm_region
*region
;
1898 region
= (struct tm_region
*)
1899 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1903 region
->next
= outer
->inner
;
1904 outer
->inner
= region
;
1908 region
->next
= all_tm_regions
;
1909 all_tm_regions
= region
;
1911 region
->inner
= NULL
;
1912 region
->outer
= outer
;
1914 region
->transaction_stmt
= stmt
;
1915 region
->original_transaction_was_outer
= false;
1916 region
->tm_state
= NULL
;
1918 /* There are either one or two edges out of the block containing
1919 the GIMPLE_TRANSACTION, one to the actual region and one to the
1920 "over" label if the region contains an abort. The former will
1921 always be the one marked FALLTHRU. */
1922 region
->entry_block
= FALLTHRU_EDGE (bb
)->dest
;
1924 region
->exit_blocks
= BITMAP_ALLOC (&tm_obstack
);
1925 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1930 /* A subroutine of tm_region_init. Record all the exit and
1931 irrevocable blocks in BB into the region's exit_blocks and
1932 irr_blocks bitmaps. Returns the new region being scanned. */
1934 static struct tm_region
*
1935 tm_region_init_1 (struct tm_region
*region
, basic_block bb
)
1937 gimple_stmt_iterator gsi
;
1941 || (!region
->irr_blocks
&& !region
->exit_blocks
))
1944 /* Check to see if this is the end of a region by seeing if it
1945 contains a call to __builtin_tm_commit{,_eh}. Note that the
1946 outermost region for DECL_IS_TM_CLONE need not collect this. */
1947 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
1950 if (gimple_code (g
) == GIMPLE_CALL
)
1952 tree fn
= gimple_call_fndecl (g
);
1953 if (fn
&& DECL_BUILT_IN_CLASS (fn
) == BUILT_IN_NORMAL
)
1955 if ((DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT
1956 || DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT_EH
)
1957 && region
->exit_blocks
)
1959 bitmap_set_bit (region
->exit_blocks
, bb
->index
);
1960 region
= region
->outer
;
1963 if (DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_IRREVOCABLE
)
1964 bitmap_set_bit (region
->irr_blocks
, bb
->index
);
1971 /* Collect all of the transaction regions within the current function
1972 and record them in ALL_TM_REGIONS. The REGION parameter may specify
1973 an "outermost" region for use by tm clones. */
1976 tm_region_init (struct tm_region
*region
)
1982 auto_vec
<basic_block
> queue
;
1983 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
1984 struct tm_region
*old_region
;
1985 auto_vec
<tm_region_p
> bb_regions
;
1987 all_tm_regions
= region
;
1988 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1990 /* We could store this information in bb->aux, but we may get called
1991 through get_all_tm_blocks() from another pass that may be already
1993 bb_regions
.safe_grow_cleared (last_basic_block_for_fn (cfun
));
1995 queue
.safe_push (bb
);
1996 bb_regions
[bb
->index
] = region
;
2000 region
= bb_regions
[bb
->index
];
2001 bb_regions
[bb
->index
] = NULL
;
2003 /* Record exit and irrevocable blocks. */
2004 region
= tm_region_init_1 (region
, bb
);
2006 /* Check for the last statement in the block beginning a new region. */
2008 old_region
= region
;
2010 if (gtransaction
*trans_stmt
= dyn_cast
<gtransaction
*> (g
))
2011 region
= tm_region_init_0 (region
, bb
, trans_stmt
);
2013 /* Process subsequent blocks. */
2014 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2015 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2017 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2018 queue
.safe_push (e
->dest
);
2020 /* If the current block started a new region, make sure that only
2021 the entry block of the new region is associated with this region.
2022 Other successors are still part of the old region. */
2023 if (old_region
!= region
&& e
->dest
!= region
->entry_block
)
2024 bb_regions
[e
->dest
->index
] = old_region
;
2026 bb_regions
[e
->dest
->index
] = region
;
2029 while (!queue
.is_empty ());
2030 BITMAP_FREE (visited_blocks
);
2033 /* The "gate" function for all transactional memory expansion and optimization
2034 passes. We collect region information for each top-level transaction, and
2035 if we don't find any, we skip all of the TM passes. Each region will have
2036 all of the exit blocks recorded, and the originating statement. */
2044 calculate_dominance_info (CDI_DOMINATORS
);
2045 bitmap_obstack_initialize (&tm_obstack
);
2047 /* If the function is a TM_CLONE, then the entire function is the region. */
2048 if (decl_is_tm_clone (current_function_decl
))
2050 struct tm_region
*region
= (struct tm_region
*)
2051 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
2052 memset (region
, 0, sizeof (*region
));
2053 region
->entry_block
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
2054 /* For a clone, the entire function is the region. But even if
2055 we don't need to record any exit blocks, we may need to
2056 record irrevocable blocks. */
2057 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
2059 tm_region_init (region
);
2063 tm_region_init (NULL
);
2065 /* If we didn't find any regions, cleanup and skip the whole tree
2066 of tm-related optimizations. */
2067 if (all_tm_regions
== NULL
)
2069 bitmap_obstack_release (&tm_obstack
);
2079 const pass_data pass_data_tm_init
=
2081 GIMPLE_PASS
, /* type */
2082 "*tminit", /* name */
2083 OPTGROUP_NONE
, /* optinfo_flags */
2084 TV_TRANS_MEM
, /* tv_id */
2085 ( PROP_ssa
| PROP_cfg
), /* properties_required */
2086 0, /* properties_provided */
2087 0, /* properties_destroyed */
2088 0, /* todo_flags_start */
2089 0, /* todo_flags_finish */
2092 class pass_tm_init
: public gimple_opt_pass
2095 pass_tm_init (gcc::context
*ctxt
)
2096 : gimple_opt_pass (pass_data_tm_init
, ctxt
)
2099 /* opt_pass methods: */
2100 virtual bool gate (function
*) { return gate_tm_init (); }
2102 }; // class pass_tm_init
2107 make_pass_tm_init (gcc::context
*ctxt
)
2109 return new pass_tm_init (ctxt
);
2112 /* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region
2113 represented by STATE. */
2116 transaction_subcode_ior (struct tm_region
*region
, unsigned flags
)
2118 if (region
&& region
->transaction_stmt
)
2120 gtransaction
*transaction_stmt
= region
->get_transaction_stmt ();
2121 flags
|= gimple_transaction_subcode (transaction_stmt
);
2122 gimple_transaction_set_subcode (transaction_stmt
, flags
);
2126 /* Construct a memory load in a transactional context. Return the
2127 gimple statement performing the load, or NULL if there is no
2128 TM_LOAD builtin of the appropriate size to do the load.
2130 LOC is the location to use for the new statement(s). */
2133 build_tm_load (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2135 enum built_in_function code
= END_BUILTINS
;
2136 tree t
, type
= TREE_TYPE (rhs
), decl
;
2139 if (type
== float_type_node
)
2140 code
= BUILT_IN_TM_LOAD_FLOAT
;
2141 else if (type
== double_type_node
)
2142 code
= BUILT_IN_TM_LOAD_DOUBLE
;
2143 else if (type
== long_double_type_node
)
2144 code
= BUILT_IN_TM_LOAD_LDOUBLE
;
2145 else if (TYPE_SIZE_UNIT (type
) != NULL
2146 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2148 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2151 code
= BUILT_IN_TM_LOAD_1
;
2154 code
= BUILT_IN_TM_LOAD_2
;
2157 code
= BUILT_IN_TM_LOAD_4
;
2160 code
= BUILT_IN_TM_LOAD_8
;
2165 if (code
== END_BUILTINS
)
2167 decl
= targetm
.vectorize
.builtin_tm_load (type
);
2172 decl
= builtin_decl_explicit (code
);
2174 t
= gimplify_addr (gsi
, rhs
);
2175 gcall
= gimple_build_call (decl
, 1, t
);
2176 gimple_set_location (gcall
, loc
);
2178 t
= TREE_TYPE (TREE_TYPE (decl
));
2179 if (useless_type_conversion_p (type
, t
))
2181 gimple_call_set_lhs (gcall
, lhs
);
2182 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2189 temp
= create_tmp_reg (t
);
2190 gimple_call_set_lhs (gcall
, temp
);
2191 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2193 t
= fold_build1 (VIEW_CONVERT_EXPR
, type
, temp
);
2194 g
= gimple_build_assign (lhs
, t
);
2195 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2202 /* Similarly for storing TYPE in a transactional context. */
2205 build_tm_store (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2207 enum built_in_function code
= END_BUILTINS
;
2208 tree t
, fn
, type
= TREE_TYPE (rhs
), simple_type
;
2211 if (type
== float_type_node
)
2212 code
= BUILT_IN_TM_STORE_FLOAT
;
2213 else if (type
== double_type_node
)
2214 code
= BUILT_IN_TM_STORE_DOUBLE
;
2215 else if (type
== long_double_type_node
)
2216 code
= BUILT_IN_TM_STORE_LDOUBLE
;
2217 else if (TYPE_SIZE_UNIT (type
) != NULL
2218 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2220 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2223 code
= BUILT_IN_TM_STORE_1
;
2226 code
= BUILT_IN_TM_STORE_2
;
2229 code
= BUILT_IN_TM_STORE_4
;
2232 code
= BUILT_IN_TM_STORE_8
;
2237 if (code
== END_BUILTINS
)
2239 fn
= targetm
.vectorize
.builtin_tm_store (type
);
2244 fn
= builtin_decl_explicit (code
);
2246 simple_type
= TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn
))));
2248 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
2250 /* Handle the easy initialization to zero. */
2251 if (!CONSTRUCTOR_ELTS (rhs
))
2252 rhs
= build_int_cst (simple_type
, 0);
2255 /* ...otherwise punt to the caller and probably use
2256 BUILT_IN_TM_MEMMOVE, because we can't wrap a
2257 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce
2262 else if (!useless_type_conversion_p (simple_type
, type
))
2267 temp
= create_tmp_reg (simple_type
);
2268 t
= fold_build1 (VIEW_CONVERT_EXPR
, simple_type
, rhs
);
2269 g
= gimple_build_assign (temp
, t
);
2270 gimple_set_location (g
, loc
);
2271 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2276 t
= gimplify_addr (gsi
, lhs
);
2277 gcall
= gimple_build_call (fn
, 2, t
, rhs
);
2278 gimple_set_location (gcall
, loc
);
2279 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2285 /* Expand an assignment statement into transactional builtins. */
2288 expand_assign_tm (struct tm_region
*region
, gimple_stmt_iterator
*gsi
)
2290 gimple stmt
= gsi_stmt (*gsi
);
2291 location_t loc
= gimple_location (stmt
);
2292 tree lhs
= gimple_assign_lhs (stmt
);
2293 tree rhs
= gimple_assign_rhs1 (stmt
);
2294 bool store_p
= requires_barrier (region
->entry_block
, lhs
, NULL
);
2295 bool load_p
= requires_barrier (region
->entry_block
, rhs
, NULL
);
2296 gimple gcall
= NULL
;
2298 if (!load_p
&& !store_p
)
2300 /* Add thread private addresses to log if applicable. */
2301 requires_barrier (region
->entry_block
, lhs
, stmt
);
2306 // Remove original load/store statement.
2307 gsi_remove (gsi
, true);
2309 if (load_p
&& !store_p
)
2311 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2312 gcall
= build_tm_load (loc
, lhs
, rhs
, gsi
);
2314 else if (store_p
&& !load_p
)
2316 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2317 gcall
= build_tm_store (loc
, lhs
, rhs
, gsi
);
2321 tree lhs_addr
, rhs_addr
, tmp
;
2324 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2326 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2328 /* ??? Figure out if there's any possible overlap between the LHS
2329 and the RHS and if not, use MEMCPY. */
2331 if (load_p
&& is_gimple_reg (lhs
))
2333 tmp
= create_tmp_var (TREE_TYPE (lhs
));
2334 lhs_addr
= build_fold_addr_expr (tmp
);
2339 lhs_addr
= gimplify_addr (gsi
, lhs
);
2341 rhs_addr
= gimplify_addr (gsi
, rhs
);
2342 gcall
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
),
2343 3, lhs_addr
, rhs_addr
,
2344 TYPE_SIZE_UNIT (TREE_TYPE (lhs
)));
2345 gimple_set_location (gcall
, loc
);
2346 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2350 gcall
= gimple_build_assign (lhs
, tmp
);
2351 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2355 /* Now that we have the load/store in its instrumented form, add
2356 thread private addresses to the log if applicable. */
2358 requires_barrier (region
->entry_block
, lhs
, gcall
);
2360 // The calls to build_tm_{store,load} above inserted the instrumented
2361 // call into the stream.
2362 // gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2366 /* Expand a call statement as appropriate for a transaction. That is,
2367 either verify that the call does not affect the transaction, or
2368 redirect the call to a clone that handles transactions, or change
2369 the transaction state to IRREVOCABLE. Return true if the call is
2370 one of the builtins that end a transaction. */
2373 expand_call_tm (struct tm_region
*region
,
2374 gimple_stmt_iterator
*gsi
)
2376 gcall
*stmt
= as_a
<gcall
*> (gsi_stmt (*gsi
));
2377 tree lhs
= gimple_call_lhs (stmt
);
2379 struct cgraph_node
*node
;
2380 bool retval
= false;
2382 fn_decl
= gimple_call_fndecl (stmt
);
2384 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMCPY
)
2385 || fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
))
2386 transaction_subcode_ior (region
, GTMA_HAVE_STORE
| GTMA_HAVE_LOAD
);
2387 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMSET
))
2388 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2390 if (is_tm_pure_call (stmt
))
2394 retval
= is_tm_ending_fndecl (fn_decl
);
2397 /* Assume all non-const/pure calls write to memory, except
2398 transaction ending builtins. */
2399 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2402 /* For indirect calls, we already generated a call into the runtime. */
2405 tree fn
= gimple_call_fn (stmt
);
2407 /* We are guaranteed never to go irrevocable on a safe or pure
2408 call, and the pure call was handled above. */
2409 if (is_tm_safe (fn
))
2412 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2417 node
= cgraph_node::get (fn_decl
);
2418 /* All calls should have cgraph here. */
2421 /* We can have a nodeless call here if some pass after IPA-tm
2422 added uninstrumented calls. For example, loop distribution
2423 can transform certain loop constructs into __builtin_mem*
2424 calls. In this case, see if we have a suitable TM
2425 replacement and fill in the gaps. */
2426 gcc_assert (DECL_BUILT_IN_CLASS (fn_decl
) == BUILT_IN_NORMAL
);
2427 enum built_in_function code
= DECL_FUNCTION_CODE (fn_decl
);
2428 gcc_assert (code
== BUILT_IN_MEMCPY
2429 || code
== BUILT_IN_MEMMOVE
2430 || code
== BUILT_IN_MEMSET
);
2432 tree repl
= find_tm_replacement_function (fn_decl
);
2435 gimple_call_set_fndecl (stmt
, repl
);
2437 node
= cgraph_node::create (repl
);
2438 node
->local
.tm_may_enter_irr
= false;
2439 return expand_call_tm (region
, gsi
);
2443 if (node
->local
.tm_may_enter_irr
)
2444 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2446 if (is_tm_abort (fn_decl
))
2448 transaction_subcode_ior (region
, GTMA_HAVE_ABORT
);
2452 /* Instrument the store if needed.
2454 If the assignment happens inside the function call (return slot
2455 optimization), there is no instrumentation to be done, since
2456 the callee should have done the right thing. */
2457 if (lhs
&& requires_barrier (region
->entry_block
, lhs
, stmt
)
2458 && !gimple_call_return_slot_opt_p (stmt
))
2460 tree tmp
= create_tmp_reg (TREE_TYPE (lhs
));
2461 location_t loc
= gimple_location (stmt
);
2462 edge fallthru_edge
= NULL
;
2463 gassign
*assign_stmt
;
2465 /* Remember if the call was going to throw. */
2466 if (stmt_can_throw_internal (stmt
))
2470 basic_block bb
= gimple_bb (stmt
);
2472 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2473 if (e
->flags
& EDGE_FALLTHRU
)
2480 gimple_call_set_lhs (stmt
, tmp
);
2482 assign_stmt
= gimple_build_assign (lhs
, tmp
);
2483 gimple_set_location (assign_stmt
, loc
);
2485 /* We cannot throw in the middle of a BB. If the call was going
2486 to throw, place the instrumentation on the fallthru edge, so
2487 the call remains the last statement in the block. */
2490 gimple_seq fallthru_seq
= gimple_seq_alloc_with_stmt (assign_stmt
);
2491 gimple_stmt_iterator fallthru_gsi
= gsi_start (fallthru_seq
);
2492 expand_assign_tm (region
, &fallthru_gsi
);
2493 gsi_insert_seq_on_edge (fallthru_edge
, fallthru_seq
);
2494 pending_edge_inserts_p
= true;
2498 gsi_insert_after (gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
2499 expand_assign_tm (region
, gsi
);
2502 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2509 /* Expand all statements in BB as appropriate for being inside
2513 expand_block_tm (struct tm_region
*region
, basic_block bb
)
2515 gimple_stmt_iterator gsi
;
2517 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2519 gimple stmt
= gsi_stmt (gsi
);
2520 switch (gimple_code (stmt
))
2523 /* Only memory reads/writes need to be instrumented. */
2524 if (gimple_assign_single_p (stmt
)
2525 && !gimple_clobber_p (stmt
))
2527 expand_assign_tm (region
, &gsi
);
2533 if (expand_call_tm (region
, &gsi
))
2543 if (!gsi_end_p (gsi
))
2548 /* Return the list of basic-blocks in REGION.
2550 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks
2551 following a TM_IRREVOCABLE call.
2553 INCLUDE_UNINSTRUMENTED_P is TRUE if we should include the
2554 uninstrumented code path blocks in the list of basic blocks
2555 returned, false otherwise. */
2557 static vec
<basic_block
>
2558 get_tm_region_blocks (basic_block entry_block
,
2561 bitmap all_region_blocks
,
2562 bool stop_at_irrevocable_p
,
2563 bool include_uninstrumented_p
= true)
2565 vec
<basic_block
> bbs
= vNULL
;
2569 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
2572 bbs
.safe_push (entry_block
);
2573 bitmap_set_bit (visited_blocks
, entry_block
->index
);
2577 basic_block bb
= bbs
[i
++];
2580 bitmap_bit_p (exit_blocks
, bb
->index
))
2583 if (stop_at_irrevocable_p
2585 && bitmap_bit_p (irr_blocks
, bb
->index
))
2588 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2589 if ((include_uninstrumented_p
2590 || !(e
->flags
& EDGE_TM_UNINSTRUMENTED
))
2591 && !bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2593 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2594 bbs
.safe_push (e
->dest
);
2597 while (i
< bbs
.length ());
2599 if (all_region_blocks
)
2600 bitmap_ior_into (all_region_blocks
, visited_blocks
);
2602 BITMAP_FREE (visited_blocks
);
2606 // Callback data for collect_bb2reg.
2609 vec
<tm_region_p
> *bb2reg
;
2610 bool include_uninstrumented_p
;
2613 // Callback for expand_regions, collect innermost region data for each bb.
2615 collect_bb2reg (struct tm_region
*region
, void *data
)
2617 struct bb2reg_stuff
*stuff
= (struct bb2reg_stuff
*)data
;
2618 vec
<tm_region_p
> *bb2reg
= stuff
->bb2reg
;
2619 vec
<basic_block
> queue
;
2623 queue
= get_tm_region_blocks (region
->entry_block
,
2624 region
->exit_blocks
,
2627 /*stop_at_irr_p=*/true,
2628 stuff
->include_uninstrumented_p
);
2630 // We expect expand_region to perform a post-order traversal of the region
2631 // tree. Therefore the last region seen for any bb is the innermost.
2632 FOR_EACH_VEC_ELT (queue
, i
, bb
)
2633 (*bb2reg
)[bb
->index
] = region
;
2639 // Returns a vector, indexed by BB->INDEX, of the innermost tm_region to
2640 // which a basic block belongs. Note that we only consider the instrumented
2641 // code paths for the region; the uninstrumented code paths are ignored if
2642 // INCLUDE_UNINSTRUMENTED_P is false.
2644 // ??? This data is very similar to the bb_regions array that is collected
2645 // during tm_region_init. Or, rather, this data is similar to what could
2646 // be used within tm_region_init. The actual computation in tm_region_init
2647 // begins and ends with bb_regions entirely full of NULL pointers, due to
2648 // the way in which pointers are swapped in and out of the array.
2650 // ??? Our callers expect that blocks are not shared between transactions.
2651 // When the optimizers get too smart, and blocks are shared, then during
2652 // the tm_mark phase we'll add log entries to only one of the two transactions,
2653 // and in the tm_edge phase we'll add edges to the CFG that create invalid
2654 // cycles. The symptom being SSA defs that do not dominate their uses.
2655 // Note that the optimizers were locally correct with their transformation,
2656 // as we have no info within the program that suggests that the blocks cannot
2659 // ??? There is currently a hack inside tree-ssa-pre.c to work around the
2660 // only known instance of this block sharing.
2662 static vec
<tm_region_p
>
2663 get_bb_regions_instrumented (bool traverse_clones
,
2664 bool include_uninstrumented_p
)
2666 unsigned n
= last_basic_block_for_fn (cfun
);
2667 struct bb2reg_stuff stuff
;
2668 vec
<tm_region_p
> ret
;
2671 ret
.safe_grow_cleared (n
);
2672 stuff
.bb2reg
= &ret
;
2673 stuff
.include_uninstrumented_p
= include_uninstrumented_p
;
2674 expand_regions (all_tm_regions
, collect_bb2reg
, &stuff
, traverse_clones
);
2679 /* Set the IN_TRANSACTION for all gimple statements that appear in a
2683 compute_transaction_bits (void)
2685 struct tm_region
*region
;
2686 vec
<basic_block
> queue
;
2690 /* ?? Perhaps we need to abstract gate_tm_init further, because we
2691 certainly don't need it to calculate CDI_DOMINATOR info. */
2694 FOR_EACH_BB_FN (bb
, cfun
)
2695 bb
->flags
&= ~BB_IN_TRANSACTION
;
2697 for (region
= all_tm_regions
; region
; region
= region
->next
)
2699 queue
= get_tm_region_blocks (region
->entry_block
,
2700 region
->exit_blocks
,
2703 /*stop_at_irr_p=*/true);
2704 for (i
= 0; queue
.iterate (i
, &bb
); ++i
)
2705 bb
->flags
|= BB_IN_TRANSACTION
;
2710 bitmap_obstack_release (&tm_obstack
);
2713 /* Replace the GIMPLE_TRANSACTION in this region with the corresponding
2714 call to BUILT_IN_TM_START. */
2717 expand_transaction (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2719 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2720 basic_block transaction_bb
= gimple_bb (region
->transaction_stmt
);
2721 tree tm_state
= region
->tm_state
;
2722 tree tm_state_type
= TREE_TYPE (tm_state
);
2723 edge abort_edge
= NULL
;
2724 edge inst_edge
= NULL
;
2725 edge uninst_edge
= NULL
;
2726 edge fallthru_edge
= NULL
;
2728 // Identify the various successors of the transaction start.
2732 FOR_EACH_EDGE (e
, i
, transaction_bb
->succs
)
2734 if (e
->flags
& EDGE_TM_ABORT
)
2736 else if (e
->flags
& EDGE_TM_UNINSTRUMENTED
)
2740 if (e
->flags
& EDGE_FALLTHRU
)
2745 /* ??? There are plenty of bits here we're not computing. */
2747 int subcode
= gimple_transaction_subcode (region
->get_transaction_stmt ());
2749 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2750 flags
|= PR_DOESGOIRREVOCABLE
;
2751 if ((subcode
& GTMA_MAY_ENTER_IRREVOCABLE
) == 0)
2752 flags
|= PR_HASNOIRREVOCABLE
;
2753 /* If the transaction does not have an abort in lexical scope and is not
2754 marked as an outer transaction, then it will never abort. */
2755 if ((subcode
& GTMA_HAVE_ABORT
) == 0 && (subcode
& GTMA_IS_OUTER
) == 0)
2756 flags
|= PR_HASNOABORT
;
2757 if ((subcode
& GTMA_HAVE_STORE
) == 0)
2758 flags
|= PR_READONLY
;
2759 if (inst_edge
&& !(subcode
& GTMA_HAS_NO_INSTRUMENTATION
))
2760 flags
|= PR_INSTRUMENTEDCODE
;
2762 flags
|= PR_UNINSTRUMENTEDCODE
;
2763 if (subcode
& GTMA_IS_OUTER
)
2764 region
->original_transaction_was_outer
= true;
2765 tree t
= build_int_cst (tm_state_type
, flags
);
2766 gcall
*call
= gimple_build_call (tm_start
, 1, t
);
2767 gimple_call_set_lhs (call
, tm_state
);
2768 gimple_set_location (call
, gimple_location (region
->transaction_stmt
));
2770 // Replace the GIMPLE_TRANSACTION with the call to BUILT_IN_TM_START.
2771 gimple_stmt_iterator gsi
= gsi_last_bb (transaction_bb
);
2772 gcc_assert (gsi_stmt (gsi
) == region
->transaction_stmt
);
2773 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
2774 gsi_remove (&gsi
, true);
2775 region
->transaction_stmt
= call
;
2778 // Generate log saves.
2779 if (!tm_log_save_addresses
.is_empty ())
2780 tm_log_emit_saves (region
->entry_block
, transaction_bb
);
2782 // In the beginning, we've no tests to perform on transaction restart.
2783 // Note that after this point, transaction_bb becomes the "most recent
2784 // block containing tests for the transaction".
2785 region
->restart_block
= region
->entry_block
;
2787 // Generate log restores.
2788 if (!tm_log_save_addresses
.is_empty ())
2790 basic_block test_bb
= create_empty_bb (transaction_bb
);
2791 basic_block code_bb
= create_empty_bb (test_bb
);
2792 basic_block join_bb
= create_empty_bb (code_bb
);
2793 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2794 add_bb_to_loop (code_bb
, transaction_bb
->loop_father
);
2795 add_bb_to_loop (join_bb
, transaction_bb
->loop_father
);
2796 if (region
->restart_block
== region
->entry_block
)
2797 region
->restart_block
= test_bb
;
2799 tree t1
= create_tmp_reg (tm_state_type
);
2800 tree t2
= build_int_cst (tm_state_type
, A_RESTORELIVEVARIABLES
);
2801 gimple stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2802 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2803 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2805 t2
= build_int_cst (tm_state_type
, 0);
2806 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2807 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2809 tm_log_emit_restores (region
->entry_block
, code_bb
);
2811 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2812 edge et
= make_edge (test_bb
, code_bb
, EDGE_TRUE_VALUE
);
2813 edge ef
= make_edge (test_bb
, join_bb
, EDGE_FALSE_VALUE
);
2814 redirect_edge_pred (fallthru_edge
, join_bb
);
2816 join_bb
->frequency
= test_bb
->frequency
= transaction_bb
->frequency
;
2817 join_bb
->count
= test_bb
->count
= transaction_bb
->count
;
2819 ei
->probability
= PROB_ALWAYS
;
2820 et
->probability
= PROB_LIKELY
;
2821 ef
->probability
= PROB_UNLIKELY
;
2822 et
->count
= apply_probability (test_bb
->count
, et
->probability
);
2823 ef
->count
= apply_probability (test_bb
->count
, ef
->probability
);
2825 code_bb
->count
= et
->count
;
2826 code_bb
->frequency
= EDGE_FREQUENCY (et
);
2828 transaction_bb
= join_bb
;
2831 // If we have an ABORT edge, create a test to perform the abort.
2834 basic_block test_bb
= create_empty_bb (transaction_bb
);
2835 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2836 if (region
->restart_block
== region
->entry_block
)
2837 region
->restart_block
= test_bb
;
2839 tree t1
= create_tmp_reg (tm_state_type
);
2840 tree t2
= build_int_cst (tm_state_type
, A_ABORTTRANSACTION
);
2841 gimple stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2842 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2843 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2845 t2
= build_int_cst (tm_state_type
, 0);
2846 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2847 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2849 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2850 test_bb
->frequency
= transaction_bb
->frequency
;
2851 test_bb
->count
= transaction_bb
->count
;
2852 ei
->probability
= PROB_ALWAYS
;
2854 // Not abort edge. If both are live, chose one at random as we'll
2855 // we'll be fixing that up below.
2856 redirect_edge_pred (fallthru_edge
, test_bb
);
2857 fallthru_edge
->flags
= EDGE_FALSE_VALUE
;
2858 fallthru_edge
->probability
= PROB_VERY_LIKELY
;
2859 fallthru_edge
->count
2860 = apply_probability (test_bb
->count
, fallthru_edge
->probability
);
2863 redirect_edge_pred (abort_edge
, test_bb
);
2864 abort_edge
->flags
= EDGE_TRUE_VALUE
;
2865 abort_edge
->probability
= PROB_VERY_UNLIKELY
;
2867 = apply_probability (test_bb
->count
, abort_edge
->probability
);
2869 transaction_bb
= test_bb
;
2872 // If we have both instrumented and uninstrumented code paths, select one.
2873 if (inst_edge
&& uninst_edge
)
2875 basic_block test_bb
= create_empty_bb (transaction_bb
);
2876 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2877 if (region
->restart_block
== region
->entry_block
)
2878 region
->restart_block
= test_bb
;
2880 tree t1
= create_tmp_reg (tm_state_type
);
2881 tree t2
= build_int_cst (tm_state_type
, A_RUNUNINSTRUMENTEDCODE
);
2883 gimple stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2884 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2885 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2887 t2
= build_int_cst (tm_state_type
, 0);
2888 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2889 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2891 // Create the edge into test_bb first, as we want to copy values
2892 // out of the fallthru edge.
2893 edge e
= make_edge (transaction_bb
, test_bb
, fallthru_edge
->flags
);
2894 e
->probability
= fallthru_edge
->probability
;
2895 test_bb
->count
= e
->count
= fallthru_edge
->count
;
2896 test_bb
->frequency
= EDGE_FREQUENCY (e
);
2898 // Now update the edges to the inst/uninist implementations.
2899 // For now assume that the paths are equally likely. When using HTM,
2900 // we'll try the uninst path first and fallback to inst path if htm
2901 // buffers are exceeded. Without HTM we start with the inst path and
2902 // use the uninst path when falling back to serial mode.
2903 redirect_edge_pred (inst_edge
, test_bb
);
2904 inst_edge
->flags
= EDGE_FALSE_VALUE
;
2905 inst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2907 = apply_probability (test_bb
->count
, inst_edge
->probability
);
2909 redirect_edge_pred (uninst_edge
, test_bb
);
2910 uninst_edge
->flags
= EDGE_TRUE_VALUE
;
2911 uninst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2913 = apply_probability (test_bb
->count
, uninst_edge
->probability
);
2916 // If we have no previous special cases, and we have PHIs at the beginning
2917 // of the atomic region, this means we have a loop at the beginning of the
2918 // atomic region that shares the first block. This can cause problems with
2919 // the transaction restart abnormal edges to be added in the tm_edges pass.
2920 // Solve this by adding a new empty block to receive the abnormal edges.
2921 if (region
->restart_block
== region
->entry_block
2922 && phi_nodes (region
->entry_block
))
2924 basic_block empty_bb
= create_empty_bb (transaction_bb
);
2925 region
->restart_block
= empty_bb
;
2926 add_bb_to_loop (empty_bb
, transaction_bb
->loop_father
);
2928 redirect_edge_pred (fallthru_edge
, empty_bb
);
2929 make_edge (transaction_bb
, empty_bb
, EDGE_FALLTHRU
);
2935 /* Generate the temporary to be used for the return value of
2936 BUILT_IN_TM_START. */
2939 generate_tm_state (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2941 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2943 create_tmp_reg (TREE_TYPE (TREE_TYPE (tm_start
)), "tm_state");
2945 // Reset the subcode, post optimizations. We'll fill this in
2946 // again as we process blocks.
2947 if (region
->exit_blocks
)
2949 gtransaction
*transaction_stmt
= region
->get_transaction_stmt ();
2950 unsigned int subcode
= gimple_transaction_subcode (transaction_stmt
);
2952 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2953 subcode
&= (GTMA_DECLARATION_MASK
| GTMA_DOES_GO_IRREVOCABLE
2954 | GTMA_MAY_ENTER_IRREVOCABLE
2955 | GTMA_HAS_NO_INSTRUMENTATION
);
2957 subcode
&= GTMA_DECLARATION_MASK
;
2958 gimple_transaction_set_subcode (transaction_stmt
, subcode
);
2964 // Propagate flags from inner transactions outwards.
2966 propagate_tm_flags_out (struct tm_region
*region
)
2970 propagate_tm_flags_out (region
->inner
);
2972 if (region
->outer
&& region
->outer
->transaction_stmt
)
2975 = gimple_transaction_subcode (region
->get_transaction_stmt ());
2976 s
&= (GTMA_HAVE_ABORT
| GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
2977 | GTMA_MAY_ENTER_IRREVOCABLE
);
2978 s
|= gimple_transaction_subcode (region
->outer
->get_transaction_stmt ());
2979 gimple_transaction_set_subcode (region
->outer
->get_transaction_stmt (),
2983 propagate_tm_flags_out (region
->next
);
2986 /* Entry point to the MARK phase of TM expansion. Here we replace
2987 transactional memory statements with calls to builtins, and function
2988 calls with their transactional clones (if available). But we don't
2989 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */
2992 execute_tm_mark (void)
2994 pending_edge_inserts_p
= false;
2996 expand_regions (all_tm_regions
, generate_tm_state
, NULL
,
2997 /*traverse_clones=*/true);
3001 vec
<tm_region_p
> bb_regions
3002 = get_bb_regions_instrumented (/*traverse_clones=*/true,
3003 /*include_uninstrumented_p=*/false);
3004 struct tm_region
*r
;
3007 // Expand memory operations into calls into the runtime.
3008 // This collects log entries as well.
3009 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3013 if (r
->transaction_stmt
)
3016 = gimple_transaction_subcode (r
->get_transaction_stmt ());
3018 /* If we're sure to go irrevocable, there won't be
3019 anything to expand, since the run-time will go
3020 irrevocable right away. */
3021 if (sub
& GTMA_DOES_GO_IRREVOCABLE
3022 && sub
& GTMA_MAY_ENTER_IRREVOCABLE
)
3025 expand_block_tm (r
, BASIC_BLOCK_FOR_FN (cfun
, i
));
3029 bb_regions
.release ();
3031 // Propagate flags from inner transactions outwards.
3032 propagate_tm_flags_out (all_tm_regions
);
3034 // Expand GIMPLE_TRANSACTIONs into calls into the runtime.
3035 expand_regions (all_tm_regions
, expand_transaction
, NULL
,
3036 /*traverse_clones=*/false);
3041 if (pending_edge_inserts_p
)
3042 gsi_commit_edge_inserts ();
3043 free_dominance_info (CDI_DOMINATORS
);
3049 const pass_data pass_data_tm_mark
=
3051 GIMPLE_PASS
, /* type */
3052 "tmmark", /* name */
3053 OPTGROUP_NONE
, /* optinfo_flags */
3054 TV_TRANS_MEM
, /* tv_id */
3055 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3056 0, /* properties_provided */
3057 0, /* properties_destroyed */
3058 0, /* todo_flags_start */
3059 TODO_update_ssa
, /* todo_flags_finish */
3062 class pass_tm_mark
: public gimple_opt_pass
3065 pass_tm_mark (gcc::context
*ctxt
)
3066 : gimple_opt_pass (pass_data_tm_mark
, ctxt
)
3069 /* opt_pass methods: */
3070 virtual unsigned int execute (function
*) { return execute_tm_mark (); }
3072 }; // class pass_tm_mark
3077 make_pass_tm_mark (gcc::context
*ctxt
)
3079 return new pass_tm_mark (ctxt
);
3083 /* Create an abnormal edge from STMT at iter, splitting the block
3084 as necessary. Adjust *PNEXT as needed for the split block. */
3087 split_bb_make_tm_edge (gimple stmt
, basic_block dest_bb
,
3088 gimple_stmt_iterator iter
, gimple_stmt_iterator
*pnext
)
3090 basic_block bb
= gimple_bb (stmt
);
3091 if (!gsi_one_before_end_p (iter
))
3093 edge e
= split_block (bb
, stmt
);
3094 *pnext
= gsi_start_bb (e
->dest
);
3096 make_edge (bb
, dest_bb
, EDGE_ABNORMAL
);
3098 // Record the need for the edge for the benefit of the rtl passes.
3099 if (cfun
->gimple_df
->tm_restart
== NULL
)
3100 cfun
->gimple_df
->tm_restart
3101 = hash_table
<tm_restart_hasher
>::create_ggc (31);
3103 struct tm_restart_node dummy
;
3105 dummy
.label_or_list
= gimple_block_label (dest_bb
);
3107 tm_restart_node
**slot
= cfun
->gimple_df
->tm_restart
->find_slot (&dummy
,
3109 struct tm_restart_node
*n
= *slot
;
3112 n
= ggc_alloc
<tm_restart_node
> ();
3117 tree old
= n
->label_or_list
;
3118 if (TREE_CODE (old
) == LABEL_DECL
)
3119 old
= tree_cons (NULL
, old
, NULL
);
3120 n
->label_or_list
= tree_cons (NULL
, dummy
.label_or_list
, old
);
3124 /* Split block BB as necessary for every builtin function we added, and
3125 wire up the abnormal back edges implied by the transaction restart. */
3128 expand_block_edges (struct tm_region
*const region
, basic_block bb
)
3130 gimple_stmt_iterator gsi
, next_gsi
;
3132 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi
= next_gsi
)
3134 gimple stmt
= gsi_stmt (gsi
);
3138 gsi_next (&next_gsi
);
3140 // ??? Shouldn't we split for any non-pure, non-irrevocable function?
3141 call_stmt
= dyn_cast
<gcall
*> (stmt
);
3143 || (gimple_call_flags (call_stmt
) & ECF_TM_BUILTIN
) == 0)
3146 if (DECL_FUNCTION_CODE (gimple_call_fndecl (call_stmt
))
3147 == BUILT_IN_TM_ABORT
)
3149 // If we have a ``_transaction_cancel [[outer]]'', there is only
3150 // one abnormal edge: to the transaction marked OUTER.
3151 // All compiler-generated instances of BUILT_IN_TM_ABORT have a
3152 // constant argument, which we can examine here. Users invoking
3153 // TM_ABORT directly get what they deserve.
3154 tree arg
= gimple_call_arg (call_stmt
, 0);
3155 if (TREE_CODE (arg
) == INTEGER_CST
3156 && (TREE_INT_CST_LOW (arg
) & AR_OUTERABORT
) != 0
3157 && !decl_is_tm_clone (current_function_decl
))
3159 // Find the GTMA_IS_OUTER transaction.
3160 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3161 if (o
->original_transaction_was_outer
)
3163 split_bb_make_tm_edge (call_stmt
, o
->restart_block
,
3168 // Otherwise, the front-end should have semantically checked
3169 // outer aborts, but in either case the target region is not
3170 // within this function.
3174 // Non-outer, TM aborts have an abnormal edge to the inner-most
3175 // transaction, the one being aborted;
3176 split_bb_make_tm_edge (call_stmt
, region
->restart_block
, gsi
,
3180 // All TM builtins have an abnormal edge to the outer-most transaction.
3181 // We never restart inner transactions. For tm clones, we know a-priori
3182 // that the outer-most transaction is outside the function.
3183 if (decl_is_tm_clone (current_function_decl
))
3186 if (cfun
->gimple_df
->tm_restart
== NULL
)
3187 cfun
->gimple_df
->tm_restart
3188 = hash_table
<tm_restart_hasher
>::create_ggc (31);
3190 // All TM builtins have an abnormal edge to the outer-most transaction.
3191 // We never restart inner transactions.
3192 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3195 split_bb_make_tm_edge (call_stmt
, o
->restart_block
, gsi
, &next_gsi
);
3199 // Delete any tail-call annotation that may have been added.
3200 // The tail-call pass may have mis-identified the commit as being
3201 // a candidate because we had not yet added this restart edge.
3202 gimple_call_set_tail (call_stmt
, false);
3206 /* Entry point to the final expansion of transactional nodes. */
3210 const pass_data pass_data_tm_edges
=
3212 GIMPLE_PASS
, /* type */
3213 "tmedge", /* name */
3214 OPTGROUP_NONE
, /* optinfo_flags */
3215 TV_TRANS_MEM
, /* tv_id */
3216 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3217 0, /* properties_provided */
3218 0, /* properties_destroyed */
3219 0, /* todo_flags_start */
3220 TODO_update_ssa
, /* todo_flags_finish */
3223 class pass_tm_edges
: public gimple_opt_pass
3226 pass_tm_edges (gcc::context
*ctxt
)
3227 : gimple_opt_pass (pass_data_tm_edges
, ctxt
)
3230 /* opt_pass methods: */
3231 virtual unsigned int execute (function
*);
3233 }; // class pass_tm_edges
3236 pass_tm_edges::execute (function
*fun
)
3238 vec
<tm_region_p
> bb_regions
3239 = get_bb_regions_instrumented (/*traverse_clones=*/false,
3240 /*include_uninstrumented_p=*/true);
3241 struct tm_region
*r
;
3244 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3246 expand_block_edges (r
, BASIC_BLOCK_FOR_FN (fun
, i
));
3248 bb_regions
.release ();
3250 /* We've got to release the dominance info now, to indicate that it
3251 must be rebuilt completely. Otherwise we'll crash trying to update
3252 the SSA web in the TODO section following this pass. */
3253 free_dominance_info (CDI_DOMINATORS
);
3254 bitmap_obstack_release (&tm_obstack
);
3255 all_tm_regions
= NULL
;
3263 make_pass_tm_edges (gcc::context
*ctxt
)
3265 return new pass_tm_edges (ctxt
);
3268 /* Helper function for expand_regions. Expand REGION and recurse to
3269 the inner region. Call CALLBACK on each region. CALLBACK returns
3270 NULL to continue the traversal, otherwise a non-null value which
3271 this function will return as well. TRAVERSE_CLONES is true if we
3272 should traverse transactional clones. */
3275 expand_regions_1 (struct tm_region
*region
,
3276 void *(*callback
)(struct tm_region
*, void *),
3278 bool traverse_clones
)
3280 void *retval
= NULL
;
3281 if (region
->exit_blocks
3282 || (traverse_clones
&& decl_is_tm_clone (current_function_decl
)))
3284 retval
= callback (region
, data
);
3290 retval
= expand_regions (region
->inner
, callback
, data
, traverse_clones
);
3297 /* Traverse the regions enclosed and including REGION. Execute
3298 CALLBACK for each region, passing DATA. CALLBACK returns NULL to
3299 continue the traversal, otherwise a non-null value which this
3300 function will return as well. TRAVERSE_CLONES is true if we should
3301 traverse transactional clones. */
3304 expand_regions (struct tm_region
*region
,
3305 void *(*callback
)(struct tm_region
*, void *),
3307 bool traverse_clones
)
3309 void *retval
= NULL
;
3312 retval
= expand_regions_1 (region
, callback
, data
, traverse_clones
);
3315 region
= region
->next
;
3321 /* A unique TM memory operation. */
3322 typedef struct tm_memop
3324 /* Unique ID that all memory operations to the same location have. */
3325 unsigned int value_id
;
3326 /* Address of load/store. */
3330 /* TM memory operation hashtable helpers. */
3332 struct tm_memop_hasher
: free_ptr_hash
<tm_memop
>
3334 static inline hashval_t
hash (const tm_memop
*);
3335 static inline bool equal (const tm_memop
*, const tm_memop
*);
3338 /* Htab support. Return a hash value for a `tm_memop'. */
3340 tm_memop_hasher::hash (const tm_memop
*mem
)
3342 tree addr
= mem
->addr
;
3343 /* We drill down to the SSA_NAME/DECL for the hash, but equality is
3344 actually done with operand_equal_p (see tm_memop_eq). */
3345 if (TREE_CODE (addr
) == ADDR_EXPR
)
3346 addr
= TREE_OPERAND (addr
, 0);
3347 return iterative_hash_expr (addr
, 0);
3350 /* Htab support. Return true if two tm_memop's are the same. */
3352 tm_memop_hasher::equal (const tm_memop
*mem1
, const tm_memop
*mem2
)
3354 return operand_equal_p (mem1
->addr
, mem2
->addr
, 0);
3357 /* Sets for solving data flow equations in the memory optimization pass. */
3358 struct tm_memopt_bitmaps
3360 /* Stores available to this BB upon entry. Basically, stores that
3361 dominate this BB. */
3362 bitmap store_avail_in
;
3363 /* Stores available at the end of this BB. */
3364 bitmap store_avail_out
;
3365 bitmap store_antic_in
;
3366 bitmap store_antic_out
;
3367 /* Reads available to this BB upon entry. Basically, reads that
3368 dominate this BB. */
3369 bitmap read_avail_in
;
3370 /* Reads available at the end of this BB. */
3371 bitmap read_avail_out
;
3372 /* Reads performed in this BB. */
3374 /* Writes performed in this BB. */
3377 /* Temporary storage for pass. */
3378 /* Is the current BB in the worklist? */
3379 bool avail_in_worklist_p
;
3380 /* Have we visited this BB? */
3384 static bitmap_obstack tm_memopt_obstack
;
3386 /* Unique counter for TM loads and stores. Loads and stores of the
3387 same address get the same ID. */
3388 static unsigned int tm_memopt_value_id
;
3389 static hash_table
<tm_memop_hasher
> *tm_memopt_value_numbers
;
3391 #define STORE_AVAIL_IN(BB) \
3392 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in
3393 #define STORE_AVAIL_OUT(BB) \
3394 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out
3395 #define STORE_ANTIC_IN(BB) \
3396 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in
3397 #define STORE_ANTIC_OUT(BB) \
3398 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out
3399 #define READ_AVAIL_IN(BB) \
3400 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in
3401 #define READ_AVAIL_OUT(BB) \
3402 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out
3403 #define READ_LOCAL(BB) \
3404 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local
3405 #define STORE_LOCAL(BB) \
3406 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local
3407 #define AVAIL_IN_WORKLIST_P(BB) \
3408 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p
3409 #define BB_VISITED_P(BB) \
3410 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p
3412 /* Given a TM load/store in STMT, return the value number for the address
3416 tm_memopt_value_number (gimple stmt
, enum insert_option op
)
3418 struct tm_memop tmpmem
, *mem
;
3421 gcc_assert (is_tm_load (stmt
) || is_tm_store (stmt
));
3422 tmpmem
.addr
= gimple_call_arg (stmt
, 0);
3423 slot
= tm_memopt_value_numbers
->find_slot (&tmpmem
, op
);
3426 else if (op
== INSERT
)
3428 mem
= XNEW (struct tm_memop
);
3430 mem
->value_id
= tm_memopt_value_id
++;
3431 mem
->addr
= tmpmem
.addr
;
3435 return mem
->value_id
;
3438 /* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */
3441 tm_memopt_accumulate_memops (basic_block bb
)
3443 gimple_stmt_iterator gsi
;
3445 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3447 gimple stmt
= gsi_stmt (gsi
);
3451 if (is_tm_store (stmt
))
3452 bits
= STORE_LOCAL (bb
);
3453 else if (is_tm_load (stmt
))
3454 bits
= READ_LOCAL (bb
);
3458 loc
= tm_memopt_value_number (stmt
, INSERT
);
3459 bitmap_set_bit (bits
, loc
);
3462 fprintf (dump_file
, "TM memopt (%s): value num=%d, BB=%d, addr=",
3463 is_tm_load (stmt
) ? "LOAD" : "STORE", loc
,
3464 gimple_bb (stmt
)->index
);
3465 print_generic_expr (dump_file
, gimple_call_arg (stmt
, 0), 0);
3466 fprintf (dump_file
, "\n");
3471 /* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */
3474 dump_tm_memopt_set (const char *set_name
, bitmap bits
)
3478 const char *comma
= "";
3480 fprintf (dump_file
, "TM memopt: %s: [", set_name
);
3481 EXECUTE_IF_SET_IN_BITMAP (bits
, 0, i
, bi
)
3483 hash_table
<tm_memop_hasher
>::iterator hi
;
3484 struct tm_memop
*mem
= NULL
;
3486 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */
3487 FOR_EACH_HASH_TABLE_ELEMENT (*tm_memopt_value_numbers
, mem
, tm_memop_t
, hi
)
3488 if (mem
->value_id
== i
)
3490 gcc_assert (mem
->value_id
== i
);
3491 fprintf (dump_file
, "%s", comma
);
3493 print_generic_expr (dump_file
, mem
->addr
, 0);
3495 fprintf (dump_file
, "]\n");
3498 /* Prettily dump all of the memopt sets in BLOCKS. */
3501 dump_tm_memopt_sets (vec
<basic_block
> blocks
)
3506 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3508 fprintf (dump_file
, "------------BB %d---------\n", bb
->index
);
3509 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb
));
3510 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb
));
3511 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb
));
3512 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb
));
3513 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb
));
3514 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb
));
3518 /* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */
3521 tm_memopt_compute_avin (basic_block bb
)
3526 /* Seed with the AVOUT of any predecessor. */
3527 for (ix
= 0; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3529 e
= EDGE_PRED (bb
, ix
);
3530 /* Make sure we have already visited this BB, and is thus
3533 If e->src->aux is NULL, this predecessor is actually on an
3534 enclosing transaction. We only care about the current
3535 transaction, so ignore it. */
3536 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3538 bitmap_copy (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3539 bitmap_copy (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3544 for (; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3546 e
= EDGE_PRED (bb
, ix
);
3547 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3549 bitmap_and_into (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3550 bitmap_and_into (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3554 BB_VISITED_P (bb
) = true;
3557 /* Compute the STORE_ANTIC_IN for the basic block BB. */
3560 tm_memopt_compute_antin (basic_block bb
)
3565 /* Seed with the ANTIC_OUT of any successor. */
3566 for (ix
= 0; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3568 e
= EDGE_SUCC (bb
, ix
);
3569 /* Make sure we have already visited this BB, and is thus
3571 if (BB_VISITED_P (e
->dest
))
3573 bitmap_copy (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3578 for (; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3580 e
= EDGE_SUCC (bb
, ix
);
3581 if (BB_VISITED_P (e
->dest
))
3582 bitmap_and_into (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3585 BB_VISITED_P (bb
) = true;
3588 /* Compute the AVAIL sets for every basic block in BLOCKS.
3590 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows:
3592 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb])
3593 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors])
3595 This is basically what we do in lcm's compute_available(), but here
3596 we calculate two sets of sets (one for STOREs and one for READs),
3597 and we work on a region instead of the entire CFG.
3599 REGION is the TM region.
3600 BLOCKS are the basic blocks in the region. */
3603 tm_memopt_compute_available (struct tm_region
*region
,
3604 vec
<basic_block
> blocks
)
3607 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3608 unsigned int qlen
, i
;
3612 /* Allocate a worklist array/queue. Entries are only added to the
3613 list if they were not already on the list. So the size is
3614 bounded by the number of basic blocks in the region. */
3615 qlen
= blocks
.length () - 1;
3616 qin
= qout
= worklist
=
3617 XNEWVEC (basic_block
, qlen
);
3619 /* Put every block in the region on the worklist. */
3620 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3622 /* Seed AVAIL_OUT with the LOCAL set. */
3623 bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_LOCAL (bb
));
3624 bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_LOCAL (bb
));
3626 AVAIL_IN_WORKLIST_P (bb
) = true;
3627 /* No need to insert the entry block, since it has an AVIN of
3628 null, and an AVOUT that has already been seeded in. */
3629 if (bb
!= region
->entry_block
)
3633 /* The entry block has been initialized with the local sets. */
3634 BB_VISITED_P (region
->entry_block
) = true;
3637 qend
= &worklist
[qlen
];
3639 /* Iterate until the worklist is empty. */
3642 /* Take the first entry off the worklist. */
3649 /* This block can be added to the worklist again if necessary. */
3650 AVAIL_IN_WORKLIST_P (bb
) = false;
3651 tm_memopt_compute_avin (bb
);
3653 /* Note: We do not add the LOCAL sets here because we already
3654 seeded the AVAIL_OUT sets with them. */
3655 changed
= bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_AVAIL_IN (bb
));
3656 changed
|= bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_AVAIL_IN (bb
));
3658 && (region
->exit_blocks
== NULL
3659 || !bitmap_bit_p (region
->exit_blocks
, bb
->index
)))
3660 /* If the out state of this block changed, then we need to add
3661 its successors to the worklist if they are not already in. */
3662 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3663 if (!AVAIL_IN_WORKLIST_P (e
->dest
)
3664 && e
->dest
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
3667 AVAIL_IN_WORKLIST_P (e
->dest
) = true;
3678 dump_tm_memopt_sets (blocks
);
3681 /* Compute ANTIC sets for every basic block in BLOCKS.
3683 We compute STORE_ANTIC_OUT as follows:
3685 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb])
3686 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors])
3688 REGION is the TM region.
3689 BLOCKS are the basic blocks in the region. */
3692 tm_memopt_compute_antic (struct tm_region
*region
,
3693 vec
<basic_block
> blocks
)
3696 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3701 /* Allocate a worklist array/queue. Entries are only added to the
3702 list if they were not already on the list. So the size is
3703 bounded by the number of basic blocks in the region. */
3704 qin
= qout
= worklist
= XNEWVEC (basic_block
, blocks
.length ());
3706 for (qlen
= 0, i
= blocks
.length () - 1; i
>= 0; --i
)
3710 /* Seed ANTIC_OUT with the LOCAL set. */
3711 bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_LOCAL (bb
));
3713 /* Put every block in the region on the worklist. */
3714 AVAIL_IN_WORKLIST_P (bb
) = true;
3715 /* No need to insert exit blocks, since their ANTIC_IN is NULL,
3716 and their ANTIC_OUT has already been seeded in. */
3717 if (region
->exit_blocks
3718 && !bitmap_bit_p (region
->exit_blocks
, bb
->index
))
3725 /* The exit blocks have been initialized with the local sets. */
3726 if (region
->exit_blocks
)
3730 EXECUTE_IF_SET_IN_BITMAP (region
->exit_blocks
, 0, i
, bi
)
3731 BB_VISITED_P (BASIC_BLOCK_FOR_FN (cfun
, i
)) = true;
3735 qend
= &worklist
[qlen
];
3737 /* Iterate until the worklist is empty. */
3740 /* Take the first entry off the worklist. */
3747 /* This block can be added to the worklist again if necessary. */
3748 AVAIL_IN_WORKLIST_P (bb
) = false;
3749 tm_memopt_compute_antin (bb
);
3751 /* Note: We do not add the LOCAL sets here because we already
3752 seeded the ANTIC_OUT sets with them. */
3753 if (bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_ANTIC_IN (bb
))
3754 && bb
!= region
->entry_block
)
3755 /* If the out state of this block changed, then we need to add
3756 its predecessors to the worklist if they are not already in. */
3757 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
3758 if (!AVAIL_IN_WORKLIST_P (e
->src
))
3761 AVAIL_IN_WORKLIST_P (e
->src
) = true;
3772 dump_tm_memopt_sets (blocks
);
3775 /* Offsets of load variants from TM_LOAD. For example,
3776 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*.
3777 See gtm-builtins.def. */
3778 #define TRANSFORM_RAR 1
3779 #define TRANSFORM_RAW 2
3780 #define TRANSFORM_RFW 3
3781 /* Offsets of store variants from TM_STORE. */
3782 #define TRANSFORM_WAR 1
3783 #define TRANSFORM_WAW 2
3785 /* Inform about a load/store optimization. */
3788 dump_tm_memopt_transform (gimple stmt
)
3792 fprintf (dump_file
, "TM memopt: transforming: ");
3793 print_gimple_stmt (dump_file
, stmt
, 0, 0);
3794 fprintf (dump_file
, "\n");
3798 /* Perform a read/write optimization. Replaces the TM builtin in STMT
3799 by a builtin that is OFFSET entries down in the builtins table in
3800 gtm-builtins.def. */
3803 tm_memopt_transform_stmt (unsigned int offset
,
3805 gimple_stmt_iterator
*gsi
)
3807 tree fn
= gimple_call_fn (stmt
);
3808 gcc_assert (TREE_CODE (fn
) == ADDR_EXPR
);
3809 TREE_OPERAND (fn
, 0)
3810 = builtin_decl_explicit ((enum built_in_function
)
3811 (DECL_FUNCTION_CODE (TREE_OPERAND (fn
, 0))
3813 gimple_call_set_fn (stmt
, fn
);
3814 gsi_replace (gsi
, stmt
, true);
3815 dump_tm_memopt_transform (stmt
);
3818 /* Perform the actual TM memory optimization transformations in the
3819 basic blocks in BLOCKS. */
3822 tm_memopt_transform_blocks (vec
<basic_block
> blocks
)
3826 gimple_stmt_iterator gsi
;
3828 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3830 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3832 gimple stmt
= gsi_stmt (gsi
);
3833 bitmap read_avail
= READ_AVAIL_IN (bb
);
3834 bitmap store_avail
= STORE_AVAIL_IN (bb
);
3835 bitmap store_antic
= STORE_ANTIC_OUT (bb
);
3838 if (is_tm_simple_load (stmt
))
3840 gcall
*call_stmt
= as_a
<gcall
*> (stmt
);
3841 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3842 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3843 tm_memopt_transform_stmt (TRANSFORM_RAW
, call_stmt
, &gsi
);
3844 else if (store_antic
&& bitmap_bit_p (store_antic
, loc
))
3846 tm_memopt_transform_stmt (TRANSFORM_RFW
, call_stmt
, &gsi
);
3847 bitmap_set_bit (store_avail
, loc
);
3849 else if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3850 tm_memopt_transform_stmt (TRANSFORM_RAR
, call_stmt
, &gsi
);
3852 bitmap_set_bit (read_avail
, loc
);
3854 else if (is_tm_simple_store (stmt
))
3856 gcall
*call_stmt
= as_a
<gcall
*> (stmt
);
3857 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3858 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3859 tm_memopt_transform_stmt (TRANSFORM_WAW
, call_stmt
, &gsi
);
3862 if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3863 tm_memopt_transform_stmt (TRANSFORM_WAR
, call_stmt
, &gsi
);
3864 bitmap_set_bit (store_avail
, loc
);
3871 /* Return a new set of bitmaps for a BB. */
3873 static struct tm_memopt_bitmaps
*
3874 tm_memopt_init_sets (void)
3876 struct tm_memopt_bitmaps
*b
3877 = XOBNEW (&tm_memopt_obstack
.obstack
, struct tm_memopt_bitmaps
);
3878 b
->store_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3879 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3880 b
->store_antic_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3881 b
->store_antic_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3882 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3883 b
->read_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3884 b
->read_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3885 b
->read_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3886 b
->store_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3890 /* Free sets computed for each BB. */
3893 tm_memopt_free_sets (vec
<basic_block
> blocks
)
3898 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3902 /* Clear the visited bit for every basic block in BLOCKS. */
3905 tm_memopt_clear_visited (vec
<basic_block
> blocks
)
3910 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3911 BB_VISITED_P (bb
) = false;
3914 /* Replace TM load/stores with hints for the runtime. We handle
3915 things like read-after-write, write-after-read, read-after-read,
3916 read-for-write, etc. */
3919 execute_tm_memopt (void)
3921 struct tm_region
*region
;
3922 vec
<basic_block
> bbs
;
3924 tm_memopt_value_id
= 0;
3925 tm_memopt_value_numbers
= new hash_table
<tm_memop_hasher
> (10);
3927 for (region
= all_tm_regions
; region
; region
= region
->next
)
3929 /* All the TM stores/loads in the current region. */
3933 bitmap_obstack_initialize (&tm_memopt_obstack
);
3935 /* Save all BBs for the current region. */
3936 bbs
= get_tm_region_blocks (region
->entry_block
,
3937 region
->exit_blocks
,
3942 /* Collect all the memory operations. */
3943 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
3945 bb
->aux
= tm_memopt_init_sets ();
3946 tm_memopt_accumulate_memops (bb
);
3949 /* Solve data flow equations and transform each block accordingly. */
3950 tm_memopt_clear_visited (bbs
);
3951 tm_memopt_compute_available (region
, bbs
);
3952 tm_memopt_clear_visited (bbs
);
3953 tm_memopt_compute_antic (region
, bbs
);
3954 tm_memopt_transform_blocks (bbs
);
3956 tm_memopt_free_sets (bbs
);
3958 bitmap_obstack_release (&tm_memopt_obstack
);
3959 tm_memopt_value_numbers
->empty ();
3962 delete tm_memopt_value_numbers
;
3963 tm_memopt_value_numbers
= NULL
;
3969 const pass_data pass_data_tm_memopt
=
3971 GIMPLE_PASS
, /* type */
3972 "tmmemopt", /* name */
3973 OPTGROUP_NONE
, /* optinfo_flags */
3974 TV_TRANS_MEM
, /* tv_id */
3975 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3976 0, /* properties_provided */
3977 0, /* properties_destroyed */
3978 0, /* todo_flags_start */
3979 0, /* todo_flags_finish */
3982 class pass_tm_memopt
: public gimple_opt_pass
3985 pass_tm_memopt (gcc::context
*ctxt
)
3986 : gimple_opt_pass (pass_data_tm_memopt
, ctxt
)
3989 /* opt_pass methods: */
3990 virtual bool gate (function
*) { return flag_tm
&& optimize
> 0; }
3991 virtual unsigned int execute (function
*) { return execute_tm_memopt (); }
3993 }; // class pass_tm_memopt
3998 make_pass_tm_memopt (gcc::context
*ctxt
)
4000 return new pass_tm_memopt (ctxt
);
4004 /* Interprocedual analysis for the creation of transactional clones.
4005 The aim of this pass is to find which functions are referenced in
4006 a non-irrevocable transaction context, and for those over which
4007 we have control (or user directive), create a version of the
4008 function which uses only the transactional interface to reference
4009 protected memories. This analysis proceeds in several steps:
4011 (1) Collect the set of all possible transactional clones:
4013 (a) For all local public functions marked tm_callable, push
4014 it onto the tm_callee queue.
4016 (b) For all local functions, scan for calls in transaction blocks.
4017 Push the caller and callee onto the tm_caller and tm_callee
4018 queues. Count the number of callers for each callee.
4020 (c) For each local function on the callee list, assume we will
4021 create a transactional clone. Push *all* calls onto the
4022 callee queues; count the number of clone callers separately
4023 to the number of original callers.
4025 (2) Propagate irrevocable status up the dominator tree:
4027 (a) Any external function on the callee list that is not marked
4028 tm_callable is irrevocable. Push all callers of such onto
4031 (b) For each function on the worklist, mark each block that
4032 contains an irrevocable call. Use the AND operator to
4033 propagate that mark up the dominator tree.
4035 (c) If we reach the entry block for a possible transactional
4036 clone, then the transactional clone is irrevocable, and
4037 we should not create the clone after all. Push all
4038 callers onto the worklist.
4040 (d) Place tm_irrevocable calls at the beginning of the relevant
4041 blocks. Special case here is the entry block for the entire
4042 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for
4043 the library to begin the region in serial mode. Decrement
4044 the call count for all callees in the irrevocable region.
4046 (3) Create the transactional clones:
4048 Any tm_callee that still has a non-zero call count is cloned.
4051 /* This structure is stored in the AUX field of each cgraph_node. */
4052 struct tm_ipa_cg_data
4054 /* The clone of the function that got created. */
4055 struct cgraph_node
*clone
;
4057 /* The tm regions in the normal function. */
4058 struct tm_region
*all_tm_regions
;
4060 /* The blocks of the normal/clone functions that contain irrevocable
4061 calls, or blocks that are post-dominated by irrevocable calls. */
4062 bitmap irrevocable_blocks_normal
;
4063 bitmap irrevocable_blocks_clone
;
4065 /* The blocks of the normal function that are involved in transactions. */
4066 bitmap transaction_blocks_normal
;
4068 /* The number of callers to the transactional clone of this function
4069 from normal and transactional clones respectively. */
4070 unsigned tm_callers_normal
;
4071 unsigned tm_callers_clone
;
4073 /* True if all calls to this function's transactional clone
4074 are irrevocable. Also automatically true if the function
4075 has no transactional clone. */
4076 bool is_irrevocable
;
4078 /* Flags indicating the presence of this function in various queues. */
4079 bool in_callee_queue
;
4082 /* Flags indicating the kind of scan desired while in the worklist. */
4083 bool want_irr_scan_normal
;
4086 typedef vec
<cgraph_node
*> cgraph_node_queue
;
4088 /* Return the ipa data associated with NODE, allocating zeroed memory
4089 if necessary. TRAVERSE_ALIASES is true if we must traverse aliases
4090 and set *NODE accordingly. */
4092 static struct tm_ipa_cg_data
*
4093 get_cg_data (struct cgraph_node
**node
, bool traverse_aliases
)
4095 struct tm_ipa_cg_data
*d
;
4097 if (traverse_aliases
&& (*node
)->alias
)
4098 *node
= (*node
)->get_alias_target ();
4100 d
= (struct tm_ipa_cg_data
*) (*node
)->aux
;
4104 d
= (struct tm_ipa_cg_data
*)
4105 obstack_alloc (&tm_obstack
.obstack
, sizeof (*d
));
4106 (*node
)->aux
= (void *) d
;
4107 memset (d
, 0, sizeof (*d
));
4113 /* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that
4114 it is already present. */
4117 maybe_push_queue (struct cgraph_node
*node
,
4118 cgraph_node_queue
*queue_p
, bool *in_queue_p
)
4123 queue_p
->safe_push (node
);
4127 /* Duplicate the basic blocks in QUEUE for use in the uninstrumented
4128 code path. QUEUE are the basic blocks inside the transaction
4129 represented in REGION.
4131 Later in split_code_paths() we will add the conditional to choose
4132 between the two alternatives. */
4135 ipa_uninstrument_transaction (struct tm_region
*region
,
4136 vec
<basic_block
> queue
)
4138 gimple transaction
= region
->transaction_stmt
;
4139 basic_block transaction_bb
= gimple_bb (transaction
);
4140 int n
= queue
.length ();
4141 basic_block
*new_bbs
= XNEWVEC (basic_block
, n
);
4143 copy_bbs (queue
.address (), n
, new_bbs
, NULL
, 0, NULL
, NULL
, transaction_bb
,
4145 edge e
= make_edge (transaction_bb
, new_bbs
[0], EDGE_TM_UNINSTRUMENTED
);
4146 add_phi_args_after_copy (new_bbs
, n
, e
);
4148 // Now we will have a GIMPLE_ATOMIC with 3 possible edges out of it.
4149 // a) EDGE_FALLTHRU into the transaction
4150 // b) EDGE_TM_ABORT out of the transaction
4151 // c) EDGE_TM_UNINSTRUMENTED into the uninstrumented blocks.
4156 /* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone.
4157 Queue all callees within block BB. */
4160 ipa_tm_scan_calls_block (cgraph_node_queue
*callees_p
,
4161 basic_block bb
, bool for_clone
)
4163 gimple_stmt_iterator gsi
;
4165 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4167 gimple stmt
= gsi_stmt (gsi
);
4168 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4170 tree fndecl
= gimple_call_fndecl (stmt
);
4173 struct tm_ipa_cg_data
*d
;
4175 struct cgraph_node
*node
;
4177 if (is_tm_ending_fndecl (fndecl
))
4179 if (find_tm_replacement_function (fndecl
))
4182 node
= cgraph_node::get (fndecl
);
4183 gcc_assert (node
!= NULL
);
4184 d
= get_cg_data (&node
, true);
4186 pcallers
= (for_clone
? &d
->tm_callers_clone
4187 : &d
->tm_callers_normal
);
4190 maybe_push_queue (node
, callees_p
, &d
->in_callee_queue
);
4196 /* Scan all calls in NODE that are within a transaction region,
4197 and push the resulting nodes into the callee queue. */
4200 ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data
*d
,
4201 cgraph_node_queue
*callees_p
)
4203 struct tm_region
*r
;
4205 d
->transaction_blocks_normal
= BITMAP_ALLOC (&tm_obstack
);
4206 d
->all_tm_regions
= all_tm_regions
;
4208 for (r
= all_tm_regions
; r
; r
= r
->next
)
4210 vec
<basic_block
> bbs
;
4214 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
, NULL
,
4215 d
->transaction_blocks_normal
, false);
4217 // Generate the uninstrumented code path for this transaction.
4218 ipa_uninstrument_transaction (r
, bbs
);
4220 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
4221 ipa_tm_scan_calls_block (callees_p
, bb
, false);
4226 // ??? copy_bbs should maintain cgraph edges for the blocks as it is
4227 // copying them, rather than forcing us to do this externally.
4228 cgraph_edge::rebuild_edges ();
4230 // ??? In ipa_uninstrument_transaction we don't try to update dominators
4231 // because copy_bbs doesn't return a VEC like iterate_fix_dominators expects.
4232 // Instead, just release dominators here so update_ssa recomputes them.
4233 free_dominance_info (CDI_DOMINATORS
);
4235 // When building the uninstrumented code path, copy_bbs will have invoked
4236 // create_new_def_for starting an "ssa update context". There is only one
4237 // instance of this context, so resolve ssa updates before moving on to
4238 // the next function.
4239 update_ssa (TODO_update_ssa
);
4242 /* Scan all calls in NODE as if this is the transactional clone,
4243 and push the destinations into the callee queue. */
4246 ipa_tm_scan_calls_clone (struct cgraph_node
*node
,
4247 cgraph_node_queue
*callees_p
)
4249 struct function
*fn
= DECL_STRUCT_FUNCTION (node
->decl
);
4252 FOR_EACH_BB_FN (bb
, fn
)
4253 ipa_tm_scan_calls_block (callees_p
, bb
, true);
4256 /* The function NODE has been detected to be irrevocable. Push all
4257 of its callers onto WORKLIST for the purpose of re-scanning them. */
4260 ipa_tm_note_irrevocable (struct cgraph_node
*node
,
4261 cgraph_node_queue
*worklist_p
)
4263 struct tm_ipa_cg_data
*d
= get_cg_data (&node
, true);
4264 struct cgraph_edge
*e
;
4266 d
->is_irrevocable
= true;
4268 for (e
= node
->callers
; e
; e
= e
->next_caller
)
4271 struct cgraph_node
*caller
;
4273 /* Don't examine recursive calls. */
4274 if (e
->caller
== node
)
4276 /* Even if we think we can go irrevocable, believe the user
4278 if (is_tm_safe_or_pure (e
->caller
->decl
))
4282 d
= get_cg_data (&caller
, true);
4284 /* Check if the callee is in a transactional region. If so,
4285 schedule the function for normal re-scan as well. */
4286 bb
= gimple_bb (e
->call_stmt
);
4287 gcc_assert (bb
!= NULL
);
4288 if (d
->transaction_blocks_normal
4289 && bitmap_bit_p (d
->transaction_blocks_normal
, bb
->index
))
4290 d
->want_irr_scan_normal
= true;
4292 maybe_push_queue (caller
, worklist_p
, &d
->in_worklist
);
4296 /* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement
4297 within the block is irrevocable. */
4300 ipa_tm_scan_irr_block (basic_block bb
)
4302 gimple_stmt_iterator gsi
;
4305 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4307 gimple stmt
= gsi_stmt (gsi
);
4308 switch (gimple_code (stmt
))
4311 if (gimple_assign_single_p (stmt
))
4313 tree lhs
= gimple_assign_lhs (stmt
);
4314 tree rhs
= gimple_assign_rhs1 (stmt
);
4315 if (volatile_var_p (lhs
) || volatile_var_p (rhs
))
4322 tree lhs
= gimple_call_lhs (stmt
);
4323 if (lhs
&& volatile_var_p (lhs
))
4326 if (is_tm_pure_call (stmt
))
4329 fn
= gimple_call_fn (stmt
);
4331 /* Functions with the attribute are by definition irrevocable. */
4332 if (is_tm_irrevocable (fn
))
4335 /* For direct function calls, go ahead and check for replacement
4336 functions, or transitive irrevocable functions. For indirect
4337 functions, we'll ask the runtime. */
4338 if (TREE_CODE (fn
) == ADDR_EXPR
)
4340 struct tm_ipa_cg_data
*d
;
4341 struct cgraph_node
*node
;
4343 fn
= TREE_OPERAND (fn
, 0);
4344 if (is_tm_ending_fndecl (fn
))
4346 if (find_tm_replacement_function (fn
))
4349 node
= cgraph_node::get (fn
);
4350 d
= get_cg_data (&node
, true);
4352 /* Return true if irrevocable, but above all, believe
4354 if (d
->is_irrevocable
4355 && !is_tm_safe_or_pure (fn
))
4362 /* ??? The Approved Method of indicating that an inline
4363 assembly statement is not relevant to the transaction
4364 is to wrap it in a __tm_waiver block. This is not
4365 yet implemented, so we can't check for it. */
4366 if (is_tm_safe (current_function_decl
))
4368 tree t
= build1 (NOP_EXPR
, void_type_node
, size_zero_node
);
4369 SET_EXPR_LOCATION (t
, gimple_location (stmt
));
4370 error ("%Kasm not allowed in %<transaction_safe%> function", t
);
4382 /* For each of the blocks seeded witin PQUEUE, walk the CFG looking
4383 for new irrevocable blocks, marking them in NEW_IRR. Don't bother
4384 scanning past OLD_IRR or EXIT_BLOCKS. */
4387 ipa_tm_scan_irr_blocks (vec
<basic_block
> *pqueue
, bitmap new_irr
,
4388 bitmap old_irr
, bitmap exit_blocks
)
4390 bool any_new_irr
= false;
4393 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4397 basic_block bb
= pqueue
->pop ();
4399 /* Don't re-scan blocks we know already are irrevocable. */
4400 if (old_irr
&& bitmap_bit_p (old_irr
, bb
->index
))
4403 if (ipa_tm_scan_irr_block (bb
))
4405 bitmap_set_bit (new_irr
, bb
->index
);
4408 else if (exit_blocks
== NULL
|| !bitmap_bit_p (exit_blocks
, bb
->index
))
4410 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4411 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
4413 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
4414 pqueue
->safe_push (e
->dest
);
4418 while (!pqueue
->is_empty ());
4420 BITMAP_FREE (visited_blocks
);
4425 /* Propagate the irrevocable property both up and down the dominator tree.
4426 BB is the current block being scanned; EXIT_BLOCKS are the edges of the
4427 TM regions; OLD_IRR are the results of a previous scan of the dominator
4428 tree which has been fully propagated; NEW_IRR is the set of new blocks
4429 which are gaining the irrevocable property during the current scan. */
4432 ipa_tm_propagate_irr (basic_block entry_block
, bitmap new_irr
,
4433 bitmap old_irr
, bitmap exit_blocks
)
4435 vec
<basic_block
> bbs
;
4436 bitmap all_region_blocks
;
4438 /* If this block is in the old set, no need to rescan. */
4439 if (old_irr
&& bitmap_bit_p (old_irr
, entry_block
->index
))
4442 all_region_blocks
= BITMAP_ALLOC (&tm_obstack
);
4443 bbs
= get_tm_region_blocks (entry_block
, exit_blocks
, NULL
,
4444 all_region_blocks
, false);
4447 basic_block bb
= bbs
.pop ();
4448 bool this_irr
= bitmap_bit_p (new_irr
, bb
->index
);
4449 bool all_son_irr
= false;
4453 /* Propagate up. If my children are, I am too, but we must have
4454 at least one child that is. */
4457 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4459 if (!bitmap_bit_p (new_irr
, e
->dest
->index
))
4461 all_son_irr
= false;
4469 /* Add block to new_irr if it hasn't already been processed. */
4470 if (!old_irr
|| !bitmap_bit_p (old_irr
, bb
->index
))
4472 bitmap_set_bit (new_irr
, bb
->index
);
4478 /* Propagate down to everyone we immediately dominate. */
4482 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
4484 son
= next_dom_son (CDI_DOMINATORS
, son
))
4486 /* Make sure block is actually in a TM region, and it
4487 isn't already in old_irr. */
4488 if ((!old_irr
|| !bitmap_bit_p (old_irr
, son
->index
))
4489 && bitmap_bit_p (all_region_blocks
, son
->index
))
4490 bitmap_set_bit (new_irr
, son
->index
);
4494 while (!bbs
.is_empty ());
4496 BITMAP_FREE (all_region_blocks
);
4501 ipa_tm_decrement_clone_counts (basic_block bb
, bool for_clone
)
4503 gimple_stmt_iterator gsi
;
4505 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4507 gimple stmt
= gsi_stmt (gsi
);
4508 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4510 tree fndecl
= gimple_call_fndecl (stmt
);
4513 struct tm_ipa_cg_data
*d
;
4515 struct cgraph_node
*tnode
;
4517 if (is_tm_ending_fndecl (fndecl
))
4519 if (find_tm_replacement_function (fndecl
))
4522 tnode
= cgraph_node::get (fndecl
);
4523 d
= get_cg_data (&tnode
, true);
4525 pcallers
= (for_clone
? &d
->tm_callers_clone
4526 : &d
->tm_callers_normal
);
4528 gcc_assert (*pcallers
> 0);
4535 /* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions,
4536 as well as other irrevocable actions such as inline assembly. Mark all
4537 such blocks as irrevocable and decrement the number of calls to
4538 transactional clones. Return true if, for the transactional clone, the
4539 entire function is irrevocable. */
4542 ipa_tm_scan_irr_function (struct cgraph_node
*node
, bool for_clone
)
4544 struct tm_ipa_cg_data
*d
;
4545 bitmap new_irr
, old_irr
;
4548 /* Builtin operators (operator new, and such). */
4549 if (DECL_STRUCT_FUNCTION (node
->decl
) == NULL
4550 || DECL_STRUCT_FUNCTION (node
->decl
)->cfg
== NULL
)
4553 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
4554 calculate_dominance_info (CDI_DOMINATORS
);
4556 d
= get_cg_data (&node
, true);
4557 auto_vec
<basic_block
, 10> queue
;
4558 new_irr
= BITMAP_ALLOC (&tm_obstack
);
4560 /* Scan each tm region, propagating irrevocable status through the tree. */
4563 old_irr
= d
->irrevocable_blocks_clone
;
4564 queue
.quick_push (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)));
4565 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
, NULL
))
4567 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
4570 ret
= bitmap_bit_p (new_irr
,
4571 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
))->index
);
4576 struct tm_region
*region
;
4578 old_irr
= d
->irrevocable_blocks_normal
;
4579 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
4581 queue
.quick_push (region
->entry_block
);
4582 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
,
4583 region
->exit_blocks
))
4584 ipa_tm_propagate_irr (region
->entry_block
, new_irr
, old_irr
,
4585 region
->exit_blocks
);
4589 /* If we found any new irrevocable blocks, reduce the call count for
4590 transactional clones within the irrevocable blocks. Save the new
4591 set of irrevocable blocks for next time. */
4592 if (!bitmap_empty_p (new_irr
))
4594 bitmap_iterator bmi
;
4597 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4598 ipa_tm_decrement_clone_counts (BASIC_BLOCK_FOR_FN (cfun
, i
),
4603 bitmap_ior_into (old_irr
, new_irr
);
4604 BITMAP_FREE (new_irr
);
4607 d
->irrevocable_blocks_clone
= new_irr
;
4609 d
->irrevocable_blocks_normal
= new_irr
;
4611 if (dump_file
&& new_irr
)
4614 bitmap_iterator bmi
;
4617 dname
= lang_hooks
.decl_printable_name (current_function_decl
, 2);
4618 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4619 fprintf (dump_file
, "%s: bb %d goes irrevocable\n", dname
, i
);
4623 BITMAP_FREE (new_irr
);
4630 /* Return true if, for the transactional clone of NODE, any call
4631 may enter irrevocable mode. */
4634 ipa_tm_mayenterirr_function (struct cgraph_node
*node
)
4636 struct tm_ipa_cg_data
*d
;
4640 d
= get_cg_data (&node
, true);
4642 flags
= flags_from_decl_or_type (decl
);
4644 /* Handle some TM builtins. Ordinarily these aren't actually generated
4645 at this point, but handling these functions when written in by the
4646 user makes it easier to build unit tests. */
4647 if (flags
& ECF_TM_BUILTIN
)
4650 /* Filter out all functions that are marked. */
4651 if (flags
& ECF_TM_PURE
)
4653 if (is_tm_safe (decl
))
4655 if (is_tm_irrevocable (decl
))
4657 if (is_tm_callable (decl
))
4659 if (find_tm_replacement_function (decl
))
4662 /* If we aren't seeing the final version of the function we don't
4663 know what it will contain at runtime. */
4664 if (node
->get_availability () < AVAIL_AVAILABLE
)
4667 /* If the function must go irrevocable, then of course true. */
4668 if (d
->is_irrevocable
)
4671 /* If there are any blocks marked irrevocable, then the function
4672 as a whole may enter irrevocable. */
4673 if (d
->irrevocable_blocks_clone
)
4676 /* We may have previously marked this function as tm_may_enter_irr;
4677 see pass_diagnose_tm_blocks. */
4678 if (node
->local
.tm_may_enter_irr
)
4681 /* Recurse on the main body for aliases. In general, this will
4682 result in one of the bits above being set so that we will not
4683 have to recurse next time. */
4685 return ipa_tm_mayenterirr_function (cgraph_node::get (node
->thunk
.alias
));
4687 /* What remains is unmarked local functions without items that force
4688 the function to go irrevocable. */
4692 /* Diagnose calls from transaction_safe functions to unmarked
4693 functions that are determined to not be safe. */
4696 ipa_tm_diagnose_tm_safe (struct cgraph_node
*node
)
4698 struct cgraph_edge
*e
;
4700 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4701 if (!is_tm_callable (e
->callee
->decl
)
4702 && e
->callee
->local
.tm_may_enter_irr
)
4703 error_at (gimple_location (e
->call_stmt
),
4704 "unsafe function call %qD within "
4705 "%<transaction_safe%> function", e
->callee
->decl
);
4708 /* Diagnose call from atomic transactions to unmarked functions
4709 that are determined to not be safe. */
4712 ipa_tm_diagnose_transaction (struct cgraph_node
*node
,
4713 struct tm_region
*all_tm_regions
)
4715 struct tm_region
*r
;
4717 for (r
= all_tm_regions
; r
; r
= r
->next
)
4718 if (gimple_transaction_subcode (r
->get_transaction_stmt ())
4721 /* Atomic transactions can be nested inside relaxed. */
4723 ipa_tm_diagnose_transaction (node
, r
->inner
);
4727 vec
<basic_block
> bbs
;
4728 gimple_stmt_iterator gsi
;
4732 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
,
4733 r
->irr_blocks
, NULL
, false);
4735 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
4736 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4738 gimple stmt
= gsi_stmt (gsi
);
4741 if (gimple_code (stmt
) == GIMPLE_ASM
)
4743 error_at (gimple_location (stmt
),
4744 "asm not allowed in atomic transaction");
4748 if (!is_gimple_call (stmt
))
4750 fndecl
= gimple_call_fndecl (stmt
);
4752 /* Indirect function calls have been diagnosed already. */
4756 /* Stop at the end of the transaction. */
4757 if (is_tm_ending_fndecl (fndecl
))
4759 if (bitmap_bit_p (r
->exit_blocks
, bb
->index
))
4764 /* Marked functions have been diagnosed already. */
4765 if (is_tm_pure_call (stmt
))
4767 if (is_tm_callable (fndecl
))
4770 if (cgraph_node::local_info (fndecl
)->tm_may_enter_irr
)
4771 error_at (gimple_location (stmt
),
4772 "unsafe function call %qD within "
4773 "atomic transaction", fndecl
);
4780 /* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in
4781 OLD_DECL. The returned value is a freshly malloced pointer that
4782 should be freed by the caller. */
4785 tm_mangle (tree old_asm_id
)
4787 const char *old_asm_name
;
4790 struct demangle_component
*dc
;
4793 /* Determine if the symbol is already a valid C++ mangled name. Do this
4794 even for C, which might be interfacing with C++ code via appropriately
4795 ugly identifiers. */
4796 /* ??? We could probably do just as well checking for "_Z" and be done. */
4797 old_asm_name
= IDENTIFIER_POINTER (old_asm_id
);
4798 dc
= cplus_demangle_v3_components (old_asm_name
, DMGL_NO_OPTS
, &alloc
);
4805 sprintf (length
, "%u", IDENTIFIER_LENGTH (old_asm_id
));
4806 tm_name
= concat ("_ZGTt", length
, old_asm_name
, NULL
);
4810 old_asm_name
+= 2; /* Skip _Z */
4814 case DEMANGLE_COMPONENT_TRANSACTION_CLONE
:
4815 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE
:
4816 /* Don't play silly games, you! */
4819 case DEMANGLE_COMPONENT_HIDDEN_ALIAS
:
4820 /* I'd really like to know if we can ever be passed one of
4821 these from the C++ front end. The Logical Thing would
4822 seem that hidden-alias should be outer-most, so that we
4823 get hidden-alias of a transaction-clone and not vice-versa. */
4831 tm_name
= concat ("_ZGTt", old_asm_name
, NULL
);
4835 new_asm_id
= get_identifier (tm_name
);
4842 ipa_tm_mark_force_output_node (struct cgraph_node
*node
)
4844 node
->mark_force_output ();
4845 node
->analyzed
= true;
4849 ipa_tm_mark_forced_by_abi_node (struct cgraph_node
*node
)
4851 node
->forced_by_abi
= true;
4852 node
->analyzed
= true;
4855 /* Callback data for ipa_tm_create_version_alias. */
4856 struct create_version_alias_info
4858 struct cgraph_node
*old_node
;
4862 /* A subroutine of ipa_tm_create_version, called via
4863 cgraph_for_node_and_aliases. Create new tm clones for each of
4864 the existing aliases. */
4866 ipa_tm_create_version_alias (struct cgraph_node
*node
, void *data
)
4868 struct create_version_alias_info
*info
4869 = (struct create_version_alias_info
*)data
;
4870 tree old_decl
, new_decl
, tm_name
;
4871 struct cgraph_node
*new_node
;
4873 if (!node
->cpp_implicit_alias
)
4876 old_decl
= node
->decl
;
4877 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4878 new_decl
= build_decl (DECL_SOURCE_LOCATION (old_decl
),
4879 TREE_CODE (old_decl
), tm_name
,
4880 TREE_TYPE (old_decl
));
4882 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4883 SET_DECL_RTL (new_decl
, NULL
);
4885 /* Based loosely on C++'s make_alias_for(). */
4886 TREE_PUBLIC (new_decl
) = TREE_PUBLIC (old_decl
);
4887 DECL_CONTEXT (new_decl
) = DECL_CONTEXT (old_decl
);
4888 DECL_LANG_SPECIFIC (new_decl
) = DECL_LANG_SPECIFIC (old_decl
);
4889 TREE_READONLY (new_decl
) = TREE_READONLY (old_decl
);
4890 DECL_EXTERNAL (new_decl
) = 0;
4891 DECL_ARTIFICIAL (new_decl
) = 1;
4892 TREE_ADDRESSABLE (new_decl
) = 1;
4893 TREE_USED (new_decl
) = 1;
4894 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4896 /* Perform the same remapping to the comdat group. */
4897 if (DECL_ONE_ONLY (new_decl
))
4898 varpool_node::get (new_decl
)->set_comdat_group
4899 (tm_mangle (decl_comdat_group_id (old_decl
)));
4901 new_node
= cgraph_node::create_same_body_alias (new_decl
, info
->new_decl
);
4902 new_node
->tm_clone
= true;
4903 new_node
->externally_visible
= info
->old_node
->externally_visible
;
4904 new_node
->no_reorder
= info
->old_node
->no_reorder
;
4905 /* ?? Do not traverse aliases here. */
4906 get_cg_data (&node
, false)->clone
= new_node
;
4908 record_tm_clone_pair (old_decl
, new_decl
);
4910 if (info
->old_node
->force_output
4911 || info
->old_node
->ref_list
.first_referring ())
4912 ipa_tm_mark_force_output_node (new_node
);
4913 if (info
->old_node
->forced_by_abi
)
4914 ipa_tm_mark_forced_by_abi_node (new_node
);
4918 /* Create a copy of the function (possibly declaration only) of OLD_NODE,
4919 appropriate for the transactional clone. */
4922 ipa_tm_create_version (struct cgraph_node
*old_node
)
4924 tree new_decl
, old_decl
, tm_name
;
4925 struct cgraph_node
*new_node
;
4927 old_decl
= old_node
->decl
;
4928 new_decl
= copy_node (old_decl
);
4930 /* DECL_ASSEMBLER_NAME needs to be set before we call
4931 cgraph_copy_node_for_versioning below, because cgraph_node will
4932 fill the assembler_name_hash. */
4933 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4934 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4935 SET_DECL_RTL (new_decl
, NULL
);
4936 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4938 /* Perform the same remapping to the comdat group. */
4939 if (DECL_ONE_ONLY (new_decl
))
4940 varpool_node::get (new_decl
)->set_comdat_group
4941 (tm_mangle (DECL_COMDAT_GROUP (old_decl
)));
4943 gcc_assert (!old_node
->ipa_transforms_to_apply
.exists ());
4944 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
4945 new_node
->local
.local
= false;
4946 new_node
->externally_visible
= old_node
->externally_visible
;
4947 new_node
->lowered
= true;
4948 new_node
->tm_clone
= 1;
4949 if (!old_node
->implicit_section
)
4950 new_node
->set_section (old_node
->get_section ());
4951 get_cg_data (&old_node
, true)->clone
= new_node
;
4953 if (old_node
->get_availability () >= AVAIL_INTERPOSABLE
)
4955 /* Remap extern inline to static inline. */
4956 /* ??? Is it worth trying to use make_decl_one_only? */
4957 if (DECL_DECLARED_INLINE_P (new_decl
) && DECL_EXTERNAL (new_decl
))
4959 DECL_EXTERNAL (new_decl
) = 0;
4960 TREE_PUBLIC (new_decl
) = 0;
4961 DECL_WEAK (new_decl
) = 0;
4964 tree_function_versioning (old_decl
, new_decl
,
4969 record_tm_clone_pair (old_decl
, new_decl
);
4971 symtab
->call_cgraph_insertion_hooks (new_node
);
4972 if (old_node
->force_output
4973 || old_node
->ref_list
.first_referring ())
4974 ipa_tm_mark_force_output_node (new_node
);
4975 if (old_node
->forced_by_abi
)
4976 ipa_tm_mark_forced_by_abi_node (new_node
);
4978 /* Do the same thing, but for any aliases of the original node. */
4980 struct create_version_alias_info data
;
4981 data
.old_node
= old_node
;
4982 data
.new_decl
= new_decl
;
4983 old_node
->call_for_symbol_thunks_and_aliases (ipa_tm_create_version_alias
,
4988 /* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */
4991 ipa_tm_insert_irr_call (struct cgraph_node
*node
, struct tm_region
*region
,
4994 gimple_stmt_iterator gsi
;
4997 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4999 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
),
5000 1, build_int_cst (NULL_TREE
, MODE_SERIALIRREVOCABLE
));
5002 split_block_after_labels (bb
);
5003 gsi
= gsi_after_labels (bb
);
5004 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
5006 node
->create_edge (cgraph_node::get_create
5007 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
)),
5009 compute_call_stmt_bb_frequency (node
->decl
,
5013 /* Construct a call to TM_GETTMCLONE and insert it before GSI. */
5016 ipa_tm_insert_gettmclone_call (struct cgraph_node
*node
,
5017 struct tm_region
*region
,
5018 gimple_stmt_iterator
*gsi
, gcall
*stmt
)
5020 tree gettm_fn
, ret
, old_fn
, callfn
;
5025 old_fn
= gimple_call_fn (stmt
);
5027 if (TREE_CODE (old_fn
) == ADDR_EXPR
)
5029 tree fndecl
= TREE_OPERAND (old_fn
, 0);
5030 tree clone
= get_tm_clone_pair (fndecl
);
5032 /* By transforming the call into a TM_GETTMCLONE, we are
5033 technically taking the address of the original function and
5034 its clone. Explain this so inlining will know this function
5036 cgraph_node::get (fndecl
)->mark_address_taken () ;
5038 cgraph_node::get (clone
)->mark_address_taken ();
5041 safe
= is_tm_safe (TREE_TYPE (old_fn
));
5042 gettm_fn
= builtin_decl_explicit (safe
? BUILT_IN_TM_GETTMCLONE_SAFE
5043 : BUILT_IN_TM_GETTMCLONE_IRR
);
5044 ret
= create_tmp_var (ptr_type_node
);
5047 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
5049 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */
5050 if (TREE_CODE (old_fn
) == OBJ_TYPE_REF
)
5051 old_fn
= OBJ_TYPE_REF_EXPR (old_fn
);
5053 g
= gimple_build_call (gettm_fn
, 1, old_fn
);
5054 ret
= make_ssa_name (ret
, g
);
5055 gimple_call_set_lhs (g
, ret
);
5057 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
5059 node
->create_edge (cgraph_node::get_create (gettm_fn
), g
, 0,
5060 compute_call_stmt_bb_frequency (node
->decl
,
5063 /* Cast return value from tm_gettmclone* into appropriate function
5065 callfn
= create_tmp_var (TREE_TYPE (old_fn
));
5066 g2
= gimple_build_assign (callfn
,
5067 fold_build1 (NOP_EXPR
, TREE_TYPE (callfn
), ret
));
5068 callfn
= make_ssa_name (callfn
, g2
);
5069 gimple_assign_set_lhs (g2
, callfn
);
5070 gsi_insert_before (gsi
, g2
, GSI_SAME_STMT
);
5072 /* ??? This is a hack to preserve the NOTHROW bit on the call,
5073 which we would have derived from the decl. Failure to save
5074 this bit means we might have to split the basic block. */
5075 if (gimple_call_nothrow_p (stmt
))
5076 gimple_call_set_nothrow (stmt
, true);
5078 gimple_call_set_fn (stmt
, callfn
);
5080 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS
5081 for a call statement. Fix it. */
5083 tree lhs
= gimple_call_lhs (stmt
);
5084 tree rettype
= TREE_TYPE (gimple_call_fntype (stmt
));
5086 && !useless_type_conversion_p (TREE_TYPE (lhs
), rettype
))
5090 temp
= create_tmp_reg (rettype
);
5091 gimple_call_set_lhs (stmt
, temp
);
5093 g2
= gimple_build_assign (lhs
,
5094 fold_build1 (VIEW_CONVERT_EXPR
,
5095 TREE_TYPE (lhs
), temp
));
5096 gsi_insert_after (gsi
, g2
, GSI_SAME_STMT
);
5101 cgraph_edge
*e
= cgraph_node::get (current_function_decl
)->get_edge (stmt
);
5102 if (e
&& e
->indirect_info
)
5103 e
->indirect_info
->polymorphic
= false;
5108 /* Helper function for ipa_tm_transform_calls*. Given a call
5109 statement in GSI which resides inside transaction REGION, redirect
5110 the call to either its wrapper function, or its clone. */
5113 ipa_tm_transform_calls_redirect (struct cgraph_node
*node
,
5114 struct tm_region
*region
,
5115 gimple_stmt_iterator
*gsi
,
5116 bool *need_ssa_rename_p
)
5118 gcall
*stmt
= as_a
<gcall
*> (gsi_stmt (*gsi
));
5119 struct cgraph_node
*new_node
;
5120 struct cgraph_edge
*e
= node
->get_edge (stmt
);
5121 tree fndecl
= gimple_call_fndecl (stmt
);
5123 /* For indirect calls, pass the address through the runtime. */
5126 *need_ssa_rename_p
|=
5127 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5131 /* Handle some TM builtins. Ordinarily these aren't actually generated
5132 at this point, but handling these functions when written in by the
5133 user makes it easier to build unit tests. */
5134 if (flags_from_decl_or_type (fndecl
) & ECF_TM_BUILTIN
)
5137 /* Fixup recursive calls inside clones. */
5138 /* ??? Why did cgraph_copy_node_for_versioning update the call edges
5139 for recursion but not update the call statements themselves? */
5140 if (e
->caller
== e
->callee
&& decl_is_tm_clone (current_function_decl
))
5142 gimple_call_set_fndecl (stmt
, current_function_decl
);
5146 /* If there is a replacement, use it. */
5147 fndecl
= find_tm_replacement_function (fndecl
);
5150 new_node
= cgraph_node::get_create (fndecl
);
5152 /* ??? Mark all transaction_wrap functions tm_may_enter_irr.
5154 We can't do this earlier in record_tm_replacement because
5155 cgraph_remove_unreachable_nodes is called before we inject
5156 references to the node. Further, we can't do this in some
5157 nice central place in ipa_tm_execute because we don't have
5158 the exact list of wrapper functions that would be used.
5159 Marking more wrappers than necessary results in the creation
5160 of unnecessary cgraph_nodes, which can cause some of the
5161 other IPA passes to crash.
5163 We do need to mark these nodes so that we get the proper
5164 result in expand_call_tm. */
5165 /* ??? This seems broken. How is it that we're marking the
5166 CALLEE as may_enter_irr? Surely we should be marking the
5167 CALLER. Also note that find_tm_replacement_function also
5168 contains mappings into the TM runtime, e.g. memcpy. These
5169 we know won't go irrevocable. */
5170 new_node
->local
.tm_may_enter_irr
= 1;
5174 struct tm_ipa_cg_data
*d
;
5175 struct cgraph_node
*tnode
= e
->callee
;
5177 d
= get_cg_data (&tnode
, true);
5178 new_node
= d
->clone
;
5180 /* As we've already skipped pure calls and appropriate builtins,
5181 and we've already marked irrevocable blocks, if we can't come
5182 up with a static replacement, then ask the runtime. */
5183 if (new_node
== NULL
)
5185 *need_ssa_rename_p
|=
5186 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5190 fndecl
= new_node
->decl
;
5193 e
->redirect_callee (new_node
);
5194 gimple_call_set_fndecl (stmt
, fndecl
);
5197 /* Helper function for ipa_tm_transform_calls. For a given BB,
5198 install calls to tm_irrevocable when IRR_BLOCKS are reached,
5199 redirect other calls to the generated transactional clone. */
5202 ipa_tm_transform_calls_1 (struct cgraph_node
*node
, struct tm_region
*region
,
5203 basic_block bb
, bitmap irr_blocks
)
5205 gimple_stmt_iterator gsi
;
5206 bool need_ssa_rename
= false;
5208 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5210 ipa_tm_insert_irr_call (node
, region
, bb
);
5214 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5216 gimple stmt
= gsi_stmt (gsi
);
5218 if (!is_gimple_call (stmt
))
5220 if (is_tm_pure_call (stmt
))
5223 /* Redirect edges to the appropriate replacement or clone. */
5224 ipa_tm_transform_calls_redirect (node
, region
, &gsi
, &need_ssa_rename
);
5227 return need_ssa_rename
;
5230 /* Walk the CFG for REGION, beginning at BB. Install calls to
5231 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to
5232 the generated transactional clone. */
5235 ipa_tm_transform_calls (struct cgraph_node
*node
, struct tm_region
*region
,
5236 basic_block bb
, bitmap irr_blocks
)
5238 bool need_ssa_rename
= false;
5241 auto_vec
<basic_block
> queue
;
5242 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
5244 queue
.safe_push (bb
);
5250 ipa_tm_transform_calls_1 (node
, region
, bb
, irr_blocks
);
5252 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5255 if (region
&& bitmap_bit_p (region
->exit_blocks
, bb
->index
))
5258 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
5259 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
5261 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
5262 queue
.safe_push (e
->dest
);
5265 while (!queue
.is_empty ());
5267 BITMAP_FREE (visited_blocks
);
5269 return need_ssa_rename
;
5272 /* Transform the calls within the TM regions within NODE. */
5275 ipa_tm_transform_transaction (struct cgraph_node
*node
)
5277 struct tm_ipa_cg_data
*d
;
5278 struct tm_region
*region
;
5279 bool need_ssa_rename
= false;
5281 d
= get_cg_data (&node
, true);
5283 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5284 calculate_dominance_info (CDI_DOMINATORS
);
5286 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
5288 /* If we're sure to go irrevocable, don't transform anything. */
5289 if (d
->irrevocable_blocks_normal
5290 && bitmap_bit_p (d
->irrevocable_blocks_normal
,
5291 region
->entry_block
->index
))
5293 transaction_subcode_ior (region
, GTMA_DOES_GO_IRREVOCABLE
5294 | GTMA_MAY_ENTER_IRREVOCABLE
5295 | GTMA_HAS_NO_INSTRUMENTATION
);
5300 ipa_tm_transform_calls (node
, region
, region
->entry_block
,
5301 d
->irrevocable_blocks_normal
);
5304 if (need_ssa_rename
)
5305 update_ssa (TODO_update_ssa_only_virtuals
);
5310 /* Transform the calls within the transactional clone of NODE. */
5313 ipa_tm_transform_clone (struct cgraph_node
*node
)
5315 struct tm_ipa_cg_data
*d
;
5316 bool need_ssa_rename
;
5318 d
= get_cg_data (&node
, true);
5320 /* If this function makes no calls and has no irrevocable blocks,
5321 then there's nothing to do. */
5322 /* ??? Remove non-aborting top-level transactions. */
5323 if (!node
->callees
&& !node
->indirect_calls
&& !d
->irrevocable_blocks_clone
)
5326 push_cfun (DECL_STRUCT_FUNCTION (d
->clone
->decl
));
5327 calculate_dominance_info (CDI_DOMINATORS
);
5330 ipa_tm_transform_calls (d
->clone
, NULL
,
5331 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
5332 d
->irrevocable_blocks_clone
);
5334 if (need_ssa_rename
)
5335 update_ssa (TODO_update_ssa_only_virtuals
);
5340 /* Main entry point for the transactional memory IPA pass. */
5343 ipa_tm_execute (void)
5345 cgraph_node_queue tm_callees
= cgraph_node_queue ();
5346 /* List of functions that will go irrevocable. */
5347 cgraph_node_queue irr_worklist
= cgraph_node_queue ();
5349 struct cgraph_node
*node
;
5350 struct tm_ipa_cg_data
*d
;
5351 enum availability a
;
5354 #ifdef ENABLE_CHECKING
5355 cgraph_node::verify_cgraph_nodes ();
5358 bitmap_obstack_initialize (&tm_obstack
);
5359 initialize_original_copy_tables ();
5361 /* For all local functions marked tm_callable, queue them. */
5362 FOR_EACH_DEFINED_FUNCTION (node
)
5363 if (is_tm_callable (node
->decl
)
5364 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5366 d
= get_cg_data (&node
, true);
5367 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5370 /* For all local reachable functions... */
5371 FOR_EACH_DEFINED_FUNCTION (node
)
5373 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5375 /* ... marked tm_pure, record that fact for the runtime by
5376 indicating that the pure function is its own tm_callable.
5377 No need to do this if the function's address can't be taken. */
5378 if (is_tm_pure (node
->decl
))
5380 if (!node
->local
.local
)
5381 record_tm_clone_pair (node
->decl
, node
->decl
);
5385 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5386 calculate_dominance_info (CDI_DOMINATORS
);
5388 tm_region_init (NULL
);
5391 d
= get_cg_data (&node
, true);
5393 /* Scan for calls that are in each transaction, and
5394 generate the uninstrumented code path. */
5395 ipa_tm_scan_calls_transaction (d
, &tm_callees
);
5397 /* Put it in the worklist so we can scan the function
5398 later (ipa_tm_scan_irr_function) and mark the
5399 irrevocable blocks. */
5400 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5401 d
->want_irr_scan_normal
= true;
5407 /* For every local function on the callee list, scan as if we will be
5408 creating a transactional clone, queueing all new functions we find
5410 for (i
= 0; i
< tm_callees
.length (); ++i
)
5412 node
= tm_callees
[i
];
5413 a
= node
->get_availability ();
5414 d
= get_cg_data (&node
, true);
5416 /* Put it in the worklist so we can scan the function later
5417 (ipa_tm_scan_irr_function) and mark the irrevocable
5419 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5421 /* Some callees cannot be arbitrarily cloned. These will always be
5422 irrevocable. Mark these now, so that we need not scan them. */
5423 if (is_tm_irrevocable (node
->decl
))
5424 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5425 else if (a
<= AVAIL_NOT_AVAILABLE
5426 && !is_tm_safe_or_pure (node
->decl
))
5427 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5428 else if (a
>= AVAIL_INTERPOSABLE
)
5430 if (!tree_versionable_function_p (node
->decl
))
5431 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5432 else if (!d
->is_irrevocable
)
5434 /* If this is an alias, make sure its base is queued as well.
5435 we need not scan the callees now, as the base will do. */
5438 node
= cgraph_node::get (node
->thunk
.alias
);
5439 d
= get_cg_data (&node
, true);
5440 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5444 /* Add all nodes called by this function into
5445 tm_callees as well. */
5446 ipa_tm_scan_calls_clone (node
, &tm_callees
);
5451 /* Iterate scans until no more work to be done. Prefer not to use
5452 vec::pop because the worklist tends to follow a breadth-first
5453 search of the callgraph, which should allow convergance with a
5454 minimum number of scans. But we also don't want the worklist
5455 array to grow without bound, so we shift the array up periodically. */
5456 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5458 if (i
> 256 && i
== irr_worklist
.length () / 8)
5460 irr_worklist
.block_remove (0, i
);
5464 node
= irr_worklist
[i
];
5465 d
= get_cg_data (&node
, true);
5466 d
->in_worklist
= false;
5468 if (d
->want_irr_scan_normal
)
5470 d
->want_irr_scan_normal
= false;
5471 ipa_tm_scan_irr_function (node
, false);
5473 if (d
->in_callee_queue
&& ipa_tm_scan_irr_function (node
, true))
5474 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5477 /* For every function on the callee list, collect the tm_may_enter_irr
5479 irr_worklist
.truncate (0);
5480 for (i
= 0; i
< tm_callees
.length (); ++i
)
5482 node
= tm_callees
[i
];
5483 if (ipa_tm_mayenterirr_function (node
))
5485 d
= get_cg_data (&node
, true);
5486 gcc_assert (d
->in_worklist
== false);
5487 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5491 /* Propagate the tm_may_enter_irr bit to callers until stable. */
5492 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5494 struct cgraph_node
*caller
;
5495 struct cgraph_edge
*e
;
5496 struct ipa_ref
*ref
;
5498 if (i
> 256 && i
== irr_worklist
.length () / 8)
5500 irr_worklist
.block_remove (0, i
);
5504 node
= irr_worklist
[i
];
5505 d
= get_cg_data (&node
, true);
5506 d
->in_worklist
= false;
5507 node
->local
.tm_may_enter_irr
= true;
5509 /* Propagate back to normal callers. */
5510 for (e
= node
->callers
; e
; e
= e
->next_caller
)
5513 if (!is_tm_safe_or_pure (caller
->decl
)
5514 && !caller
->local
.tm_may_enter_irr
)
5516 d
= get_cg_data (&caller
, true);
5517 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5521 /* Propagate back to referring aliases as well. */
5522 FOR_EACH_ALIAS (node
, ref
)
5524 caller
= dyn_cast
<cgraph_node
*> (ref
->referring
);
5525 if (!caller
->local
.tm_may_enter_irr
)
5527 /* ?? Do not traverse aliases here. */
5528 d
= get_cg_data (&caller
, false);
5529 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5534 /* Now validate all tm_safe functions, and all atomic regions in
5536 FOR_EACH_DEFINED_FUNCTION (node
)
5538 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5540 d
= get_cg_data (&node
, true);
5541 if (is_tm_safe (node
->decl
))
5542 ipa_tm_diagnose_tm_safe (node
);
5543 else if (d
->all_tm_regions
)
5544 ipa_tm_diagnose_transaction (node
, d
->all_tm_regions
);
5547 /* Create clones. Do those that are not irrevocable and have a
5548 positive call count. Do those publicly visible functions that
5549 the user directed us to clone. */
5550 for (i
= 0; i
< tm_callees
.length (); ++i
)
5554 node
= tm_callees
[i
];
5555 if (node
->cpp_implicit_alias
)
5558 a
= node
->get_availability ();
5559 d
= get_cg_data (&node
, true);
5561 if (a
<= AVAIL_NOT_AVAILABLE
)
5562 doit
= is_tm_callable (node
->decl
);
5563 else if (a
<= AVAIL_AVAILABLE
&& is_tm_callable (node
->decl
))
5565 else if (!d
->is_irrevocable
5566 && d
->tm_callers_normal
+ d
->tm_callers_clone
> 0)
5570 ipa_tm_create_version (node
);
5573 /* Redirect calls to the new clones, and insert irrevocable marks. */
5574 for (i
= 0; i
< tm_callees
.length (); ++i
)
5576 node
= tm_callees
[i
];
5579 d
= get_cg_data (&node
, true);
5581 ipa_tm_transform_clone (node
);
5584 FOR_EACH_DEFINED_FUNCTION (node
)
5586 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5588 d
= get_cg_data (&node
, true);
5589 if (d
->all_tm_regions
)
5590 ipa_tm_transform_transaction (node
);
5593 /* Free and clear all data structures. */
5594 tm_callees
.release ();
5595 irr_worklist
.release ();
5596 bitmap_obstack_release (&tm_obstack
);
5597 free_original_copy_tables ();
5599 FOR_EACH_FUNCTION (node
)
5602 #ifdef ENABLE_CHECKING
5603 cgraph_node::verify_cgraph_nodes ();
5611 const pass_data pass_data_ipa_tm
=
5613 SIMPLE_IPA_PASS
, /* type */
5615 OPTGROUP_NONE
, /* optinfo_flags */
5616 TV_TRANS_MEM
, /* tv_id */
5617 ( PROP_ssa
| PROP_cfg
), /* properties_required */
5618 0, /* properties_provided */
5619 0, /* properties_destroyed */
5620 0, /* todo_flags_start */
5621 0, /* todo_flags_finish */
5624 class pass_ipa_tm
: public simple_ipa_opt_pass
5627 pass_ipa_tm (gcc::context
*ctxt
)
5628 : simple_ipa_opt_pass (pass_data_ipa_tm
, ctxt
)
5631 /* opt_pass methods: */
5632 virtual bool gate (function
*) { return flag_tm
; }
5633 virtual unsigned int execute (function
*) { return ipa_tm_execute (); }
5635 }; // class pass_ipa_tm
5639 simple_ipa_opt_pass
*
5640 make_pass_ipa_tm (gcc::context
*ctxt
)
5642 return new pass_ipa_tm (ctxt
);
5645 #include "gt-trans-mem.h"