1 /* Passes for transactional memory support.
2 Copyright (C) 2008-2015 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
27 #include "fold-const.h"
30 #include "hard-reg-set.h"
32 #include "dominance.h"
34 #include "basic-block.h"
35 #include "tree-ssa-alias.h"
36 #include "internal-fn.h"
38 #include "gimple-expr.h"
44 #include "gimple-iterator.h"
45 #include "gimplify-me.h"
46 #include "gimple-walk.h"
47 #include "gimple-ssa.h"
48 #include "plugin-api.h"
52 #include "stringpool.h"
53 #include "tree-ssanames.h"
54 #include "tree-into-ssa.h"
55 #include "tree-pass.h"
56 #include "tree-inline.h"
57 #include "diagnostic-core.h"
60 #include "trans-mem.h"
63 #include "langhooks.h"
64 #include "gimple-pretty-print.h"
66 #include "tree-ssa-address.h"
69 #define A_RUNINSTRUMENTEDCODE 0x0001
70 #define A_RUNUNINSTRUMENTEDCODE 0x0002
71 #define A_SAVELIVEVARIABLES 0x0004
72 #define A_RESTORELIVEVARIABLES 0x0008
73 #define A_ABORTTRANSACTION 0x0010
75 #define AR_USERABORT 0x0001
76 #define AR_USERRETRY 0x0002
77 #define AR_TMCONFLICT 0x0004
78 #define AR_EXCEPTIONBLOCKABORT 0x0008
79 #define AR_OUTERABORT 0x0010
81 #define MODE_SERIALIRREVOCABLE 0x0000
84 /* The representation of a transaction changes several times during the
85 lowering process. In the beginning, in the front-end we have the
86 GENERIC tree TRANSACTION_EXPR. For example,
94 During initial gimplification (gimplify.c) the TRANSACTION_EXPR node is
95 trivially replaced with a GIMPLE_TRANSACTION node.
97 During pass_lower_tm, we examine the body of transactions looking
98 for aborts. Transactions that do not contain an abort may be
99 merged into an outer transaction. We also add a TRY-FINALLY node
100 to arrange for the transaction to be committed on any exit.
102 [??? Think about how this arrangement affects throw-with-commit
103 and throw-with-abort operations. In this case we want the TRY to
104 handle gotos, but not to catch any exceptions because the transaction
105 will already be closed.]
107 GIMPLE_TRANSACTION [label=NULL] {
114 __builtin___tm_abort ();
116 __builtin___tm_commit ();
120 During pass_lower_eh, we create EH regions for the transactions,
121 intermixed with the regular EH stuff. This gives us a nice persistent
122 mapping (all the way through rtl) from transactional memory operation
123 back to the transaction, which allows us to get the abnormal edges
124 correct to model transaction aborts and restarts:
126 GIMPLE_TRANSACTION [label=over]
132 __builtin___tm_abort ();
133 __builtin___tm_commit ();
136 This is the end of all_lowering_passes, and so is what is present
137 during the IPA passes, and through all of the optimization passes.
139 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all
140 functions and mark functions for cloning.
142 At the end of gimple optimization, before exiting SSA form,
143 pass_tm_edges replaces statements that perform transactional
144 memory operations with the appropriate TM builtins, and swap
145 out function calls with their transactional clones. At this
146 point we introduce the abnormal transaction restart edges and
147 complete lowering of the GIMPLE_TRANSACTION node.
149 x = __builtin___tm_start (MAY_ABORT);
151 if (x & abort_transaction)
154 t0 = __builtin___tm_load (global);
156 __builtin___tm_store (&global, t1);
158 __builtin___tm_abort ();
159 __builtin___tm_commit ();
163 static void *expand_regions (struct tm_region
*,
164 void *(*callback
)(struct tm_region
*, void *),
168 /* Return the attributes we want to examine for X, or NULL if it's not
169 something we examine. We look at function types, but allow pointers
170 to function types and function decls and peek through. */
173 get_attrs_for (const_tree x
)
178 switch (TREE_CODE (x
))
181 return TYPE_ATTRIBUTES (TREE_TYPE (x
));
188 if (TREE_CODE (x
) != POINTER_TYPE
)
194 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
200 return TYPE_ATTRIBUTES (x
);
204 /* Return true if X has been marked TM_PURE. */
207 is_tm_pure (const_tree x
)
211 switch (TREE_CODE (x
))
222 if (TREE_CODE (x
) != POINTER_TYPE
)
228 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
233 flags
= flags_from_decl_or_type (x
);
234 return (flags
& ECF_TM_PURE
) != 0;
237 /* Return true if X has been marked TM_IRREVOCABLE. */
240 is_tm_irrevocable (tree x
)
242 tree attrs
= get_attrs_for (x
);
244 if (attrs
&& lookup_attribute ("transaction_unsafe", attrs
))
247 /* A call to the irrevocable builtin is by definition,
249 if (TREE_CODE (x
) == ADDR_EXPR
)
250 x
= TREE_OPERAND (x
, 0);
251 if (TREE_CODE (x
) == FUNCTION_DECL
252 && DECL_BUILT_IN_CLASS (x
) == BUILT_IN_NORMAL
253 && DECL_FUNCTION_CODE (x
) == BUILT_IN_TM_IRREVOCABLE
)
259 /* Return true if X has been marked TM_SAFE. */
262 is_tm_safe (const_tree x
)
266 tree attrs
= get_attrs_for (x
);
269 if (lookup_attribute ("transaction_safe", attrs
))
271 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
278 /* Return true if CALL is const, or tm_pure. */
281 is_tm_pure_call (gimple call
)
283 tree fn
= gimple_call_fn (call
);
285 if (TREE_CODE (fn
) == ADDR_EXPR
)
287 fn
= TREE_OPERAND (fn
, 0);
288 gcc_assert (TREE_CODE (fn
) == FUNCTION_DECL
);
293 return is_tm_pure (fn
);
296 /* Return true if X has been marked TM_CALLABLE. */
299 is_tm_callable (tree x
)
301 tree attrs
= get_attrs_for (x
);
304 if (lookup_attribute ("transaction_callable", attrs
))
306 if (lookup_attribute ("transaction_safe", attrs
))
308 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
314 /* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */
317 is_tm_may_cancel_outer (tree x
)
319 tree attrs
= get_attrs_for (x
);
321 return lookup_attribute ("transaction_may_cancel_outer", attrs
) != NULL
;
325 /* Return true for built in functions that "end" a transaction. */
328 is_tm_ending_fndecl (tree fndecl
)
330 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
331 switch (DECL_FUNCTION_CODE (fndecl
))
333 case BUILT_IN_TM_COMMIT
:
334 case BUILT_IN_TM_COMMIT_EH
:
335 case BUILT_IN_TM_ABORT
:
336 case BUILT_IN_TM_IRREVOCABLE
:
345 /* Return true if STMT is a built in function call that "ends" a
349 is_tm_ending (gimple stmt
)
353 if (gimple_code (stmt
) != GIMPLE_CALL
)
356 fndecl
= gimple_call_fndecl (stmt
);
357 return (fndecl
!= NULL_TREE
358 && is_tm_ending_fndecl (fndecl
));
361 /* Return true if STMT is a TM load. */
364 is_tm_load (gimple stmt
)
368 if (gimple_code (stmt
) != GIMPLE_CALL
)
371 fndecl
= gimple_call_fndecl (stmt
);
372 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
373 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl
)));
376 /* Same as above, but for simple TM loads, that is, not the
377 after-write, after-read, etc optimized variants. */
380 is_tm_simple_load (gimple stmt
)
384 if (gimple_code (stmt
) != GIMPLE_CALL
)
387 fndecl
= gimple_call_fndecl (stmt
);
388 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
390 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
391 return (fcode
== BUILT_IN_TM_LOAD_1
392 || fcode
== BUILT_IN_TM_LOAD_2
393 || fcode
== BUILT_IN_TM_LOAD_4
394 || fcode
== BUILT_IN_TM_LOAD_8
395 || fcode
== BUILT_IN_TM_LOAD_FLOAT
396 || fcode
== BUILT_IN_TM_LOAD_DOUBLE
397 || fcode
== BUILT_IN_TM_LOAD_LDOUBLE
398 || fcode
== BUILT_IN_TM_LOAD_M64
399 || fcode
== BUILT_IN_TM_LOAD_M128
400 || fcode
== BUILT_IN_TM_LOAD_M256
);
405 /* Return true if STMT is a TM store. */
408 is_tm_store (gimple stmt
)
412 if (gimple_code (stmt
) != GIMPLE_CALL
)
415 fndecl
= gimple_call_fndecl (stmt
);
416 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
417 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl
)));
420 /* Same as above, but for simple TM stores, that is, not the
421 after-write, after-read, etc optimized variants. */
424 is_tm_simple_store (gimple stmt
)
428 if (gimple_code (stmt
) != GIMPLE_CALL
)
431 fndecl
= gimple_call_fndecl (stmt
);
432 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
434 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
435 return (fcode
== BUILT_IN_TM_STORE_1
436 || fcode
== BUILT_IN_TM_STORE_2
437 || fcode
== BUILT_IN_TM_STORE_4
438 || fcode
== BUILT_IN_TM_STORE_8
439 || fcode
== BUILT_IN_TM_STORE_FLOAT
440 || fcode
== BUILT_IN_TM_STORE_DOUBLE
441 || fcode
== BUILT_IN_TM_STORE_LDOUBLE
442 || fcode
== BUILT_IN_TM_STORE_M64
443 || fcode
== BUILT_IN_TM_STORE_M128
444 || fcode
== BUILT_IN_TM_STORE_M256
);
449 /* Return true if FNDECL is BUILT_IN_TM_ABORT. */
452 is_tm_abort (tree fndecl
)
455 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
456 && DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_TM_ABORT
);
459 /* Build a GENERIC tree for a user abort. This is called by front ends
460 while transforming the __tm_abort statement. */
463 build_tm_abort_call (location_t loc
, bool is_outer
)
465 return build_call_expr_loc (loc
, builtin_decl_explicit (BUILT_IN_TM_ABORT
), 1,
466 build_int_cst (integer_type_node
,
468 | (is_outer
? AR_OUTERABORT
: 0)));
471 /* Map for aribtrary function replacement under TM, as created
472 by the tm_wrap attribute. */
474 struct tm_wrapper_hasher
: ggc_cache_hasher
<tree_map
*>
476 static inline hashval_t
hash (tree_map
*m
) { return m
->hash
; }
478 equal (tree_map
*a
, tree_map
*b
)
480 return a
->base
.from
== b
->base
.from
;
484 keep_cache_entry (tree_map
*&m
)
486 return ggc_marked_p (m
->base
.from
);
490 static GTY((cache
)) hash_table
<tm_wrapper_hasher
> *tm_wrap_map
;
493 record_tm_replacement (tree from
, tree to
)
495 struct tree_map
**slot
, *h
;
497 /* Do not inline wrapper functions that will get replaced in the TM
500 Suppose you have foo() that will get replaced into tmfoo(). Make
501 sure the inliner doesn't try to outsmart us and inline foo()
502 before we get a chance to do the TM replacement. */
503 DECL_UNINLINABLE (from
) = 1;
505 if (tm_wrap_map
== NULL
)
506 tm_wrap_map
= hash_table
<tm_wrapper_hasher
>::create_ggc (32);
508 h
= ggc_alloc
<tree_map
> ();
509 h
->hash
= htab_hash_pointer (from
);
513 slot
= tm_wrap_map
->find_slot_with_hash (h
, h
->hash
, INSERT
);
517 /* Return a TM-aware replacement function for DECL. */
520 find_tm_replacement_function (tree fndecl
)
524 struct tree_map
*h
, in
;
526 in
.base
.from
= fndecl
;
527 in
.hash
= htab_hash_pointer (fndecl
);
528 h
= tm_wrap_map
->find_with_hash (&in
, in
.hash
);
533 /* ??? We may well want TM versions of most of the common <string.h>
534 functions. For now, we've already these two defined. */
535 /* Adjust expand_call_tm() attributes as necessary for the cases
537 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
538 switch (DECL_FUNCTION_CODE (fndecl
))
540 case BUILT_IN_MEMCPY
:
541 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY
);
542 case BUILT_IN_MEMMOVE
:
543 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
);
544 case BUILT_IN_MEMSET
:
545 return builtin_decl_explicit (BUILT_IN_TM_MEMSET
);
553 /* When appropriate, record TM replacement for memory allocation functions.
555 FROM is the FNDECL to wrap. */
557 tm_malloc_replacement (tree from
)
562 if (TREE_CODE (from
) != FUNCTION_DECL
)
565 /* If we have a previous replacement, the user must be explicitly
566 wrapping malloc/calloc/free. They better know what they're
568 if (find_tm_replacement_function (from
))
571 str
= IDENTIFIER_POINTER (DECL_NAME (from
));
573 if (!strcmp (str
, "malloc"))
574 to
= builtin_decl_explicit (BUILT_IN_TM_MALLOC
);
575 else if (!strcmp (str
, "calloc"))
576 to
= builtin_decl_explicit (BUILT_IN_TM_CALLOC
);
577 else if (!strcmp (str
, "free"))
578 to
= builtin_decl_explicit (BUILT_IN_TM_FREE
);
582 TREE_NOTHROW (to
) = 0;
584 record_tm_replacement (from
, to
);
587 /* Diagnostics for tm_safe functions/regions. Called by the front end
588 once we've lowered the function to high-gimple. */
590 /* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq.
591 Process exactly one statement. WI->INFO is set to non-null when in
592 the context of a tm_safe function, and null for a __transaction block. */
594 #define DIAG_TM_OUTER 1
595 #define DIAG_TM_SAFE 2
596 #define DIAG_TM_RELAXED 4
600 unsigned int summary_flags
: 8;
601 unsigned int block_flags
: 8;
602 unsigned int func_flags
: 8;
603 unsigned int saw_volatile
: 1;
607 /* Return true if T is a volatile variable of some kind. */
610 volatile_var_p (tree t
)
612 return (SSA_VAR_P (t
)
613 && TREE_THIS_VOLATILE (TREE_TYPE (t
)));
616 /* Tree callback function for diagnose_tm pass. */
619 diagnose_tm_1_op (tree
*tp
, int *walk_subtrees ATTRIBUTE_UNUSED
,
622 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
623 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
625 if (volatile_var_p (*tp
)
626 && d
->block_flags
& DIAG_TM_SAFE
630 error_at (gimple_location (d
->stmt
),
631 "invalid volatile use of %qD inside transaction",
639 is_tm_safe_or_pure (const_tree x
)
641 return is_tm_safe (x
) || is_tm_pure (x
);
645 diagnose_tm_1 (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
646 struct walk_stmt_info
*wi
)
648 gimple stmt
= gsi_stmt (*gsi
);
649 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
651 /* Save stmt for use in leaf analysis. */
654 switch (gimple_code (stmt
))
658 tree fn
= gimple_call_fn (stmt
);
660 if ((d
->summary_flags
& DIAG_TM_OUTER
) == 0
661 && is_tm_may_cancel_outer (fn
))
662 error_at (gimple_location (stmt
),
663 "%<transaction_may_cancel_outer%> function call not within"
664 " outer transaction or %<transaction_may_cancel_outer%>");
666 if (d
->summary_flags
& DIAG_TM_SAFE
)
668 bool is_safe
, direct_call_p
;
671 if (TREE_CODE (fn
) == ADDR_EXPR
672 && TREE_CODE (TREE_OPERAND (fn
, 0)) == FUNCTION_DECL
)
674 direct_call_p
= true;
675 replacement
= TREE_OPERAND (fn
, 0);
676 replacement
= find_tm_replacement_function (replacement
);
682 direct_call_p
= false;
683 replacement
= NULL_TREE
;
686 if (is_tm_safe_or_pure (fn
))
688 else if (is_tm_callable (fn
) || is_tm_irrevocable (fn
))
690 /* A function explicitly marked transaction_callable as
691 opposed to transaction_safe is being defined to be
692 unsafe as part of its ABI, regardless of its contents. */
695 else if (direct_call_p
)
697 if (IS_TYPE_OR_DECL_P (fn
)
698 && flags_from_decl_or_type (fn
) & ECF_TM_BUILTIN
)
700 else if (replacement
)
702 /* ??? At present we've been considering replacements
703 merely transaction_callable, and therefore might
704 enter irrevocable. The tm_wrap attribute has not
705 yet made it into the new language spec. */
710 /* ??? Diagnostics for unmarked direct calls moved into
711 the IPA pass. Section 3.2 of the spec details how
712 functions not marked should be considered "implicitly
713 safe" based on having examined the function body. */
719 /* An unmarked indirect call. Consider it unsafe even
720 though optimization may yet figure out how to inline. */
726 if (TREE_CODE (fn
) == ADDR_EXPR
)
727 fn
= TREE_OPERAND (fn
, 0);
728 if (d
->block_flags
& DIAG_TM_SAFE
)
731 error_at (gimple_location (stmt
),
732 "unsafe function call %qD within "
733 "atomic transaction", fn
);
736 if (!DECL_P (fn
) || DECL_NAME (fn
))
737 error_at (gimple_location (stmt
),
738 "unsafe function call %qE within "
739 "atomic transaction", fn
);
741 error_at (gimple_location (stmt
),
742 "unsafe indirect function call within "
743 "atomic transaction");
749 error_at (gimple_location (stmt
),
750 "unsafe function call %qD within "
751 "%<transaction_safe%> function", fn
);
754 if (!DECL_P (fn
) || DECL_NAME (fn
))
755 error_at (gimple_location (stmt
),
756 "unsafe function call %qE within "
757 "%<transaction_safe%> function", fn
);
759 error_at (gimple_location (stmt
),
760 "unsafe indirect function call within "
761 "%<transaction_safe%> function");
770 /* ??? We ought to come up with a way to add attributes to
771 asm statements, and then add "transaction_safe" to it.
772 Either that or get the language spec to resurrect __tm_waiver. */
773 if (d
->block_flags
& DIAG_TM_SAFE
)
774 error_at (gimple_location (stmt
),
775 "asm not allowed in atomic transaction");
776 else if (d
->func_flags
& DIAG_TM_SAFE
)
777 error_at (gimple_location (stmt
),
778 "asm not allowed in %<transaction_safe%> function");
781 case GIMPLE_TRANSACTION
:
783 gtransaction
*trans_stmt
= as_a
<gtransaction
*> (stmt
);
784 unsigned char inner_flags
= DIAG_TM_SAFE
;
786 if (gimple_transaction_subcode (trans_stmt
) & GTMA_IS_RELAXED
)
788 if (d
->block_flags
& DIAG_TM_SAFE
)
789 error_at (gimple_location (stmt
),
790 "relaxed transaction in atomic transaction");
791 else if (d
->func_flags
& DIAG_TM_SAFE
)
792 error_at (gimple_location (stmt
),
793 "relaxed transaction in %<transaction_safe%> function");
794 inner_flags
= DIAG_TM_RELAXED
;
796 else if (gimple_transaction_subcode (trans_stmt
) & GTMA_IS_OUTER
)
799 error_at (gimple_location (stmt
),
800 "outer transaction in transaction");
801 else if (d
->func_flags
& DIAG_TM_OUTER
)
802 error_at (gimple_location (stmt
),
803 "outer transaction in "
804 "%<transaction_may_cancel_outer%> function");
805 else if (d
->func_flags
& DIAG_TM_SAFE
)
806 error_at (gimple_location (stmt
),
807 "outer transaction in %<transaction_safe%> function");
808 inner_flags
|= DIAG_TM_OUTER
;
811 *handled_ops_p
= true;
812 if (gimple_transaction_body (trans_stmt
))
814 struct walk_stmt_info wi_inner
;
815 struct diagnose_tm d_inner
;
817 memset (&d_inner
, 0, sizeof (d_inner
));
818 d_inner
.func_flags
= d
->func_flags
;
819 d_inner
.block_flags
= d
->block_flags
| inner_flags
;
820 d_inner
.summary_flags
= d_inner
.func_flags
| d_inner
.block_flags
;
822 memset (&wi_inner
, 0, sizeof (wi_inner
));
823 wi_inner
.info
= &d_inner
;
825 walk_gimple_seq (gimple_transaction_body (trans_stmt
),
826 diagnose_tm_1
, diagnose_tm_1_op
, &wi_inner
);
839 diagnose_tm_blocks (void)
841 struct walk_stmt_info wi
;
842 struct diagnose_tm d
;
844 memset (&d
, 0, sizeof (d
));
845 if (is_tm_may_cancel_outer (current_function_decl
))
846 d
.func_flags
= DIAG_TM_OUTER
| DIAG_TM_SAFE
;
847 else if (is_tm_safe (current_function_decl
))
848 d
.func_flags
= DIAG_TM_SAFE
;
849 d
.summary_flags
= d
.func_flags
;
851 memset (&wi
, 0, sizeof (wi
));
854 walk_gimple_seq (gimple_body (current_function_decl
),
855 diagnose_tm_1
, diagnose_tm_1_op
, &wi
);
862 const pass_data pass_data_diagnose_tm_blocks
=
864 GIMPLE_PASS
, /* type */
865 "*diagnose_tm_blocks", /* name */
866 OPTGROUP_NONE
, /* optinfo_flags */
867 TV_TRANS_MEM
, /* tv_id */
868 PROP_gimple_any
, /* properties_required */
869 0, /* properties_provided */
870 0, /* properties_destroyed */
871 0, /* todo_flags_start */
872 0, /* todo_flags_finish */
875 class pass_diagnose_tm_blocks
: public gimple_opt_pass
878 pass_diagnose_tm_blocks (gcc::context
*ctxt
)
879 : gimple_opt_pass (pass_data_diagnose_tm_blocks
, ctxt
)
882 /* opt_pass methods: */
883 virtual bool gate (function
*) { return flag_tm
; }
884 virtual unsigned int execute (function
*) { return diagnose_tm_blocks (); }
886 }; // class pass_diagnose_tm_blocks
891 make_pass_diagnose_tm_blocks (gcc::context
*ctxt
)
893 return new pass_diagnose_tm_blocks (ctxt
);
896 /* Instead of instrumenting thread private memory, we save the
897 addresses in a log which we later use to save/restore the addresses
898 upon transaction start/restart.
900 The log is keyed by address, where each element contains individual
901 statements among different code paths that perform the store.
903 This log is later used to generate either plain save/restore of the
904 addresses upon transaction start/restart, or calls to the ITM_L*
907 So for something like:
909 struct large { int x[1000]; };
910 struct large lala = { 0 };
916 We can either save/restore:
919 trxn = _ITM_startTransaction ();
920 if (trxn & a_saveLiveVariables)
921 tmp_lala1 = lala.x[i];
922 else if (a & a_restoreLiveVariables)
923 lala.x[i] = tmp_lala1;
925 or use the logging functions:
928 trxn = _ITM_startTransaction ();
929 _ITM_LU4 (&lala.x[i]);
931 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as
932 far up the dominator tree to shadow all of the writes to a given
933 location (thus reducing the total number of logging calls), but not
934 so high as to be called on a path that does not perform a
937 /* One individual log entry. We may have multiple statements for the
938 same location if neither dominate each other (on different
940 typedef struct tm_log_entry
942 /* Address to save. */
944 /* Entry block for the transaction this address occurs in. */
945 basic_block entry_block
;
946 /* Dominating statements the store occurs in. */
948 /* Initially, while we are building the log, we place a nonzero
949 value here to mean that this address *will* be saved with a
950 save/restore sequence. Later, when generating the save sequence
951 we place the SSA temp generated here. */
956 /* Log entry hashtable helpers. */
958 struct log_entry_hasher
960 typedef tm_log_entry
*value_type
;
961 typedef tm_log_entry
*compare_type
;
962 static inline hashval_t
hash (const tm_log_entry
*);
963 static inline bool equal (const tm_log_entry
*, const tm_log_entry
*);
964 static inline void remove (tm_log_entry
*);
967 /* Htab support. Return hash value for a `tm_log_entry'. */
969 log_entry_hasher::hash (const tm_log_entry
*log
)
971 return iterative_hash_expr (log
->addr
, 0);
974 /* Htab support. Return true if two log entries are the same. */
976 log_entry_hasher::equal (const tm_log_entry
*log1
, const tm_log_entry
*log2
)
980 rth: I suggest that we get rid of the component refs etc.
981 I.e. resolve the reference to base + offset.
983 We may need to actually finish a merge with mainline for this,
984 since we'd like to be presented with Richi's MEM_REF_EXPRs more
985 often than not. But in the meantime your tm_log_entry could save
986 the results of get_inner_reference.
988 See: g++.dg/tm/pr46653.C
991 /* Special case plain equality because operand_equal_p() below will
992 return FALSE if the addresses are equal but they have
993 side-effects (e.g. a volatile address). */
994 if (log1
->addr
== log2
->addr
)
997 return operand_equal_p (log1
->addr
, log2
->addr
, 0);
1000 /* Htab support. Free one tm_log_entry. */
1002 log_entry_hasher::remove (tm_log_entry
*lp
)
1004 lp
->stmts
.release ();
1009 /* The actual log. */
1010 static hash_table
<log_entry_hasher
> *tm_log
;
1012 /* Addresses to log with a save/restore sequence. These should be in
1014 static vec
<tree
> tm_log_save_addresses
;
1016 enum thread_memory_type
1020 mem_transaction_local
,
1024 typedef struct tm_new_mem_map
1026 /* SSA_NAME being dereferenced. */
1028 enum thread_memory_type local_new_memory
;
1031 /* Hashtable helpers. */
1033 struct tm_mem_map_hasher
: typed_free_remove
<tm_new_mem_map_t
>
1035 typedef tm_new_mem_map_t
*value_type
;
1036 typedef tm_new_mem_map_t
*compare_type
;
1037 static inline hashval_t
hash (const tm_new_mem_map_t
*);
1038 static inline bool equal (const tm_new_mem_map_t
*, const tm_new_mem_map_t
*);
1042 tm_mem_map_hasher::hash (const tm_new_mem_map_t
*v
)
1044 return (intptr_t)v
->val
>> 4;
1048 tm_mem_map_hasher::equal (const tm_new_mem_map_t
*v
, const tm_new_mem_map_t
*c
)
1050 return v
->val
== c
->val
;
1053 /* Map for an SSA_NAME originally pointing to a non aliased new piece
1054 of memory (malloc, alloc, etc). */
1055 static hash_table
<tm_mem_map_hasher
> *tm_new_mem_hash
;
1057 /* Initialize logging data structures. */
1061 tm_log
= new hash_table
<log_entry_hasher
> (10);
1062 tm_new_mem_hash
= new hash_table
<tm_mem_map_hasher
> (5);
1063 tm_log_save_addresses
.create (5);
1066 /* Free logging data structures. */
1068 tm_log_delete (void)
1072 delete tm_new_mem_hash
;
1073 tm_new_mem_hash
= NULL
;
1074 tm_log_save_addresses
.release ();
1077 /* Return true if MEM is a transaction invariant memory for the TM
1078 region starting at REGION_ENTRY_BLOCK. */
1080 transaction_invariant_address_p (const_tree mem
, basic_block region_entry_block
)
1082 if ((TREE_CODE (mem
) == INDIRECT_REF
|| TREE_CODE (mem
) == MEM_REF
)
1083 && TREE_CODE (TREE_OPERAND (mem
, 0)) == SSA_NAME
)
1087 def_bb
= gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem
, 0)));
1088 return def_bb
!= region_entry_block
1089 && dominated_by_p (CDI_DOMINATORS
, region_entry_block
, def_bb
);
1092 mem
= strip_invariant_refs (mem
);
1093 return mem
&& (CONSTANT_CLASS_P (mem
) || decl_address_invariant_p (mem
));
1096 /* Given an address ADDR in STMT, find it in the memory log or add it,
1097 making sure to keep only the addresses highest in the dominator
1100 ENTRY_BLOCK is the entry_block for the transaction.
1102 If we find the address in the log, make sure it's either the same
1103 address, or an equivalent one that dominates ADDR.
1105 If we find the address, but neither ADDR dominates the found
1106 address, nor the found one dominates ADDR, we're on different
1107 execution paths. Add it.
1109 If known, ENTRY_BLOCK is the entry block for the region, otherwise
1112 tm_log_add (basic_block entry_block
, tree addr
, gimple stmt
)
1114 tm_log_entry
**slot
;
1115 struct tm_log_entry l
, *lp
;
1118 slot
= tm_log
->find_slot (&l
, INSERT
);
1121 tree type
= TREE_TYPE (addr
);
1123 lp
= XNEW (struct tm_log_entry
);
1127 /* Small invariant addresses can be handled as save/restores. */
1129 && transaction_invariant_address_p (lp
->addr
, entry_block
)
1130 && TYPE_SIZE_UNIT (type
) != NULL
1131 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
))
1132 && ((HOST_WIDE_INT
) tree_to_uhwi (TYPE_SIZE_UNIT (type
))
1133 < PARAM_VALUE (PARAM_TM_MAX_AGGREGATE_SIZE
))
1134 /* We must be able to copy this type normally. I.e., no
1135 special constructors and the like. */
1136 && !TREE_ADDRESSABLE (type
))
1138 lp
->save_var
= create_tmp_reg (TREE_TYPE (lp
->addr
), "tm_save");
1139 lp
->stmts
.create (0);
1140 lp
->entry_block
= entry_block
;
1141 /* Save addresses separately in dominator order so we don't
1142 get confused by overlapping addresses in the save/restore
1144 tm_log_save_addresses
.safe_push (lp
->addr
);
1148 /* Use the logging functions. */
1149 lp
->stmts
.create (5);
1150 lp
->stmts
.quick_push (stmt
);
1151 lp
->save_var
= NULL
;
1161 /* If we're generating a save/restore sequence, we don't care
1162 about statements. */
1166 for (i
= 0; lp
->stmts
.iterate (i
, &oldstmt
); ++i
)
1168 if (stmt
== oldstmt
)
1170 /* We already have a store to the same address, higher up the
1171 dominator tree. Nothing to do. */
1172 if (dominated_by_p (CDI_DOMINATORS
,
1173 gimple_bb (stmt
), gimple_bb (oldstmt
)))
1175 /* We should be processing blocks in dominator tree order. */
1176 gcc_assert (!dominated_by_p (CDI_DOMINATORS
,
1177 gimple_bb (oldstmt
), gimple_bb (stmt
)));
1179 /* Store is on a different code path. */
1180 lp
->stmts
.safe_push (stmt
);
1184 /* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME
1185 result, insert the new statements before GSI. */
1188 gimplify_addr (gimple_stmt_iterator
*gsi
, tree x
)
1190 if (TREE_CODE (x
) == TARGET_MEM_REF
)
1191 x
= tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x
)), x
);
1193 x
= build_fold_addr_expr (x
);
1194 return force_gimple_operand_gsi (gsi
, x
, true, NULL
, true, GSI_SAME_STMT
);
1197 /* Instrument one address with the logging functions.
1198 ADDR is the address to save.
1199 STMT is the statement before which to place it. */
1201 tm_log_emit_stmt (tree addr
, gimple stmt
)
1203 tree type
= TREE_TYPE (addr
);
1204 tree size
= TYPE_SIZE_UNIT (type
);
1205 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
1207 enum built_in_function code
= BUILT_IN_TM_LOG
;
1209 if (type
== float_type_node
)
1210 code
= BUILT_IN_TM_LOG_FLOAT
;
1211 else if (type
== double_type_node
)
1212 code
= BUILT_IN_TM_LOG_DOUBLE
;
1213 else if (type
== long_double_type_node
)
1214 code
= BUILT_IN_TM_LOG_LDOUBLE
;
1215 else if (tree_fits_uhwi_p (size
))
1217 unsigned int n
= tree_to_uhwi (size
);
1221 code
= BUILT_IN_TM_LOG_1
;
1224 code
= BUILT_IN_TM_LOG_2
;
1227 code
= BUILT_IN_TM_LOG_4
;
1230 code
= BUILT_IN_TM_LOG_8
;
1233 code
= BUILT_IN_TM_LOG
;
1234 if (TREE_CODE (type
) == VECTOR_TYPE
)
1236 if (n
== 8 && builtin_decl_explicit (BUILT_IN_TM_LOG_M64
))
1237 code
= BUILT_IN_TM_LOG_M64
;
1238 else if (n
== 16 && builtin_decl_explicit (BUILT_IN_TM_LOG_M128
))
1239 code
= BUILT_IN_TM_LOG_M128
;
1240 else if (n
== 32 && builtin_decl_explicit (BUILT_IN_TM_LOG_M256
))
1241 code
= BUILT_IN_TM_LOG_M256
;
1247 addr
= gimplify_addr (&gsi
, addr
);
1248 if (code
== BUILT_IN_TM_LOG
)
1249 log
= gimple_build_call (builtin_decl_explicit (code
), 2, addr
, size
);
1251 log
= gimple_build_call (builtin_decl_explicit (code
), 1, addr
);
1252 gsi_insert_before (&gsi
, log
, GSI_SAME_STMT
);
1255 /* Go through the log and instrument address that must be instrumented
1256 with the logging functions. Leave the save/restore addresses for
1261 hash_table
<log_entry_hasher
>::iterator hi
;
1262 struct tm_log_entry
*lp
;
1264 FOR_EACH_HASH_TABLE_ELEMENT (*tm_log
, lp
, tm_log_entry_t
, hi
)
1271 fprintf (dump_file
, "TM thread private mem logging: ");
1272 print_generic_expr (dump_file
, lp
->addr
, 0);
1273 fprintf (dump_file
, "\n");
1279 fprintf (dump_file
, "DUMPING to variable\n");
1285 fprintf (dump_file
, "DUMPING with logging functions\n");
1286 for (i
= 0; lp
->stmts
.iterate (i
, &stmt
); ++i
)
1287 tm_log_emit_stmt (lp
->addr
, stmt
);
1292 /* Emit the save sequence for the corresponding addresses in the log.
1293 ENTRY_BLOCK is the entry block for the transaction.
1294 BB is the basic block to insert the code in. */
1296 tm_log_emit_saves (basic_block entry_block
, basic_block bb
)
1299 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1301 struct tm_log_entry l
, *lp
;
1303 for (i
= 0; i
< tm_log_save_addresses
.length (); ++i
)
1305 l
.addr
= tm_log_save_addresses
[i
];
1306 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1307 gcc_assert (lp
->save_var
!= NULL
);
1309 /* We only care about variables in the current transaction. */
1310 if (lp
->entry_block
!= entry_block
)
1313 stmt
= gimple_build_assign (lp
->save_var
, unshare_expr (lp
->addr
));
1315 /* Make sure we can create an SSA_NAME for this type. For
1316 instance, aggregates aren't allowed, in which case the system
1317 will create a VOP for us and everything will just work. */
1318 if (is_gimple_reg_type (TREE_TYPE (lp
->save_var
)))
1320 lp
->save_var
= make_ssa_name (lp
->save_var
, stmt
);
1321 gimple_assign_set_lhs (stmt
, lp
->save_var
);
1324 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
1328 /* Emit the restore sequence for the corresponding addresses in the log.
1329 ENTRY_BLOCK is the entry block for the transaction.
1330 BB is the basic block to insert the code in. */
1332 tm_log_emit_restores (basic_block entry_block
, basic_block bb
)
1335 struct tm_log_entry l
, *lp
;
1336 gimple_stmt_iterator gsi
;
1339 for (i
= tm_log_save_addresses
.length () - 1; i
>= 0; i
--)
1341 l
.addr
= tm_log_save_addresses
[i
];
1342 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1343 gcc_assert (lp
->save_var
!= NULL
);
1345 /* We only care about variables in the current transaction. */
1346 if (lp
->entry_block
!= entry_block
)
1349 /* Restores are in LIFO order from the saves in case we have
1351 gsi
= gsi_start_bb (bb
);
1353 stmt
= gimple_build_assign (unshare_expr (lp
->addr
), lp
->save_var
);
1354 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1359 static tree
lower_sequence_tm (gimple_stmt_iterator
*, bool *,
1360 struct walk_stmt_info
*);
1361 static tree
lower_sequence_no_tm (gimple_stmt_iterator
*, bool *,
1362 struct walk_stmt_info
*);
1364 /* Evaluate an address X being dereferenced and determine if it
1365 originally points to a non aliased new chunk of memory (malloc,
1368 Return MEM_THREAD_LOCAL if it points to a thread-local address.
1369 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address.
1370 Return MEM_NON_LOCAL otherwise.
1372 ENTRY_BLOCK is the entry block to the transaction containing the
1373 dereference of X. */
1374 static enum thread_memory_type
1375 thread_private_new_memory (basic_block entry_block
, tree x
)
1378 enum tree_code code
;
1379 tm_new_mem_map_t
**slot
;
1380 tm_new_mem_map_t elt
, *elt_p
;
1382 enum thread_memory_type retval
= mem_transaction_local
;
1385 || TREE_CODE (x
) != SSA_NAME
1386 /* Possible uninitialized use, or a function argument. In
1387 either case, we don't care. */
1388 || SSA_NAME_IS_DEFAULT_DEF (x
))
1389 return mem_non_local
;
1391 /* Look in cache first. */
1393 slot
= tm_new_mem_hash
->find_slot (&elt
, INSERT
);
1396 return elt_p
->local_new_memory
;
1398 /* Optimistically assume the memory is transaction local during
1399 processing. This catches recursion into this variable. */
1400 *slot
= elt_p
= XNEW (tm_new_mem_map_t
);
1402 elt_p
->local_new_memory
= mem_transaction_local
;
1404 /* Search DEF chain to find the original definition of this address. */
1407 if (ptr_deref_may_alias_global_p (x
))
1409 /* Address escapes. This is not thread-private. */
1410 retval
= mem_non_local
;
1411 goto new_memory_ret
;
1414 stmt
= SSA_NAME_DEF_STMT (x
);
1416 /* If the malloc call is outside the transaction, this is
1418 if (retval
!= mem_thread_local
1419 && !dominated_by_p (CDI_DOMINATORS
, gimple_bb (stmt
), entry_block
))
1420 retval
= mem_thread_local
;
1422 if (is_gimple_assign (stmt
))
1424 code
= gimple_assign_rhs_code (stmt
);
1425 /* x = foo ==> foo */
1426 if (code
== SSA_NAME
)
1427 x
= gimple_assign_rhs1 (stmt
);
1428 /* x = foo + n ==> foo */
1429 else if (code
== POINTER_PLUS_EXPR
)
1430 x
= gimple_assign_rhs1 (stmt
);
1431 /* x = (cast*) foo ==> foo */
1432 else if (code
== VIEW_CONVERT_EXPR
|| CONVERT_EXPR_CODE_P (code
))
1433 x
= gimple_assign_rhs1 (stmt
);
1434 /* x = c ? op1 : op2 == > op1 or op2 just like a PHI */
1435 else if (code
== COND_EXPR
)
1437 tree op1
= gimple_assign_rhs2 (stmt
);
1438 tree op2
= gimple_assign_rhs3 (stmt
);
1439 enum thread_memory_type mem
;
1440 retval
= thread_private_new_memory (entry_block
, op1
);
1441 if (retval
== mem_non_local
)
1442 goto new_memory_ret
;
1443 mem
= thread_private_new_memory (entry_block
, op2
);
1444 retval
= MIN (retval
, mem
);
1445 goto new_memory_ret
;
1449 retval
= mem_non_local
;
1450 goto new_memory_ret
;
1455 if (gimple_code (stmt
) == GIMPLE_PHI
)
1458 enum thread_memory_type mem
;
1459 tree phi_result
= gimple_phi_result (stmt
);
1461 /* If any of the ancestors are non-local, we are sure to
1462 be non-local. Otherwise we can avoid doing anything
1463 and inherit what has already been generated. */
1465 for (i
= 0; i
< gimple_phi_num_args (stmt
); ++i
)
1467 tree op
= PHI_ARG_DEF (stmt
, i
);
1469 /* Exclude self-assignment. */
1470 if (phi_result
== op
)
1473 mem
= thread_private_new_memory (entry_block
, op
);
1474 if (mem
== mem_non_local
)
1477 goto new_memory_ret
;
1479 retval
= MIN (retval
, mem
);
1481 goto new_memory_ret
;
1486 while (TREE_CODE (x
) == SSA_NAME
);
1488 if (stmt
&& is_gimple_call (stmt
) && gimple_call_flags (stmt
) & ECF_MALLOC
)
1489 /* Thread-local or transaction-local. */
1492 retval
= mem_non_local
;
1495 elt_p
->local_new_memory
= retval
;
1499 /* Determine whether X has to be instrumented using a read
1502 ENTRY_BLOCK is the entry block for the region where stmt resides
1503 in. NULL if unknown.
1505 STMT is the statement in which X occurs in. It is used for thread
1506 private memory instrumentation. If no TPM instrumentation is
1507 desired, STMT should be null. */
1509 requires_barrier (basic_block entry_block
, tree x
, gimple stmt
)
1512 while (handled_component_p (x
))
1513 x
= TREE_OPERAND (x
, 0);
1515 switch (TREE_CODE (x
))
1520 enum thread_memory_type ret
;
1522 ret
= thread_private_new_memory (entry_block
, TREE_OPERAND (x
, 0));
1523 if (ret
== mem_non_local
)
1525 if (stmt
&& ret
== mem_thread_local
)
1526 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */
1527 tm_log_add (entry_block
, orig
, stmt
);
1529 /* Transaction-locals require nothing at all. For malloc, a
1530 transaction restart frees the memory and we reallocate.
1531 For alloca, the stack pointer gets reset by the retry and
1536 case TARGET_MEM_REF
:
1537 if (TREE_CODE (TMR_BASE (x
)) != ADDR_EXPR
)
1539 x
= TREE_OPERAND (TMR_BASE (x
), 0);
1540 if (TREE_CODE (x
) == PARM_DECL
)
1542 gcc_assert (TREE_CODE (x
) == VAR_DECL
);
1548 if (DECL_BY_REFERENCE (x
))
1550 /* ??? This value is a pointer, but aggregate_value_p has been
1551 jigged to return true which confuses needs_to_live_in_memory.
1552 This ought to be cleaned up generically.
1554 FIXME: Verify this still happens after the next mainline
1555 merge. Testcase ie g++.dg/tm/pr47554.C.
1560 if (is_global_var (x
))
1561 return !TREE_READONLY (x
);
1562 if (/* FIXME: This condition should actually go below in the
1563 tm_log_add() call, however is_call_clobbered() depends on
1564 aliasing info which is not available during
1565 gimplification. Since requires_barrier() gets called
1566 during lower_sequence_tm/gimplification, leave the call
1567 to needs_to_live_in_memory until we eliminate
1568 lower_sequence_tm altogether. */
1569 needs_to_live_in_memory (x
))
1573 /* For local memory that doesn't escape (aka thread private
1574 memory), we can either save the value at the beginning of
1575 the transaction and restore on restart, or call a tm
1576 function to dynamically save and restore on restart
1579 tm_log_add (entry_block
, orig
, stmt
);
1588 /* Mark the GIMPLE_ASSIGN statement as appropriate for being inside
1589 a transaction region. */
1592 examine_assign_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1594 gimple stmt
= gsi_stmt (*gsi
);
1596 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_rhs1 (stmt
), NULL
))
1597 *state
|= GTMA_HAVE_LOAD
;
1598 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_lhs (stmt
), NULL
))
1599 *state
|= GTMA_HAVE_STORE
;
1602 /* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */
1605 examine_call_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1607 gimple stmt
= gsi_stmt (*gsi
);
1610 if (is_tm_pure_call (stmt
))
1613 /* Check if this call is a transaction abort. */
1614 fn
= gimple_call_fndecl (stmt
);
1615 if (is_tm_abort (fn
))
1616 *state
|= GTMA_HAVE_ABORT
;
1618 /* Note that something may happen. */
1619 *state
|= GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
;
1622 /* Lower a GIMPLE_TRANSACTION statement. */
1625 lower_transaction (gimple_stmt_iterator
*gsi
, struct walk_stmt_info
*wi
)
1628 gtransaction
*stmt
= as_a
<gtransaction
*> (gsi_stmt (*gsi
));
1629 unsigned int *outer_state
= (unsigned int *) wi
->info
;
1630 unsigned int this_state
= 0;
1631 struct walk_stmt_info this_wi
;
1633 /* First, lower the body. The scanning that we do inside gives
1634 us some idea of what we're dealing with. */
1635 memset (&this_wi
, 0, sizeof (this_wi
));
1636 this_wi
.info
= (void *) &this_state
;
1637 walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt
),
1638 lower_sequence_tm
, NULL
, &this_wi
);
1640 /* If there was absolutely nothing transaction related inside the
1641 transaction, we may elide it. Likewise if this is a nested
1642 transaction and does not contain an abort. */
1644 || (!(this_state
& GTMA_HAVE_ABORT
) && outer_state
!= NULL
))
1647 *outer_state
|= this_state
;
1649 gsi_insert_seq_before (gsi
, gimple_transaction_body (stmt
),
1651 gimple_transaction_set_body (stmt
, NULL
);
1653 gsi_remove (gsi
, true);
1654 wi
->removed_stmt
= true;
1658 /* Wrap the body of the transaction in a try-finally node so that
1659 the commit call is always properly called. */
1660 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT
), 0);
1661 if (flag_exceptions
)
1664 gimple_seq n_seq
, e_seq
;
1666 n_seq
= gimple_seq_alloc_with_stmt (g
);
1669 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER
),
1670 1, integer_zero_node
);
1671 ptr
= create_tmp_var (ptr_type_node
);
1672 gimple_call_set_lhs (g
, ptr
);
1673 gimple_seq_add_stmt (&e_seq
, g
);
1675 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH
),
1677 gimple_seq_add_stmt (&e_seq
, g
);
1679 g
= gimple_build_eh_else (n_seq
, e_seq
);
1682 g
= gimple_build_try (gimple_transaction_body (stmt
),
1683 gimple_seq_alloc_with_stmt (g
), GIMPLE_TRY_FINALLY
);
1684 gsi_insert_after (gsi
, g
, GSI_CONTINUE_LINKING
);
1686 gimple_transaction_set_body (stmt
, NULL
);
1688 /* If the transaction calls abort or if this is an outer transaction,
1689 add an "over" label afterwards. */
1690 if ((this_state
& (GTMA_HAVE_ABORT
))
1691 || (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
))
1693 tree label
= create_artificial_label (UNKNOWN_LOCATION
);
1694 gimple_transaction_set_label (stmt
, label
);
1695 gsi_insert_after (gsi
, gimple_build_label (label
), GSI_CONTINUE_LINKING
);
1698 /* Record the set of operations found for use later. */
1699 this_state
|= gimple_transaction_subcode (stmt
) & GTMA_DECLARATION_MASK
;
1700 gimple_transaction_set_subcode (stmt
, this_state
);
1703 /* Iterate through the statements in the sequence, lowering them all
1704 as appropriate for being in a transaction. */
1707 lower_sequence_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1708 struct walk_stmt_info
*wi
)
1710 unsigned int *state
= (unsigned int *) wi
->info
;
1711 gimple stmt
= gsi_stmt (*gsi
);
1713 *handled_ops_p
= true;
1714 switch (gimple_code (stmt
))
1717 /* Only memory reads/writes need to be instrumented. */
1718 if (gimple_assign_single_p (stmt
))
1719 examine_assign_tm (state
, gsi
);
1723 examine_call_tm (state
, gsi
);
1727 *state
|= GTMA_MAY_ENTER_IRREVOCABLE
;
1730 case GIMPLE_TRANSACTION
:
1731 lower_transaction (gsi
, wi
);
1735 *handled_ops_p
= !gimple_has_substatements (stmt
);
1742 /* Iterate through the statements in the sequence, lowering them all
1743 as appropriate for being outside of a transaction. */
1746 lower_sequence_no_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1747 struct walk_stmt_info
* wi
)
1749 gimple stmt
= gsi_stmt (*gsi
);
1751 if (gimple_code (stmt
) == GIMPLE_TRANSACTION
)
1753 *handled_ops_p
= true;
1754 lower_transaction (gsi
, wi
);
1757 *handled_ops_p
= !gimple_has_substatements (stmt
);
1762 /* Main entry point for flattening GIMPLE_TRANSACTION constructs. After
1763 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has
1764 been moved out, and all the data required for constructing a proper
1765 CFG has been recorded. */
1768 execute_lower_tm (void)
1770 struct walk_stmt_info wi
;
1773 /* Transactional clones aren't created until a later pass. */
1774 gcc_assert (!decl_is_tm_clone (current_function_decl
));
1776 body
= gimple_body (current_function_decl
);
1777 memset (&wi
, 0, sizeof (wi
));
1778 walk_gimple_seq_mod (&body
, lower_sequence_no_tm
, NULL
, &wi
);
1779 gimple_set_body (current_function_decl
, body
);
1786 const pass_data pass_data_lower_tm
=
1788 GIMPLE_PASS
, /* type */
1789 "tmlower", /* name */
1790 OPTGROUP_NONE
, /* optinfo_flags */
1791 TV_TRANS_MEM
, /* tv_id */
1792 PROP_gimple_lcf
, /* properties_required */
1793 0, /* properties_provided */
1794 0, /* properties_destroyed */
1795 0, /* todo_flags_start */
1796 0, /* todo_flags_finish */
1799 class pass_lower_tm
: public gimple_opt_pass
1802 pass_lower_tm (gcc::context
*ctxt
)
1803 : gimple_opt_pass (pass_data_lower_tm
, ctxt
)
1806 /* opt_pass methods: */
1807 virtual bool gate (function
*) { return flag_tm
; }
1808 virtual unsigned int execute (function
*) { return execute_lower_tm (); }
1810 }; // class pass_lower_tm
1815 make_pass_lower_tm (gcc::context
*ctxt
)
1817 return new pass_lower_tm (ctxt
);
1820 /* Collect region information for each transaction. */
1826 /* The field "transaction_stmt" is initially a gtransaction *,
1827 but eventually gets lowered to a gcall *(to BUILT_IN_TM_START).
1829 Helper method to get it as a gtransaction *, with code-checking
1830 in a checked-build. */
1833 get_transaction_stmt () const
1835 return as_a
<gtransaction
*> (transaction_stmt
);
1840 /* Link to the next unnested transaction. */
1841 struct tm_region
*next
;
1843 /* Link to the next inner transaction. */
1844 struct tm_region
*inner
;
1846 /* Link to the next outer transaction. */
1847 struct tm_region
*outer
;
1849 /* The GIMPLE_TRANSACTION statement beginning this transaction.
1850 After TM_MARK, this gets replaced by a call to
1852 Hence this will be either a gtransaction *or a gcall *. */
1853 gimple transaction_stmt
;
1855 /* After TM_MARK expands the GIMPLE_TRANSACTION into a call to
1856 BUILT_IN_TM_START, this field is true if the transaction is an
1857 outer transaction. */
1858 bool original_transaction_was_outer
;
1860 /* Return value from BUILT_IN_TM_START. */
1863 /* The entry block to this region. This will always be the first
1864 block of the body of the transaction. */
1865 basic_block entry_block
;
1867 /* The first block after an expanded call to _ITM_beginTransaction. */
1868 basic_block restart_block
;
1870 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK.
1871 These blocks are still a part of the region (i.e., the border is
1872 inclusive). Note that this set is only complete for paths in the CFG
1873 starting at ENTRY_BLOCK, and that there is no exit block recorded for
1874 the edge to the "over" label. */
1877 /* The set of all blocks that have an TM_IRREVOCABLE call. */
1881 typedef struct tm_region
*tm_region_p
;
1883 /* True if there are pending edge statements to be committed for the
1884 current function being scanned in the tmmark pass. */
1885 bool pending_edge_inserts_p
;
1887 static struct tm_region
*all_tm_regions
;
1888 static bitmap_obstack tm_obstack
;
1891 /* A subroutine of tm_region_init. Record the existence of the
1892 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */
1894 static struct tm_region
*
1895 tm_region_init_0 (struct tm_region
*outer
, basic_block bb
,
1898 struct tm_region
*region
;
1900 region
= (struct tm_region
*)
1901 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1905 region
->next
= outer
->inner
;
1906 outer
->inner
= region
;
1910 region
->next
= all_tm_regions
;
1911 all_tm_regions
= region
;
1913 region
->inner
= NULL
;
1914 region
->outer
= outer
;
1916 region
->transaction_stmt
= stmt
;
1917 region
->original_transaction_was_outer
= false;
1918 region
->tm_state
= NULL
;
1920 /* There are either one or two edges out of the block containing
1921 the GIMPLE_TRANSACTION, one to the actual region and one to the
1922 "over" label if the region contains an abort. The former will
1923 always be the one marked FALLTHRU. */
1924 region
->entry_block
= FALLTHRU_EDGE (bb
)->dest
;
1926 region
->exit_blocks
= BITMAP_ALLOC (&tm_obstack
);
1927 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1932 /* A subroutine of tm_region_init. Record all the exit and
1933 irrevocable blocks in BB into the region's exit_blocks and
1934 irr_blocks bitmaps. Returns the new region being scanned. */
1936 static struct tm_region
*
1937 tm_region_init_1 (struct tm_region
*region
, basic_block bb
)
1939 gimple_stmt_iterator gsi
;
1943 || (!region
->irr_blocks
&& !region
->exit_blocks
))
1946 /* Check to see if this is the end of a region by seeing if it
1947 contains a call to __builtin_tm_commit{,_eh}. Note that the
1948 outermost region for DECL_IS_TM_CLONE need not collect this. */
1949 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
1952 if (gimple_code (g
) == GIMPLE_CALL
)
1954 tree fn
= gimple_call_fndecl (g
);
1955 if (fn
&& DECL_BUILT_IN_CLASS (fn
) == BUILT_IN_NORMAL
)
1957 if ((DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT
1958 || DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT_EH
)
1959 && region
->exit_blocks
)
1961 bitmap_set_bit (region
->exit_blocks
, bb
->index
);
1962 region
= region
->outer
;
1965 if (DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_IRREVOCABLE
)
1966 bitmap_set_bit (region
->irr_blocks
, bb
->index
);
1973 /* Collect all of the transaction regions within the current function
1974 and record them in ALL_TM_REGIONS. The REGION parameter may specify
1975 an "outermost" region for use by tm clones. */
1978 tm_region_init (struct tm_region
*region
)
1984 auto_vec
<basic_block
> queue
;
1985 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
1986 struct tm_region
*old_region
;
1987 auto_vec
<tm_region_p
> bb_regions
;
1989 all_tm_regions
= region
;
1990 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1992 /* We could store this information in bb->aux, but we may get called
1993 through get_all_tm_blocks() from another pass that may be already
1995 bb_regions
.safe_grow_cleared (last_basic_block_for_fn (cfun
));
1997 queue
.safe_push (bb
);
1998 bb_regions
[bb
->index
] = region
;
2002 region
= bb_regions
[bb
->index
];
2003 bb_regions
[bb
->index
] = NULL
;
2005 /* Record exit and irrevocable blocks. */
2006 region
= tm_region_init_1 (region
, bb
);
2008 /* Check for the last statement in the block beginning a new region. */
2010 old_region
= region
;
2012 if (gtransaction
*trans_stmt
= dyn_cast
<gtransaction
*> (g
))
2013 region
= tm_region_init_0 (region
, bb
, trans_stmt
);
2015 /* Process subsequent blocks. */
2016 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2017 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2019 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2020 queue
.safe_push (e
->dest
);
2022 /* If the current block started a new region, make sure that only
2023 the entry block of the new region is associated with this region.
2024 Other successors are still part of the old region. */
2025 if (old_region
!= region
&& e
->dest
!= region
->entry_block
)
2026 bb_regions
[e
->dest
->index
] = old_region
;
2028 bb_regions
[e
->dest
->index
] = region
;
2031 while (!queue
.is_empty ());
2032 BITMAP_FREE (visited_blocks
);
2035 /* The "gate" function for all transactional memory expansion and optimization
2036 passes. We collect region information for each top-level transaction, and
2037 if we don't find any, we skip all of the TM passes. Each region will have
2038 all of the exit blocks recorded, and the originating statement. */
2046 calculate_dominance_info (CDI_DOMINATORS
);
2047 bitmap_obstack_initialize (&tm_obstack
);
2049 /* If the function is a TM_CLONE, then the entire function is the region. */
2050 if (decl_is_tm_clone (current_function_decl
))
2052 struct tm_region
*region
= (struct tm_region
*)
2053 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
2054 memset (region
, 0, sizeof (*region
));
2055 region
->entry_block
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
2056 /* For a clone, the entire function is the region. But even if
2057 we don't need to record any exit blocks, we may need to
2058 record irrevocable blocks. */
2059 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
2061 tm_region_init (region
);
2065 tm_region_init (NULL
);
2067 /* If we didn't find any regions, cleanup and skip the whole tree
2068 of tm-related optimizations. */
2069 if (all_tm_regions
== NULL
)
2071 bitmap_obstack_release (&tm_obstack
);
2081 const pass_data pass_data_tm_init
=
2083 GIMPLE_PASS
, /* type */
2084 "*tminit", /* name */
2085 OPTGROUP_NONE
, /* optinfo_flags */
2086 TV_TRANS_MEM
, /* tv_id */
2087 ( PROP_ssa
| PROP_cfg
), /* properties_required */
2088 0, /* properties_provided */
2089 0, /* properties_destroyed */
2090 0, /* todo_flags_start */
2091 0, /* todo_flags_finish */
2094 class pass_tm_init
: public gimple_opt_pass
2097 pass_tm_init (gcc::context
*ctxt
)
2098 : gimple_opt_pass (pass_data_tm_init
, ctxt
)
2101 /* opt_pass methods: */
2102 virtual bool gate (function
*) { return gate_tm_init (); }
2104 }; // class pass_tm_init
2109 make_pass_tm_init (gcc::context
*ctxt
)
2111 return new pass_tm_init (ctxt
);
2114 /* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region
2115 represented by STATE. */
2118 transaction_subcode_ior (struct tm_region
*region
, unsigned flags
)
2120 if (region
&& region
->transaction_stmt
)
2122 gtransaction
*transaction_stmt
= region
->get_transaction_stmt ();
2123 flags
|= gimple_transaction_subcode (transaction_stmt
);
2124 gimple_transaction_set_subcode (transaction_stmt
, flags
);
2128 /* Construct a memory load in a transactional context. Return the
2129 gimple statement performing the load, or NULL if there is no
2130 TM_LOAD builtin of the appropriate size to do the load.
2132 LOC is the location to use for the new statement(s). */
2135 build_tm_load (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2137 enum built_in_function code
= END_BUILTINS
;
2138 tree t
, type
= TREE_TYPE (rhs
), decl
;
2141 if (type
== float_type_node
)
2142 code
= BUILT_IN_TM_LOAD_FLOAT
;
2143 else if (type
== double_type_node
)
2144 code
= BUILT_IN_TM_LOAD_DOUBLE
;
2145 else if (type
== long_double_type_node
)
2146 code
= BUILT_IN_TM_LOAD_LDOUBLE
;
2147 else if (TYPE_SIZE_UNIT (type
) != NULL
2148 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2150 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2153 code
= BUILT_IN_TM_LOAD_1
;
2156 code
= BUILT_IN_TM_LOAD_2
;
2159 code
= BUILT_IN_TM_LOAD_4
;
2162 code
= BUILT_IN_TM_LOAD_8
;
2167 if (code
== END_BUILTINS
)
2169 decl
= targetm
.vectorize
.builtin_tm_load (type
);
2174 decl
= builtin_decl_explicit (code
);
2176 t
= gimplify_addr (gsi
, rhs
);
2177 gcall
= gimple_build_call (decl
, 1, t
);
2178 gimple_set_location (gcall
, loc
);
2180 t
= TREE_TYPE (TREE_TYPE (decl
));
2181 if (useless_type_conversion_p (type
, t
))
2183 gimple_call_set_lhs (gcall
, lhs
);
2184 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2191 temp
= create_tmp_reg (t
);
2192 gimple_call_set_lhs (gcall
, temp
);
2193 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2195 t
= fold_build1 (VIEW_CONVERT_EXPR
, type
, temp
);
2196 g
= gimple_build_assign (lhs
, t
);
2197 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2204 /* Similarly for storing TYPE in a transactional context. */
2207 build_tm_store (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2209 enum built_in_function code
= END_BUILTINS
;
2210 tree t
, fn
, type
= TREE_TYPE (rhs
), simple_type
;
2213 if (type
== float_type_node
)
2214 code
= BUILT_IN_TM_STORE_FLOAT
;
2215 else if (type
== double_type_node
)
2216 code
= BUILT_IN_TM_STORE_DOUBLE
;
2217 else if (type
== long_double_type_node
)
2218 code
= BUILT_IN_TM_STORE_LDOUBLE
;
2219 else if (TYPE_SIZE_UNIT (type
) != NULL
2220 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2222 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2225 code
= BUILT_IN_TM_STORE_1
;
2228 code
= BUILT_IN_TM_STORE_2
;
2231 code
= BUILT_IN_TM_STORE_4
;
2234 code
= BUILT_IN_TM_STORE_8
;
2239 if (code
== END_BUILTINS
)
2241 fn
= targetm
.vectorize
.builtin_tm_store (type
);
2246 fn
= builtin_decl_explicit (code
);
2248 simple_type
= TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn
))));
2250 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
2252 /* Handle the easy initialization to zero. */
2253 if (!CONSTRUCTOR_ELTS (rhs
))
2254 rhs
= build_int_cst (simple_type
, 0);
2257 /* ...otherwise punt to the caller and probably use
2258 BUILT_IN_TM_MEMMOVE, because we can't wrap a
2259 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce
2264 else if (!useless_type_conversion_p (simple_type
, type
))
2269 temp
= create_tmp_reg (simple_type
);
2270 t
= fold_build1 (VIEW_CONVERT_EXPR
, simple_type
, rhs
);
2271 g
= gimple_build_assign (temp
, t
);
2272 gimple_set_location (g
, loc
);
2273 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2278 t
= gimplify_addr (gsi
, lhs
);
2279 gcall
= gimple_build_call (fn
, 2, t
, rhs
);
2280 gimple_set_location (gcall
, loc
);
2281 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2287 /* Expand an assignment statement into transactional builtins. */
2290 expand_assign_tm (struct tm_region
*region
, gimple_stmt_iterator
*gsi
)
2292 gimple stmt
= gsi_stmt (*gsi
);
2293 location_t loc
= gimple_location (stmt
);
2294 tree lhs
= gimple_assign_lhs (stmt
);
2295 tree rhs
= gimple_assign_rhs1 (stmt
);
2296 bool store_p
= requires_barrier (region
->entry_block
, lhs
, NULL
);
2297 bool load_p
= requires_barrier (region
->entry_block
, rhs
, NULL
);
2298 gimple gcall
= NULL
;
2300 if (!load_p
&& !store_p
)
2302 /* Add thread private addresses to log if applicable. */
2303 requires_barrier (region
->entry_block
, lhs
, stmt
);
2308 // Remove original load/store statement.
2309 gsi_remove (gsi
, true);
2311 if (load_p
&& !store_p
)
2313 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2314 gcall
= build_tm_load (loc
, lhs
, rhs
, gsi
);
2316 else if (store_p
&& !load_p
)
2318 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2319 gcall
= build_tm_store (loc
, lhs
, rhs
, gsi
);
2323 tree lhs_addr
, rhs_addr
, tmp
;
2326 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2328 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2330 /* ??? Figure out if there's any possible overlap between the LHS
2331 and the RHS and if not, use MEMCPY. */
2333 if (load_p
&& is_gimple_reg (lhs
))
2335 tmp
= create_tmp_var (TREE_TYPE (lhs
));
2336 lhs_addr
= build_fold_addr_expr (tmp
);
2341 lhs_addr
= gimplify_addr (gsi
, lhs
);
2343 rhs_addr
= gimplify_addr (gsi
, rhs
);
2344 gcall
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
),
2345 3, lhs_addr
, rhs_addr
,
2346 TYPE_SIZE_UNIT (TREE_TYPE (lhs
)));
2347 gimple_set_location (gcall
, loc
);
2348 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2352 gcall
= gimple_build_assign (lhs
, tmp
);
2353 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2357 /* Now that we have the load/store in its instrumented form, add
2358 thread private addresses to the log if applicable. */
2360 requires_barrier (region
->entry_block
, lhs
, gcall
);
2362 // The calls to build_tm_{store,load} above inserted the instrumented
2363 // call into the stream.
2364 // gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2368 /* Expand a call statement as appropriate for a transaction. That is,
2369 either verify that the call does not affect the transaction, or
2370 redirect the call to a clone that handles transactions, or change
2371 the transaction state to IRREVOCABLE. Return true if the call is
2372 one of the builtins that end a transaction. */
2375 expand_call_tm (struct tm_region
*region
,
2376 gimple_stmt_iterator
*gsi
)
2378 gcall
*stmt
= as_a
<gcall
*> (gsi_stmt (*gsi
));
2379 tree lhs
= gimple_call_lhs (stmt
);
2381 struct cgraph_node
*node
;
2382 bool retval
= false;
2384 fn_decl
= gimple_call_fndecl (stmt
);
2386 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMCPY
)
2387 || fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
))
2388 transaction_subcode_ior (region
, GTMA_HAVE_STORE
| GTMA_HAVE_LOAD
);
2389 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMSET
))
2390 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2392 if (is_tm_pure_call (stmt
))
2396 retval
= is_tm_ending_fndecl (fn_decl
);
2399 /* Assume all non-const/pure calls write to memory, except
2400 transaction ending builtins. */
2401 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2404 /* For indirect calls, we already generated a call into the runtime. */
2407 tree fn
= gimple_call_fn (stmt
);
2409 /* We are guaranteed never to go irrevocable on a safe or pure
2410 call, and the pure call was handled above. */
2411 if (is_tm_safe (fn
))
2414 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2419 node
= cgraph_node::get (fn_decl
);
2420 /* All calls should have cgraph here. */
2423 /* We can have a nodeless call here if some pass after IPA-tm
2424 added uninstrumented calls. For example, loop distribution
2425 can transform certain loop constructs into __builtin_mem*
2426 calls. In this case, see if we have a suitable TM
2427 replacement and fill in the gaps. */
2428 gcc_assert (DECL_BUILT_IN_CLASS (fn_decl
) == BUILT_IN_NORMAL
);
2429 enum built_in_function code
= DECL_FUNCTION_CODE (fn_decl
);
2430 gcc_assert (code
== BUILT_IN_MEMCPY
2431 || code
== BUILT_IN_MEMMOVE
2432 || code
== BUILT_IN_MEMSET
);
2434 tree repl
= find_tm_replacement_function (fn_decl
);
2437 gimple_call_set_fndecl (stmt
, repl
);
2439 node
= cgraph_node::create (repl
);
2440 node
->local
.tm_may_enter_irr
= false;
2441 return expand_call_tm (region
, gsi
);
2445 if (node
->local
.tm_may_enter_irr
)
2446 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2448 if (is_tm_abort (fn_decl
))
2450 transaction_subcode_ior (region
, GTMA_HAVE_ABORT
);
2454 /* Instrument the store if needed.
2456 If the assignment happens inside the function call (return slot
2457 optimization), there is no instrumentation to be done, since
2458 the callee should have done the right thing. */
2459 if (lhs
&& requires_barrier (region
->entry_block
, lhs
, stmt
)
2460 && !gimple_call_return_slot_opt_p (stmt
))
2462 tree tmp
= create_tmp_reg (TREE_TYPE (lhs
));
2463 location_t loc
= gimple_location (stmt
);
2464 edge fallthru_edge
= NULL
;
2465 gassign
*assign_stmt
;
2467 /* Remember if the call was going to throw. */
2468 if (stmt_can_throw_internal (stmt
))
2472 basic_block bb
= gimple_bb (stmt
);
2474 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2475 if (e
->flags
& EDGE_FALLTHRU
)
2482 gimple_call_set_lhs (stmt
, tmp
);
2484 assign_stmt
= gimple_build_assign (lhs
, tmp
);
2485 gimple_set_location (assign_stmt
, loc
);
2487 /* We cannot throw in the middle of a BB. If the call was going
2488 to throw, place the instrumentation on the fallthru edge, so
2489 the call remains the last statement in the block. */
2492 gimple_seq fallthru_seq
= gimple_seq_alloc_with_stmt (assign_stmt
);
2493 gimple_stmt_iterator fallthru_gsi
= gsi_start (fallthru_seq
);
2494 expand_assign_tm (region
, &fallthru_gsi
);
2495 gsi_insert_seq_on_edge (fallthru_edge
, fallthru_seq
);
2496 pending_edge_inserts_p
= true;
2500 gsi_insert_after (gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
2501 expand_assign_tm (region
, gsi
);
2504 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2511 /* Expand all statements in BB as appropriate for being inside
2515 expand_block_tm (struct tm_region
*region
, basic_block bb
)
2517 gimple_stmt_iterator gsi
;
2519 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2521 gimple stmt
= gsi_stmt (gsi
);
2522 switch (gimple_code (stmt
))
2525 /* Only memory reads/writes need to be instrumented. */
2526 if (gimple_assign_single_p (stmt
)
2527 && !gimple_clobber_p (stmt
))
2529 expand_assign_tm (region
, &gsi
);
2535 if (expand_call_tm (region
, &gsi
))
2545 if (!gsi_end_p (gsi
))
2550 /* Return the list of basic-blocks in REGION.
2552 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks
2553 following a TM_IRREVOCABLE call.
2555 INCLUDE_UNINSTRUMENTED_P is TRUE if we should include the
2556 uninstrumented code path blocks in the list of basic blocks
2557 returned, false otherwise. */
2559 static vec
<basic_block
>
2560 get_tm_region_blocks (basic_block entry_block
,
2563 bitmap all_region_blocks
,
2564 bool stop_at_irrevocable_p
,
2565 bool include_uninstrumented_p
= true)
2567 vec
<basic_block
> bbs
= vNULL
;
2571 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
2574 bbs
.safe_push (entry_block
);
2575 bitmap_set_bit (visited_blocks
, entry_block
->index
);
2579 basic_block bb
= bbs
[i
++];
2582 bitmap_bit_p (exit_blocks
, bb
->index
))
2585 if (stop_at_irrevocable_p
2587 && bitmap_bit_p (irr_blocks
, bb
->index
))
2590 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2591 if ((include_uninstrumented_p
2592 || !(e
->flags
& EDGE_TM_UNINSTRUMENTED
))
2593 && !bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2595 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2596 bbs
.safe_push (e
->dest
);
2599 while (i
< bbs
.length ());
2601 if (all_region_blocks
)
2602 bitmap_ior_into (all_region_blocks
, visited_blocks
);
2604 BITMAP_FREE (visited_blocks
);
2608 // Callback data for collect_bb2reg.
2611 vec
<tm_region_p
> *bb2reg
;
2612 bool include_uninstrumented_p
;
2615 // Callback for expand_regions, collect innermost region data for each bb.
2617 collect_bb2reg (struct tm_region
*region
, void *data
)
2619 struct bb2reg_stuff
*stuff
= (struct bb2reg_stuff
*)data
;
2620 vec
<tm_region_p
> *bb2reg
= stuff
->bb2reg
;
2621 vec
<basic_block
> queue
;
2625 queue
= get_tm_region_blocks (region
->entry_block
,
2626 region
->exit_blocks
,
2629 /*stop_at_irr_p=*/true,
2630 stuff
->include_uninstrumented_p
);
2632 // We expect expand_region to perform a post-order traversal of the region
2633 // tree. Therefore the last region seen for any bb is the innermost.
2634 FOR_EACH_VEC_ELT (queue
, i
, bb
)
2635 (*bb2reg
)[bb
->index
] = region
;
2641 // Returns a vector, indexed by BB->INDEX, of the innermost tm_region to
2642 // which a basic block belongs. Note that we only consider the instrumented
2643 // code paths for the region; the uninstrumented code paths are ignored if
2644 // INCLUDE_UNINSTRUMENTED_P is false.
2646 // ??? This data is very similar to the bb_regions array that is collected
2647 // during tm_region_init. Or, rather, this data is similar to what could
2648 // be used within tm_region_init. The actual computation in tm_region_init
2649 // begins and ends with bb_regions entirely full of NULL pointers, due to
2650 // the way in which pointers are swapped in and out of the array.
2652 // ??? Our callers expect that blocks are not shared between transactions.
2653 // When the optimizers get too smart, and blocks are shared, then during
2654 // the tm_mark phase we'll add log entries to only one of the two transactions,
2655 // and in the tm_edge phase we'll add edges to the CFG that create invalid
2656 // cycles. The symptom being SSA defs that do not dominate their uses.
2657 // Note that the optimizers were locally correct with their transformation,
2658 // as we have no info within the program that suggests that the blocks cannot
2661 // ??? There is currently a hack inside tree-ssa-pre.c to work around the
2662 // only known instance of this block sharing.
2664 static vec
<tm_region_p
>
2665 get_bb_regions_instrumented (bool traverse_clones
,
2666 bool include_uninstrumented_p
)
2668 unsigned n
= last_basic_block_for_fn (cfun
);
2669 struct bb2reg_stuff stuff
;
2670 vec
<tm_region_p
> ret
;
2673 ret
.safe_grow_cleared (n
);
2674 stuff
.bb2reg
= &ret
;
2675 stuff
.include_uninstrumented_p
= include_uninstrumented_p
;
2676 expand_regions (all_tm_regions
, collect_bb2reg
, &stuff
, traverse_clones
);
2681 /* Set the IN_TRANSACTION for all gimple statements that appear in a
2685 compute_transaction_bits (void)
2687 struct tm_region
*region
;
2688 vec
<basic_block
> queue
;
2692 /* ?? Perhaps we need to abstract gate_tm_init further, because we
2693 certainly don't need it to calculate CDI_DOMINATOR info. */
2696 FOR_EACH_BB_FN (bb
, cfun
)
2697 bb
->flags
&= ~BB_IN_TRANSACTION
;
2699 for (region
= all_tm_regions
; region
; region
= region
->next
)
2701 queue
= get_tm_region_blocks (region
->entry_block
,
2702 region
->exit_blocks
,
2705 /*stop_at_irr_p=*/true);
2706 for (i
= 0; queue
.iterate (i
, &bb
); ++i
)
2707 bb
->flags
|= BB_IN_TRANSACTION
;
2712 bitmap_obstack_release (&tm_obstack
);
2715 /* Replace the GIMPLE_TRANSACTION in this region with the corresponding
2716 call to BUILT_IN_TM_START. */
2719 expand_transaction (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2721 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2722 basic_block transaction_bb
= gimple_bb (region
->transaction_stmt
);
2723 tree tm_state
= region
->tm_state
;
2724 tree tm_state_type
= TREE_TYPE (tm_state
);
2725 edge abort_edge
= NULL
;
2726 edge inst_edge
= NULL
;
2727 edge uninst_edge
= NULL
;
2728 edge fallthru_edge
= NULL
;
2730 // Identify the various successors of the transaction start.
2734 FOR_EACH_EDGE (e
, i
, transaction_bb
->succs
)
2736 if (e
->flags
& EDGE_TM_ABORT
)
2738 else if (e
->flags
& EDGE_TM_UNINSTRUMENTED
)
2742 if (e
->flags
& EDGE_FALLTHRU
)
2747 /* ??? There are plenty of bits here we're not computing. */
2749 int subcode
= gimple_transaction_subcode (region
->get_transaction_stmt ());
2751 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2752 flags
|= PR_DOESGOIRREVOCABLE
;
2753 if ((subcode
& GTMA_MAY_ENTER_IRREVOCABLE
) == 0)
2754 flags
|= PR_HASNOIRREVOCABLE
;
2755 /* If the transaction does not have an abort in lexical scope and is not
2756 marked as an outer transaction, then it will never abort. */
2757 if ((subcode
& GTMA_HAVE_ABORT
) == 0 && (subcode
& GTMA_IS_OUTER
) == 0)
2758 flags
|= PR_HASNOABORT
;
2759 if ((subcode
& GTMA_HAVE_STORE
) == 0)
2760 flags
|= PR_READONLY
;
2761 if (inst_edge
&& !(subcode
& GTMA_HAS_NO_INSTRUMENTATION
))
2762 flags
|= PR_INSTRUMENTEDCODE
;
2764 flags
|= PR_UNINSTRUMENTEDCODE
;
2765 if (subcode
& GTMA_IS_OUTER
)
2766 region
->original_transaction_was_outer
= true;
2767 tree t
= build_int_cst (tm_state_type
, flags
);
2768 gcall
*call
= gimple_build_call (tm_start
, 1, t
);
2769 gimple_call_set_lhs (call
, tm_state
);
2770 gimple_set_location (call
, gimple_location (region
->transaction_stmt
));
2772 // Replace the GIMPLE_TRANSACTION with the call to BUILT_IN_TM_START.
2773 gimple_stmt_iterator gsi
= gsi_last_bb (transaction_bb
);
2774 gcc_assert (gsi_stmt (gsi
) == region
->transaction_stmt
);
2775 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
2776 gsi_remove (&gsi
, true);
2777 region
->transaction_stmt
= call
;
2780 // Generate log saves.
2781 if (!tm_log_save_addresses
.is_empty ())
2782 tm_log_emit_saves (region
->entry_block
, transaction_bb
);
2784 // In the beginning, we've no tests to perform on transaction restart.
2785 // Note that after this point, transaction_bb becomes the "most recent
2786 // block containing tests for the transaction".
2787 region
->restart_block
= region
->entry_block
;
2789 // Generate log restores.
2790 if (!tm_log_save_addresses
.is_empty ())
2792 basic_block test_bb
= create_empty_bb (transaction_bb
);
2793 basic_block code_bb
= create_empty_bb (test_bb
);
2794 basic_block join_bb
= create_empty_bb (code_bb
);
2795 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2796 add_bb_to_loop (code_bb
, transaction_bb
->loop_father
);
2797 add_bb_to_loop (join_bb
, transaction_bb
->loop_father
);
2798 if (region
->restart_block
== region
->entry_block
)
2799 region
->restart_block
= test_bb
;
2801 tree t1
= create_tmp_reg (tm_state_type
);
2802 tree t2
= build_int_cst (tm_state_type
, A_RESTORELIVEVARIABLES
);
2803 gimple stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2804 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2805 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2807 t2
= build_int_cst (tm_state_type
, 0);
2808 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2809 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2811 tm_log_emit_restores (region
->entry_block
, code_bb
);
2813 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2814 edge et
= make_edge (test_bb
, code_bb
, EDGE_TRUE_VALUE
);
2815 edge ef
= make_edge (test_bb
, join_bb
, EDGE_FALSE_VALUE
);
2816 redirect_edge_pred (fallthru_edge
, join_bb
);
2818 join_bb
->frequency
= test_bb
->frequency
= transaction_bb
->frequency
;
2819 join_bb
->count
= test_bb
->count
= transaction_bb
->count
;
2821 ei
->probability
= PROB_ALWAYS
;
2822 et
->probability
= PROB_LIKELY
;
2823 ef
->probability
= PROB_UNLIKELY
;
2824 et
->count
= apply_probability (test_bb
->count
, et
->probability
);
2825 ef
->count
= apply_probability (test_bb
->count
, ef
->probability
);
2827 code_bb
->count
= et
->count
;
2828 code_bb
->frequency
= EDGE_FREQUENCY (et
);
2830 transaction_bb
= join_bb
;
2833 // If we have an ABORT edge, create a test to perform the abort.
2836 basic_block test_bb
= create_empty_bb (transaction_bb
);
2837 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2838 if (region
->restart_block
== region
->entry_block
)
2839 region
->restart_block
= test_bb
;
2841 tree t1
= create_tmp_reg (tm_state_type
);
2842 tree t2
= build_int_cst (tm_state_type
, A_ABORTTRANSACTION
);
2843 gimple stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2844 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2845 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2847 t2
= build_int_cst (tm_state_type
, 0);
2848 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2849 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2851 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2852 test_bb
->frequency
= transaction_bb
->frequency
;
2853 test_bb
->count
= transaction_bb
->count
;
2854 ei
->probability
= PROB_ALWAYS
;
2856 // Not abort edge. If both are live, chose one at random as we'll
2857 // we'll be fixing that up below.
2858 redirect_edge_pred (fallthru_edge
, test_bb
);
2859 fallthru_edge
->flags
= EDGE_FALSE_VALUE
;
2860 fallthru_edge
->probability
= PROB_VERY_LIKELY
;
2861 fallthru_edge
->count
2862 = apply_probability (test_bb
->count
, fallthru_edge
->probability
);
2865 redirect_edge_pred (abort_edge
, test_bb
);
2866 abort_edge
->flags
= EDGE_TRUE_VALUE
;
2867 abort_edge
->probability
= PROB_VERY_UNLIKELY
;
2869 = apply_probability (test_bb
->count
, abort_edge
->probability
);
2871 transaction_bb
= test_bb
;
2874 // If we have both instrumented and uninstrumented code paths, select one.
2875 if (inst_edge
&& uninst_edge
)
2877 basic_block test_bb
= create_empty_bb (transaction_bb
);
2878 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2879 if (region
->restart_block
== region
->entry_block
)
2880 region
->restart_block
= test_bb
;
2882 tree t1
= create_tmp_reg (tm_state_type
);
2883 tree t2
= build_int_cst (tm_state_type
, A_RUNUNINSTRUMENTEDCODE
);
2885 gimple stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2886 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2887 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2889 t2
= build_int_cst (tm_state_type
, 0);
2890 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2891 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2893 // Create the edge into test_bb first, as we want to copy values
2894 // out of the fallthru edge.
2895 edge e
= make_edge (transaction_bb
, test_bb
, fallthru_edge
->flags
);
2896 e
->probability
= fallthru_edge
->probability
;
2897 test_bb
->count
= e
->count
= fallthru_edge
->count
;
2898 test_bb
->frequency
= EDGE_FREQUENCY (e
);
2900 // Now update the edges to the inst/uninist implementations.
2901 // For now assume that the paths are equally likely. When using HTM,
2902 // we'll try the uninst path first and fallback to inst path if htm
2903 // buffers are exceeded. Without HTM we start with the inst path and
2904 // use the uninst path when falling back to serial mode.
2905 redirect_edge_pred (inst_edge
, test_bb
);
2906 inst_edge
->flags
= EDGE_FALSE_VALUE
;
2907 inst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2909 = apply_probability (test_bb
->count
, inst_edge
->probability
);
2911 redirect_edge_pred (uninst_edge
, test_bb
);
2912 uninst_edge
->flags
= EDGE_TRUE_VALUE
;
2913 uninst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2915 = apply_probability (test_bb
->count
, uninst_edge
->probability
);
2918 // If we have no previous special cases, and we have PHIs at the beginning
2919 // of the atomic region, this means we have a loop at the beginning of the
2920 // atomic region that shares the first block. This can cause problems with
2921 // the transaction restart abnormal edges to be added in the tm_edges pass.
2922 // Solve this by adding a new empty block to receive the abnormal edges.
2923 if (region
->restart_block
== region
->entry_block
2924 && phi_nodes (region
->entry_block
))
2926 basic_block empty_bb
= create_empty_bb (transaction_bb
);
2927 region
->restart_block
= empty_bb
;
2928 add_bb_to_loop (empty_bb
, transaction_bb
->loop_father
);
2930 redirect_edge_pred (fallthru_edge
, empty_bb
);
2931 make_edge (transaction_bb
, empty_bb
, EDGE_FALLTHRU
);
2937 /* Generate the temporary to be used for the return value of
2938 BUILT_IN_TM_START. */
2941 generate_tm_state (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2943 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2945 create_tmp_reg (TREE_TYPE (TREE_TYPE (tm_start
)), "tm_state");
2947 // Reset the subcode, post optimizations. We'll fill this in
2948 // again as we process blocks.
2949 if (region
->exit_blocks
)
2951 gtransaction
*transaction_stmt
= region
->get_transaction_stmt ();
2952 unsigned int subcode
= gimple_transaction_subcode (transaction_stmt
);
2954 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2955 subcode
&= (GTMA_DECLARATION_MASK
| GTMA_DOES_GO_IRREVOCABLE
2956 | GTMA_MAY_ENTER_IRREVOCABLE
2957 | GTMA_HAS_NO_INSTRUMENTATION
);
2959 subcode
&= GTMA_DECLARATION_MASK
;
2960 gimple_transaction_set_subcode (transaction_stmt
, subcode
);
2966 // Propagate flags from inner transactions outwards.
2968 propagate_tm_flags_out (struct tm_region
*region
)
2972 propagate_tm_flags_out (region
->inner
);
2974 if (region
->outer
&& region
->outer
->transaction_stmt
)
2977 = gimple_transaction_subcode (region
->get_transaction_stmt ());
2978 s
&= (GTMA_HAVE_ABORT
| GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
2979 | GTMA_MAY_ENTER_IRREVOCABLE
);
2980 s
|= gimple_transaction_subcode (region
->outer
->get_transaction_stmt ());
2981 gimple_transaction_set_subcode (region
->outer
->get_transaction_stmt (),
2985 propagate_tm_flags_out (region
->next
);
2988 /* Entry point to the MARK phase of TM expansion. Here we replace
2989 transactional memory statements with calls to builtins, and function
2990 calls with their transactional clones (if available). But we don't
2991 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */
2994 execute_tm_mark (void)
2996 pending_edge_inserts_p
= false;
2998 expand_regions (all_tm_regions
, generate_tm_state
, NULL
,
2999 /*traverse_clones=*/true);
3003 vec
<tm_region_p
> bb_regions
3004 = get_bb_regions_instrumented (/*traverse_clones=*/true,
3005 /*include_uninstrumented_p=*/false);
3006 struct tm_region
*r
;
3009 // Expand memory operations into calls into the runtime.
3010 // This collects log entries as well.
3011 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3015 if (r
->transaction_stmt
)
3018 = gimple_transaction_subcode (r
->get_transaction_stmt ());
3020 /* If we're sure to go irrevocable, there won't be
3021 anything to expand, since the run-time will go
3022 irrevocable right away. */
3023 if (sub
& GTMA_DOES_GO_IRREVOCABLE
3024 && sub
& GTMA_MAY_ENTER_IRREVOCABLE
)
3027 expand_block_tm (r
, BASIC_BLOCK_FOR_FN (cfun
, i
));
3031 bb_regions
.release ();
3033 // Propagate flags from inner transactions outwards.
3034 propagate_tm_flags_out (all_tm_regions
);
3036 // Expand GIMPLE_TRANSACTIONs into calls into the runtime.
3037 expand_regions (all_tm_regions
, expand_transaction
, NULL
,
3038 /*traverse_clones=*/false);
3043 if (pending_edge_inserts_p
)
3044 gsi_commit_edge_inserts ();
3045 free_dominance_info (CDI_DOMINATORS
);
3051 const pass_data pass_data_tm_mark
=
3053 GIMPLE_PASS
, /* type */
3054 "tmmark", /* name */
3055 OPTGROUP_NONE
, /* optinfo_flags */
3056 TV_TRANS_MEM
, /* tv_id */
3057 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3058 0, /* properties_provided */
3059 0, /* properties_destroyed */
3060 0, /* todo_flags_start */
3061 TODO_update_ssa
, /* todo_flags_finish */
3064 class pass_tm_mark
: public gimple_opt_pass
3067 pass_tm_mark (gcc::context
*ctxt
)
3068 : gimple_opt_pass (pass_data_tm_mark
, ctxt
)
3071 /* opt_pass methods: */
3072 virtual unsigned int execute (function
*) { return execute_tm_mark (); }
3074 }; // class pass_tm_mark
3079 make_pass_tm_mark (gcc::context
*ctxt
)
3081 return new pass_tm_mark (ctxt
);
3085 /* Create an abnormal edge from STMT at iter, splitting the block
3086 as necessary. Adjust *PNEXT as needed for the split block. */
3089 split_bb_make_tm_edge (gimple stmt
, basic_block dest_bb
,
3090 gimple_stmt_iterator iter
, gimple_stmt_iterator
*pnext
)
3092 basic_block bb
= gimple_bb (stmt
);
3093 if (!gsi_one_before_end_p (iter
))
3095 edge e
= split_block (bb
, stmt
);
3096 *pnext
= gsi_start_bb (e
->dest
);
3098 make_edge (bb
, dest_bb
, EDGE_ABNORMAL
);
3100 // Record the need for the edge for the benefit of the rtl passes.
3101 if (cfun
->gimple_df
->tm_restart
== NULL
)
3102 cfun
->gimple_df
->tm_restart
3103 = hash_table
<tm_restart_hasher
>::create_ggc (31);
3105 struct tm_restart_node dummy
;
3107 dummy
.label_or_list
= gimple_block_label (dest_bb
);
3109 tm_restart_node
**slot
= cfun
->gimple_df
->tm_restart
->find_slot (&dummy
,
3111 struct tm_restart_node
*n
= *slot
;
3114 n
= ggc_alloc
<tm_restart_node
> ();
3119 tree old
= n
->label_or_list
;
3120 if (TREE_CODE (old
) == LABEL_DECL
)
3121 old
= tree_cons (NULL
, old
, NULL
);
3122 n
->label_or_list
= tree_cons (NULL
, dummy
.label_or_list
, old
);
3126 /* Split block BB as necessary for every builtin function we added, and
3127 wire up the abnormal back edges implied by the transaction restart. */
3130 expand_block_edges (struct tm_region
*const region
, basic_block bb
)
3132 gimple_stmt_iterator gsi
, next_gsi
;
3134 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi
= next_gsi
)
3136 gimple stmt
= gsi_stmt (gsi
);
3140 gsi_next (&next_gsi
);
3142 // ??? Shouldn't we split for any non-pure, non-irrevocable function?
3143 call_stmt
= dyn_cast
<gcall
*> (stmt
);
3145 || (gimple_call_flags (call_stmt
) & ECF_TM_BUILTIN
) == 0)
3148 if (DECL_FUNCTION_CODE (gimple_call_fndecl (call_stmt
))
3149 == BUILT_IN_TM_ABORT
)
3151 // If we have a ``_transaction_cancel [[outer]]'', there is only
3152 // one abnormal edge: to the transaction marked OUTER.
3153 // All compiler-generated instances of BUILT_IN_TM_ABORT have a
3154 // constant argument, which we can examine here. Users invoking
3155 // TM_ABORT directly get what they deserve.
3156 tree arg
= gimple_call_arg (call_stmt
, 0);
3157 if (TREE_CODE (arg
) == INTEGER_CST
3158 && (TREE_INT_CST_LOW (arg
) & AR_OUTERABORT
) != 0
3159 && !decl_is_tm_clone (current_function_decl
))
3161 // Find the GTMA_IS_OUTER transaction.
3162 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3163 if (o
->original_transaction_was_outer
)
3165 split_bb_make_tm_edge (call_stmt
, o
->restart_block
,
3170 // Otherwise, the front-end should have semantically checked
3171 // outer aborts, but in either case the target region is not
3172 // within this function.
3176 // Non-outer, TM aborts have an abnormal edge to the inner-most
3177 // transaction, the one being aborted;
3178 split_bb_make_tm_edge (call_stmt
, region
->restart_block
, gsi
,
3182 // All TM builtins have an abnormal edge to the outer-most transaction.
3183 // We never restart inner transactions. For tm clones, we know a-priori
3184 // that the outer-most transaction is outside the function.
3185 if (decl_is_tm_clone (current_function_decl
))
3188 if (cfun
->gimple_df
->tm_restart
== NULL
)
3189 cfun
->gimple_df
->tm_restart
3190 = hash_table
<tm_restart_hasher
>::create_ggc (31);
3192 // All TM builtins have an abnormal edge to the outer-most transaction.
3193 // We never restart inner transactions.
3194 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3197 split_bb_make_tm_edge (call_stmt
, o
->restart_block
, gsi
, &next_gsi
);
3201 // Delete any tail-call annotation that may have been added.
3202 // The tail-call pass may have mis-identified the commit as being
3203 // a candidate because we had not yet added this restart edge.
3204 gimple_call_set_tail (call_stmt
, false);
3208 /* Entry point to the final expansion of transactional nodes. */
3212 const pass_data pass_data_tm_edges
=
3214 GIMPLE_PASS
, /* type */
3215 "tmedge", /* name */
3216 OPTGROUP_NONE
, /* optinfo_flags */
3217 TV_TRANS_MEM
, /* tv_id */
3218 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3219 0, /* properties_provided */
3220 0, /* properties_destroyed */
3221 0, /* todo_flags_start */
3222 TODO_update_ssa
, /* todo_flags_finish */
3225 class pass_tm_edges
: public gimple_opt_pass
3228 pass_tm_edges (gcc::context
*ctxt
)
3229 : gimple_opt_pass (pass_data_tm_edges
, ctxt
)
3232 /* opt_pass methods: */
3233 virtual unsigned int execute (function
*);
3235 }; // class pass_tm_edges
3238 pass_tm_edges::execute (function
*fun
)
3240 vec
<tm_region_p
> bb_regions
3241 = get_bb_regions_instrumented (/*traverse_clones=*/false,
3242 /*include_uninstrumented_p=*/true);
3243 struct tm_region
*r
;
3246 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3248 expand_block_edges (r
, BASIC_BLOCK_FOR_FN (fun
, i
));
3250 bb_regions
.release ();
3252 /* We've got to release the dominance info now, to indicate that it
3253 must be rebuilt completely. Otherwise we'll crash trying to update
3254 the SSA web in the TODO section following this pass. */
3255 free_dominance_info (CDI_DOMINATORS
);
3256 bitmap_obstack_release (&tm_obstack
);
3257 all_tm_regions
= NULL
;
3265 make_pass_tm_edges (gcc::context
*ctxt
)
3267 return new pass_tm_edges (ctxt
);
3270 /* Helper function for expand_regions. Expand REGION and recurse to
3271 the inner region. Call CALLBACK on each region. CALLBACK returns
3272 NULL to continue the traversal, otherwise a non-null value which
3273 this function will return as well. TRAVERSE_CLONES is true if we
3274 should traverse transactional clones. */
3277 expand_regions_1 (struct tm_region
*region
,
3278 void *(*callback
)(struct tm_region
*, void *),
3280 bool traverse_clones
)
3282 void *retval
= NULL
;
3283 if (region
->exit_blocks
3284 || (traverse_clones
&& decl_is_tm_clone (current_function_decl
)))
3286 retval
= callback (region
, data
);
3292 retval
= expand_regions (region
->inner
, callback
, data
, traverse_clones
);
3299 /* Traverse the regions enclosed and including REGION. Execute
3300 CALLBACK for each region, passing DATA. CALLBACK returns NULL to
3301 continue the traversal, otherwise a non-null value which this
3302 function will return as well. TRAVERSE_CLONES is true if we should
3303 traverse transactional clones. */
3306 expand_regions (struct tm_region
*region
,
3307 void *(*callback
)(struct tm_region
*, void *),
3309 bool traverse_clones
)
3311 void *retval
= NULL
;
3314 retval
= expand_regions_1 (region
, callback
, data
, traverse_clones
);
3317 region
= region
->next
;
3323 /* A unique TM memory operation. */
3324 typedef struct tm_memop
3326 /* Unique ID that all memory operations to the same location have. */
3327 unsigned int value_id
;
3328 /* Address of load/store. */
3332 /* TM memory operation hashtable helpers. */
3334 struct tm_memop_hasher
: typed_free_remove
<tm_memop
>
3336 typedef tm_memop
*value_type
;
3337 typedef tm_memop
*compare_type
;
3338 static inline hashval_t
hash (const tm_memop
*);
3339 static inline bool equal (const tm_memop
*, const tm_memop
*);
3342 /* Htab support. Return a hash value for a `tm_memop'. */
3344 tm_memop_hasher::hash (const tm_memop
*mem
)
3346 tree addr
= mem
->addr
;
3347 /* We drill down to the SSA_NAME/DECL for the hash, but equality is
3348 actually done with operand_equal_p (see tm_memop_eq). */
3349 if (TREE_CODE (addr
) == ADDR_EXPR
)
3350 addr
= TREE_OPERAND (addr
, 0);
3351 return iterative_hash_expr (addr
, 0);
3354 /* Htab support. Return true if two tm_memop's are the same. */
3356 tm_memop_hasher::equal (const tm_memop
*mem1
, const tm_memop
*mem2
)
3358 return operand_equal_p (mem1
->addr
, mem2
->addr
, 0);
3361 /* Sets for solving data flow equations in the memory optimization pass. */
3362 struct tm_memopt_bitmaps
3364 /* Stores available to this BB upon entry. Basically, stores that
3365 dominate this BB. */
3366 bitmap store_avail_in
;
3367 /* Stores available at the end of this BB. */
3368 bitmap store_avail_out
;
3369 bitmap store_antic_in
;
3370 bitmap store_antic_out
;
3371 /* Reads available to this BB upon entry. Basically, reads that
3372 dominate this BB. */
3373 bitmap read_avail_in
;
3374 /* Reads available at the end of this BB. */
3375 bitmap read_avail_out
;
3376 /* Reads performed in this BB. */
3378 /* Writes performed in this BB. */
3381 /* Temporary storage for pass. */
3382 /* Is the current BB in the worklist? */
3383 bool avail_in_worklist_p
;
3384 /* Have we visited this BB? */
3388 static bitmap_obstack tm_memopt_obstack
;
3390 /* Unique counter for TM loads and stores. Loads and stores of the
3391 same address get the same ID. */
3392 static unsigned int tm_memopt_value_id
;
3393 static hash_table
<tm_memop_hasher
> *tm_memopt_value_numbers
;
3395 #define STORE_AVAIL_IN(BB) \
3396 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in
3397 #define STORE_AVAIL_OUT(BB) \
3398 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out
3399 #define STORE_ANTIC_IN(BB) \
3400 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in
3401 #define STORE_ANTIC_OUT(BB) \
3402 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out
3403 #define READ_AVAIL_IN(BB) \
3404 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in
3405 #define READ_AVAIL_OUT(BB) \
3406 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out
3407 #define READ_LOCAL(BB) \
3408 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local
3409 #define STORE_LOCAL(BB) \
3410 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local
3411 #define AVAIL_IN_WORKLIST_P(BB) \
3412 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p
3413 #define BB_VISITED_P(BB) \
3414 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p
3416 /* Given a TM load/store in STMT, return the value number for the address
3420 tm_memopt_value_number (gimple stmt
, enum insert_option op
)
3422 struct tm_memop tmpmem
, *mem
;
3425 gcc_assert (is_tm_load (stmt
) || is_tm_store (stmt
));
3426 tmpmem
.addr
= gimple_call_arg (stmt
, 0);
3427 slot
= tm_memopt_value_numbers
->find_slot (&tmpmem
, op
);
3430 else if (op
== INSERT
)
3432 mem
= XNEW (struct tm_memop
);
3434 mem
->value_id
= tm_memopt_value_id
++;
3435 mem
->addr
= tmpmem
.addr
;
3439 return mem
->value_id
;
3442 /* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */
3445 tm_memopt_accumulate_memops (basic_block bb
)
3447 gimple_stmt_iterator gsi
;
3449 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3451 gimple stmt
= gsi_stmt (gsi
);
3455 if (is_tm_store (stmt
))
3456 bits
= STORE_LOCAL (bb
);
3457 else if (is_tm_load (stmt
))
3458 bits
= READ_LOCAL (bb
);
3462 loc
= tm_memopt_value_number (stmt
, INSERT
);
3463 bitmap_set_bit (bits
, loc
);
3466 fprintf (dump_file
, "TM memopt (%s): value num=%d, BB=%d, addr=",
3467 is_tm_load (stmt
) ? "LOAD" : "STORE", loc
,
3468 gimple_bb (stmt
)->index
);
3469 print_generic_expr (dump_file
, gimple_call_arg (stmt
, 0), 0);
3470 fprintf (dump_file
, "\n");
3475 /* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */
3478 dump_tm_memopt_set (const char *set_name
, bitmap bits
)
3482 const char *comma
= "";
3484 fprintf (dump_file
, "TM memopt: %s: [", set_name
);
3485 EXECUTE_IF_SET_IN_BITMAP (bits
, 0, i
, bi
)
3487 hash_table
<tm_memop_hasher
>::iterator hi
;
3488 struct tm_memop
*mem
= NULL
;
3490 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */
3491 FOR_EACH_HASH_TABLE_ELEMENT (*tm_memopt_value_numbers
, mem
, tm_memop_t
, hi
)
3492 if (mem
->value_id
== i
)
3494 gcc_assert (mem
->value_id
== i
);
3495 fprintf (dump_file
, "%s", comma
);
3497 print_generic_expr (dump_file
, mem
->addr
, 0);
3499 fprintf (dump_file
, "]\n");
3502 /* Prettily dump all of the memopt sets in BLOCKS. */
3505 dump_tm_memopt_sets (vec
<basic_block
> blocks
)
3510 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3512 fprintf (dump_file
, "------------BB %d---------\n", bb
->index
);
3513 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb
));
3514 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb
));
3515 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb
));
3516 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb
));
3517 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb
));
3518 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb
));
3522 /* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */
3525 tm_memopt_compute_avin (basic_block bb
)
3530 /* Seed with the AVOUT of any predecessor. */
3531 for (ix
= 0; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3533 e
= EDGE_PRED (bb
, ix
);
3534 /* Make sure we have already visited this BB, and is thus
3537 If e->src->aux is NULL, this predecessor is actually on an
3538 enclosing transaction. We only care about the current
3539 transaction, so ignore it. */
3540 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3542 bitmap_copy (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3543 bitmap_copy (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3548 for (; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3550 e
= EDGE_PRED (bb
, ix
);
3551 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3553 bitmap_and_into (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3554 bitmap_and_into (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3558 BB_VISITED_P (bb
) = true;
3561 /* Compute the STORE_ANTIC_IN for the basic block BB. */
3564 tm_memopt_compute_antin (basic_block bb
)
3569 /* Seed with the ANTIC_OUT of any successor. */
3570 for (ix
= 0; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3572 e
= EDGE_SUCC (bb
, ix
);
3573 /* Make sure we have already visited this BB, and is thus
3575 if (BB_VISITED_P (e
->dest
))
3577 bitmap_copy (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3582 for (; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3584 e
= EDGE_SUCC (bb
, ix
);
3585 if (BB_VISITED_P (e
->dest
))
3586 bitmap_and_into (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3589 BB_VISITED_P (bb
) = true;
3592 /* Compute the AVAIL sets for every basic block in BLOCKS.
3594 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows:
3596 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb])
3597 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors])
3599 This is basically what we do in lcm's compute_available(), but here
3600 we calculate two sets of sets (one for STOREs and one for READs),
3601 and we work on a region instead of the entire CFG.
3603 REGION is the TM region.
3604 BLOCKS are the basic blocks in the region. */
3607 tm_memopt_compute_available (struct tm_region
*region
,
3608 vec
<basic_block
> blocks
)
3611 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3612 unsigned int qlen
, i
;
3616 /* Allocate a worklist array/queue. Entries are only added to the
3617 list if they were not already on the list. So the size is
3618 bounded by the number of basic blocks in the region. */
3619 qlen
= blocks
.length () - 1;
3620 qin
= qout
= worklist
=
3621 XNEWVEC (basic_block
, qlen
);
3623 /* Put every block in the region on the worklist. */
3624 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3626 /* Seed AVAIL_OUT with the LOCAL set. */
3627 bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_LOCAL (bb
));
3628 bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_LOCAL (bb
));
3630 AVAIL_IN_WORKLIST_P (bb
) = true;
3631 /* No need to insert the entry block, since it has an AVIN of
3632 null, and an AVOUT that has already been seeded in. */
3633 if (bb
!= region
->entry_block
)
3637 /* The entry block has been initialized with the local sets. */
3638 BB_VISITED_P (region
->entry_block
) = true;
3641 qend
= &worklist
[qlen
];
3643 /* Iterate until the worklist is empty. */
3646 /* Take the first entry off the worklist. */
3653 /* This block can be added to the worklist again if necessary. */
3654 AVAIL_IN_WORKLIST_P (bb
) = false;
3655 tm_memopt_compute_avin (bb
);
3657 /* Note: We do not add the LOCAL sets here because we already
3658 seeded the AVAIL_OUT sets with them. */
3659 changed
= bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_AVAIL_IN (bb
));
3660 changed
|= bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_AVAIL_IN (bb
));
3662 && (region
->exit_blocks
== NULL
3663 || !bitmap_bit_p (region
->exit_blocks
, bb
->index
)))
3664 /* If the out state of this block changed, then we need to add
3665 its successors to the worklist if they are not already in. */
3666 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3667 if (!AVAIL_IN_WORKLIST_P (e
->dest
)
3668 && e
->dest
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
3671 AVAIL_IN_WORKLIST_P (e
->dest
) = true;
3682 dump_tm_memopt_sets (blocks
);
3685 /* Compute ANTIC sets for every basic block in BLOCKS.
3687 We compute STORE_ANTIC_OUT as follows:
3689 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb])
3690 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors])
3692 REGION is the TM region.
3693 BLOCKS are the basic blocks in the region. */
3696 tm_memopt_compute_antic (struct tm_region
*region
,
3697 vec
<basic_block
> blocks
)
3700 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3705 /* Allocate a worklist array/queue. Entries are only added to the
3706 list if they were not already on the list. So the size is
3707 bounded by the number of basic blocks in the region. */
3708 qin
= qout
= worklist
= XNEWVEC (basic_block
, blocks
.length ());
3710 for (qlen
= 0, i
= blocks
.length () - 1; i
>= 0; --i
)
3714 /* Seed ANTIC_OUT with the LOCAL set. */
3715 bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_LOCAL (bb
));
3717 /* Put every block in the region on the worklist. */
3718 AVAIL_IN_WORKLIST_P (bb
) = true;
3719 /* No need to insert exit blocks, since their ANTIC_IN is NULL,
3720 and their ANTIC_OUT has already been seeded in. */
3721 if (region
->exit_blocks
3722 && !bitmap_bit_p (region
->exit_blocks
, bb
->index
))
3729 /* The exit blocks have been initialized with the local sets. */
3730 if (region
->exit_blocks
)
3734 EXECUTE_IF_SET_IN_BITMAP (region
->exit_blocks
, 0, i
, bi
)
3735 BB_VISITED_P (BASIC_BLOCK_FOR_FN (cfun
, i
)) = true;
3739 qend
= &worklist
[qlen
];
3741 /* Iterate until the worklist is empty. */
3744 /* Take the first entry off the worklist. */
3751 /* This block can be added to the worklist again if necessary. */
3752 AVAIL_IN_WORKLIST_P (bb
) = false;
3753 tm_memopt_compute_antin (bb
);
3755 /* Note: We do not add the LOCAL sets here because we already
3756 seeded the ANTIC_OUT sets with them. */
3757 if (bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_ANTIC_IN (bb
))
3758 && bb
!= region
->entry_block
)
3759 /* If the out state of this block changed, then we need to add
3760 its predecessors to the worklist if they are not already in. */
3761 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
3762 if (!AVAIL_IN_WORKLIST_P (e
->src
))
3765 AVAIL_IN_WORKLIST_P (e
->src
) = true;
3776 dump_tm_memopt_sets (blocks
);
3779 /* Offsets of load variants from TM_LOAD. For example,
3780 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*.
3781 See gtm-builtins.def. */
3782 #define TRANSFORM_RAR 1
3783 #define TRANSFORM_RAW 2
3784 #define TRANSFORM_RFW 3
3785 /* Offsets of store variants from TM_STORE. */
3786 #define TRANSFORM_WAR 1
3787 #define TRANSFORM_WAW 2
3789 /* Inform about a load/store optimization. */
3792 dump_tm_memopt_transform (gimple stmt
)
3796 fprintf (dump_file
, "TM memopt: transforming: ");
3797 print_gimple_stmt (dump_file
, stmt
, 0, 0);
3798 fprintf (dump_file
, "\n");
3802 /* Perform a read/write optimization. Replaces the TM builtin in STMT
3803 by a builtin that is OFFSET entries down in the builtins table in
3804 gtm-builtins.def. */
3807 tm_memopt_transform_stmt (unsigned int offset
,
3809 gimple_stmt_iterator
*gsi
)
3811 tree fn
= gimple_call_fn (stmt
);
3812 gcc_assert (TREE_CODE (fn
) == ADDR_EXPR
);
3813 TREE_OPERAND (fn
, 0)
3814 = builtin_decl_explicit ((enum built_in_function
)
3815 (DECL_FUNCTION_CODE (TREE_OPERAND (fn
, 0))
3817 gimple_call_set_fn (stmt
, fn
);
3818 gsi_replace (gsi
, stmt
, true);
3819 dump_tm_memopt_transform (stmt
);
3822 /* Perform the actual TM memory optimization transformations in the
3823 basic blocks in BLOCKS. */
3826 tm_memopt_transform_blocks (vec
<basic_block
> blocks
)
3830 gimple_stmt_iterator gsi
;
3832 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3834 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3836 gimple stmt
= gsi_stmt (gsi
);
3837 bitmap read_avail
= READ_AVAIL_IN (bb
);
3838 bitmap store_avail
= STORE_AVAIL_IN (bb
);
3839 bitmap store_antic
= STORE_ANTIC_OUT (bb
);
3842 if (is_tm_simple_load (stmt
))
3844 gcall
*call_stmt
= as_a
<gcall
*> (stmt
);
3845 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3846 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3847 tm_memopt_transform_stmt (TRANSFORM_RAW
, call_stmt
, &gsi
);
3848 else if (store_antic
&& bitmap_bit_p (store_antic
, loc
))
3850 tm_memopt_transform_stmt (TRANSFORM_RFW
, call_stmt
, &gsi
);
3851 bitmap_set_bit (store_avail
, loc
);
3853 else if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3854 tm_memopt_transform_stmt (TRANSFORM_RAR
, call_stmt
, &gsi
);
3856 bitmap_set_bit (read_avail
, loc
);
3858 else if (is_tm_simple_store (stmt
))
3860 gcall
*call_stmt
= as_a
<gcall
*> (stmt
);
3861 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3862 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3863 tm_memopt_transform_stmt (TRANSFORM_WAW
, call_stmt
, &gsi
);
3866 if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3867 tm_memopt_transform_stmt (TRANSFORM_WAR
, call_stmt
, &gsi
);
3868 bitmap_set_bit (store_avail
, loc
);
3875 /* Return a new set of bitmaps for a BB. */
3877 static struct tm_memopt_bitmaps
*
3878 tm_memopt_init_sets (void)
3880 struct tm_memopt_bitmaps
*b
3881 = XOBNEW (&tm_memopt_obstack
.obstack
, struct tm_memopt_bitmaps
);
3882 b
->store_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3883 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3884 b
->store_antic_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3885 b
->store_antic_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3886 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3887 b
->read_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3888 b
->read_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3889 b
->read_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3890 b
->store_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3894 /* Free sets computed for each BB. */
3897 tm_memopt_free_sets (vec
<basic_block
> blocks
)
3902 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3906 /* Clear the visited bit for every basic block in BLOCKS. */
3909 tm_memopt_clear_visited (vec
<basic_block
> blocks
)
3914 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3915 BB_VISITED_P (bb
) = false;
3918 /* Replace TM load/stores with hints for the runtime. We handle
3919 things like read-after-write, write-after-read, read-after-read,
3920 read-for-write, etc. */
3923 execute_tm_memopt (void)
3925 struct tm_region
*region
;
3926 vec
<basic_block
> bbs
;
3928 tm_memopt_value_id
= 0;
3929 tm_memopt_value_numbers
= new hash_table
<tm_memop_hasher
> (10);
3931 for (region
= all_tm_regions
; region
; region
= region
->next
)
3933 /* All the TM stores/loads in the current region. */
3937 bitmap_obstack_initialize (&tm_memopt_obstack
);
3939 /* Save all BBs for the current region. */
3940 bbs
= get_tm_region_blocks (region
->entry_block
,
3941 region
->exit_blocks
,
3946 /* Collect all the memory operations. */
3947 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
3949 bb
->aux
= tm_memopt_init_sets ();
3950 tm_memopt_accumulate_memops (bb
);
3953 /* Solve data flow equations and transform each block accordingly. */
3954 tm_memopt_clear_visited (bbs
);
3955 tm_memopt_compute_available (region
, bbs
);
3956 tm_memopt_clear_visited (bbs
);
3957 tm_memopt_compute_antic (region
, bbs
);
3958 tm_memopt_transform_blocks (bbs
);
3960 tm_memopt_free_sets (bbs
);
3962 bitmap_obstack_release (&tm_memopt_obstack
);
3963 tm_memopt_value_numbers
->empty ();
3966 delete tm_memopt_value_numbers
;
3967 tm_memopt_value_numbers
= NULL
;
3973 const pass_data pass_data_tm_memopt
=
3975 GIMPLE_PASS
, /* type */
3976 "tmmemopt", /* name */
3977 OPTGROUP_NONE
, /* optinfo_flags */
3978 TV_TRANS_MEM
, /* tv_id */
3979 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3980 0, /* properties_provided */
3981 0, /* properties_destroyed */
3982 0, /* todo_flags_start */
3983 0, /* todo_flags_finish */
3986 class pass_tm_memopt
: public gimple_opt_pass
3989 pass_tm_memopt (gcc::context
*ctxt
)
3990 : gimple_opt_pass (pass_data_tm_memopt
, ctxt
)
3993 /* opt_pass methods: */
3994 virtual bool gate (function
*) { return flag_tm
&& optimize
> 0; }
3995 virtual unsigned int execute (function
*) { return execute_tm_memopt (); }
3997 }; // class pass_tm_memopt
4002 make_pass_tm_memopt (gcc::context
*ctxt
)
4004 return new pass_tm_memopt (ctxt
);
4008 /* Interprocedual analysis for the creation of transactional clones.
4009 The aim of this pass is to find which functions are referenced in
4010 a non-irrevocable transaction context, and for those over which
4011 we have control (or user directive), create a version of the
4012 function which uses only the transactional interface to reference
4013 protected memories. This analysis proceeds in several steps:
4015 (1) Collect the set of all possible transactional clones:
4017 (a) For all local public functions marked tm_callable, push
4018 it onto the tm_callee queue.
4020 (b) For all local functions, scan for calls in transaction blocks.
4021 Push the caller and callee onto the tm_caller and tm_callee
4022 queues. Count the number of callers for each callee.
4024 (c) For each local function on the callee list, assume we will
4025 create a transactional clone. Push *all* calls onto the
4026 callee queues; count the number of clone callers separately
4027 to the number of original callers.
4029 (2) Propagate irrevocable status up the dominator tree:
4031 (a) Any external function on the callee list that is not marked
4032 tm_callable is irrevocable. Push all callers of such onto
4035 (b) For each function on the worklist, mark each block that
4036 contains an irrevocable call. Use the AND operator to
4037 propagate that mark up the dominator tree.
4039 (c) If we reach the entry block for a possible transactional
4040 clone, then the transactional clone is irrevocable, and
4041 we should not create the clone after all. Push all
4042 callers onto the worklist.
4044 (d) Place tm_irrevocable calls at the beginning of the relevant
4045 blocks. Special case here is the entry block for the entire
4046 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for
4047 the library to begin the region in serial mode. Decrement
4048 the call count for all callees in the irrevocable region.
4050 (3) Create the transactional clones:
4052 Any tm_callee that still has a non-zero call count is cloned.
4055 /* This structure is stored in the AUX field of each cgraph_node. */
4056 struct tm_ipa_cg_data
4058 /* The clone of the function that got created. */
4059 struct cgraph_node
*clone
;
4061 /* The tm regions in the normal function. */
4062 struct tm_region
*all_tm_regions
;
4064 /* The blocks of the normal/clone functions that contain irrevocable
4065 calls, or blocks that are post-dominated by irrevocable calls. */
4066 bitmap irrevocable_blocks_normal
;
4067 bitmap irrevocable_blocks_clone
;
4069 /* The blocks of the normal function that are involved in transactions. */
4070 bitmap transaction_blocks_normal
;
4072 /* The number of callers to the transactional clone of this function
4073 from normal and transactional clones respectively. */
4074 unsigned tm_callers_normal
;
4075 unsigned tm_callers_clone
;
4077 /* True if all calls to this function's transactional clone
4078 are irrevocable. Also automatically true if the function
4079 has no transactional clone. */
4080 bool is_irrevocable
;
4082 /* Flags indicating the presence of this function in various queues. */
4083 bool in_callee_queue
;
4086 /* Flags indicating the kind of scan desired while in the worklist. */
4087 bool want_irr_scan_normal
;
4090 typedef vec
<cgraph_node
*> cgraph_node_queue
;
4092 /* Return the ipa data associated with NODE, allocating zeroed memory
4093 if necessary. TRAVERSE_ALIASES is true if we must traverse aliases
4094 and set *NODE accordingly. */
4096 static struct tm_ipa_cg_data
*
4097 get_cg_data (struct cgraph_node
**node
, bool traverse_aliases
)
4099 struct tm_ipa_cg_data
*d
;
4101 if (traverse_aliases
&& (*node
)->alias
)
4102 *node
= (*node
)->get_alias_target ();
4104 d
= (struct tm_ipa_cg_data
*) (*node
)->aux
;
4108 d
= (struct tm_ipa_cg_data
*)
4109 obstack_alloc (&tm_obstack
.obstack
, sizeof (*d
));
4110 (*node
)->aux
= (void *) d
;
4111 memset (d
, 0, sizeof (*d
));
4117 /* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that
4118 it is already present. */
4121 maybe_push_queue (struct cgraph_node
*node
,
4122 cgraph_node_queue
*queue_p
, bool *in_queue_p
)
4127 queue_p
->safe_push (node
);
4131 /* Duplicate the basic blocks in QUEUE for use in the uninstrumented
4132 code path. QUEUE are the basic blocks inside the transaction
4133 represented in REGION.
4135 Later in split_code_paths() we will add the conditional to choose
4136 between the two alternatives. */
4139 ipa_uninstrument_transaction (struct tm_region
*region
,
4140 vec
<basic_block
> queue
)
4142 gimple transaction
= region
->transaction_stmt
;
4143 basic_block transaction_bb
= gimple_bb (transaction
);
4144 int n
= queue
.length ();
4145 basic_block
*new_bbs
= XNEWVEC (basic_block
, n
);
4147 copy_bbs (queue
.address (), n
, new_bbs
, NULL
, 0, NULL
, NULL
, transaction_bb
,
4149 edge e
= make_edge (transaction_bb
, new_bbs
[0], EDGE_TM_UNINSTRUMENTED
);
4150 add_phi_args_after_copy (new_bbs
, n
, e
);
4152 // Now we will have a GIMPLE_ATOMIC with 3 possible edges out of it.
4153 // a) EDGE_FALLTHRU into the transaction
4154 // b) EDGE_TM_ABORT out of the transaction
4155 // c) EDGE_TM_UNINSTRUMENTED into the uninstrumented blocks.
4160 /* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone.
4161 Queue all callees within block BB. */
4164 ipa_tm_scan_calls_block (cgraph_node_queue
*callees_p
,
4165 basic_block bb
, bool for_clone
)
4167 gimple_stmt_iterator gsi
;
4169 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4171 gimple stmt
= gsi_stmt (gsi
);
4172 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4174 tree fndecl
= gimple_call_fndecl (stmt
);
4177 struct tm_ipa_cg_data
*d
;
4179 struct cgraph_node
*node
;
4181 if (is_tm_ending_fndecl (fndecl
))
4183 if (find_tm_replacement_function (fndecl
))
4186 node
= cgraph_node::get (fndecl
);
4187 gcc_assert (node
!= NULL
);
4188 d
= get_cg_data (&node
, true);
4190 pcallers
= (for_clone
? &d
->tm_callers_clone
4191 : &d
->tm_callers_normal
);
4194 maybe_push_queue (node
, callees_p
, &d
->in_callee_queue
);
4200 /* Scan all calls in NODE that are within a transaction region,
4201 and push the resulting nodes into the callee queue. */
4204 ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data
*d
,
4205 cgraph_node_queue
*callees_p
)
4207 struct tm_region
*r
;
4209 d
->transaction_blocks_normal
= BITMAP_ALLOC (&tm_obstack
);
4210 d
->all_tm_regions
= all_tm_regions
;
4212 for (r
= all_tm_regions
; r
; r
= r
->next
)
4214 vec
<basic_block
> bbs
;
4218 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
, NULL
,
4219 d
->transaction_blocks_normal
, false);
4221 // Generate the uninstrumented code path for this transaction.
4222 ipa_uninstrument_transaction (r
, bbs
);
4224 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
4225 ipa_tm_scan_calls_block (callees_p
, bb
, false);
4230 // ??? copy_bbs should maintain cgraph edges for the blocks as it is
4231 // copying them, rather than forcing us to do this externally.
4232 cgraph_edge::rebuild_edges ();
4234 // ??? In ipa_uninstrument_transaction we don't try to update dominators
4235 // because copy_bbs doesn't return a VEC like iterate_fix_dominators expects.
4236 // Instead, just release dominators here so update_ssa recomputes them.
4237 free_dominance_info (CDI_DOMINATORS
);
4239 // When building the uninstrumented code path, copy_bbs will have invoked
4240 // create_new_def_for starting an "ssa update context". There is only one
4241 // instance of this context, so resolve ssa updates before moving on to
4242 // the next function.
4243 update_ssa (TODO_update_ssa
);
4246 /* Scan all calls in NODE as if this is the transactional clone,
4247 and push the destinations into the callee queue. */
4250 ipa_tm_scan_calls_clone (struct cgraph_node
*node
,
4251 cgraph_node_queue
*callees_p
)
4253 struct function
*fn
= DECL_STRUCT_FUNCTION (node
->decl
);
4256 FOR_EACH_BB_FN (bb
, fn
)
4257 ipa_tm_scan_calls_block (callees_p
, bb
, true);
4260 /* The function NODE has been detected to be irrevocable. Push all
4261 of its callers onto WORKLIST for the purpose of re-scanning them. */
4264 ipa_tm_note_irrevocable (struct cgraph_node
*node
,
4265 cgraph_node_queue
*worklist_p
)
4267 struct tm_ipa_cg_data
*d
= get_cg_data (&node
, true);
4268 struct cgraph_edge
*e
;
4270 d
->is_irrevocable
= true;
4272 for (e
= node
->callers
; e
; e
= e
->next_caller
)
4275 struct cgraph_node
*caller
;
4277 /* Don't examine recursive calls. */
4278 if (e
->caller
== node
)
4280 /* Even if we think we can go irrevocable, believe the user
4282 if (is_tm_safe_or_pure (e
->caller
->decl
))
4286 d
= get_cg_data (&caller
, true);
4288 /* Check if the callee is in a transactional region. If so,
4289 schedule the function for normal re-scan as well. */
4290 bb
= gimple_bb (e
->call_stmt
);
4291 gcc_assert (bb
!= NULL
);
4292 if (d
->transaction_blocks_normal
4293 && bitmap_bit_p (d
->transaction_blocks_normal
, bb
->index
))
4294 d
->want_irr_scan_normal
= true;
4296 maybe_push_queue (caller
, worklist_p
, &d
->in_worklist
);
4300 /* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement
4301 within the block is irrevocable. */
4304 ipa_tm_scan_irr_block (basic_block bb
)
4306 gimple_stmt_iterator gsi
;
4309 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4311 gimple stmt
= gsi_stmt (gsi
);
4312 switch (gimple_code (stmt
))
4315 if (gimple_assign_single_p (stmt
))
4317 tree lhs
= gimple_assign_lhs (stmt
);
4318 tree rhs
= gimple_assign_rhs1 (stmt
);
4319 if (volatile_var_p (lhs
) || volatile_var_p (rhs
))
4326 tree lhs
= gimple_call_lhs (stmt
);
4327 if (lhs
&& volatile_var_p (lhs
))
4330 if (is_tm_pure_call (stmt
))
4333 fn
= gimple_call_fn (stmt
);
4335 /* Functions with the attribute are by definition irrevocable. */
4336 if (is_tm_irrevocable (fn
))
4339 /* For direct function calls, go ahead and check for replacement
4340 functions, or transitive irrevocable functions. For indirect
4341 functions, we'll ask the runtime. */
4342 if (TREE_CODE (fn
) == ADDR_EXPR
)
4344 struct tm_ipa_cg_data
*d
;
4345 struct cgraph_node
*node
;
4347 fn
= TREE_OPERAND (fn
, 0);
4348 if (is_tm_ending_fndecl (fn
))
4350 if (find_tm_replacement_function (fn
))
4353 node
= cgraph_node::get (fn
);
4354 d
= get_cg_data (&node
, true);
4356 /* Return true if irrevocable, but above all, believe
4358 if (d
->is_irrevocable
4359 && !is_tm_safe_or_pure (fn
))
4366 /* ??? The Approved Method of indicating that an inline
4367 assembly statement is not relevant to the transaction
4368 is to wrap it in a __tm_waiver block. This is not
4369 yet implemented, so we can't check for it. */
4370 if (is_tm_safe (current_function_decl
))
4372 tree t
= build1 (NOP_EXPR
, void_type_node
, size_zero_node
);
4373 SET_EXPR_LOCATION (t
, gimple_location (stmt
));
4374 error ("%Kasm not allowed in %<transaction_safe%> function", t
);
4386 /* For each of the blocks seeded witin PQUEUE, walk the CFG looking
4387 for new irrevocable blocks, marking them in NEW_IRR. Don't bother
4388 scanning past OLD_IRR or EXIT_BLOCKS. */
4391 ipa_tm_scan_irr_blocks (vec
<basic_block
> *pqueue
, bitmap new_irr
,
4392 bitmap old_irr
, bitmap exit_blocks
)
4394 bool any_new_irr
= false;
4397 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4401 basic_block bb
= pqueue
->pop ();
4403 /* Don't re-scan blocks we know already are irrevocable. */
4404 if (old_irr
&& bitmap_bit_p (old_irr
, bb
->index
))
4407 if (ipa_tm_scan_irr_block (bb
))
4409 bitmap_set_bit (new_irr
, bb
->index
);
4412 else if (exit_blocks
== NULL
|| !bitmap_bit_p (exit_blocks
, bb
->index
))
4414 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4415 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
4417 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
4418 pqueue
->safe_push (e
->dest
);
4422 while (!pqueue
->is_empty ());
4424 BITMAP_FREE (visited_blocks
);
4429 /* Propagate the irrevocable property both up and down the dominator tree.
4430 BB is the current block being scanned; EXIT_BLOCKS are the edges of the
4431 TM regions; OLD_IRR are the results of a previous scan of the dominator
4432 tree which has been fully propagated; NEW_IRR is the set of new blocks
4433 which are gaining the irrevocable property during the current scan. */
4436 ipa_tm_propagate_irr (basic_block entry_block
, bitmap new_irr
,
4437 bitmap old_irr
, bitmap exit_blocks
)
4439 vec
<basic_block
> bbs
;
4440 bitmap all_region_blocks
;
4442 /* If this block is in the old set, no need to rescan. */
4443 if (old_irr
&& bitmap_bit_p (old_irr
, entry_block
->index
))
4446 all_region_blocks
= BITMAP_ALLOC (&tm_obstack
);
4447 bbs
= get_tm_region_blocks (entry_block
, exit_blocks
, NULL
,
4448 all_region_blocks
, false);
4451 basic_block bb
= bbs
.pop ();
4452 bool this_irr
= bitmap_bit_p (new_irr
, bb
->index
);
4453 bool all_son_irr
= false;
4457 /* Propagate up. If my children are, I am too, but we must have
4458 at least one child that is. */
4461 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4463 if (!bitmap_bit_p (new_irr
, e
->dest
->index
))
4465 all_son_irr
= false;
4473 /* Add block to new_irr if it hasn't already been processed. */
4474 if (!old_irr
|| !bitmap_bit_p (old_irr
, bb
->index
))
4476 bitmap_set_bit (new_irr
, bb
->index
);
4482 /* Propagate down to everyone we immediately dominate. */
4486 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
4488 son
= next_dom_son (CDI_DOMINATORS
, son
))
4490 /* Make sure block is actually in a TM region, and it
4491 isn't already in old_irr. */
4492 if ((!old_irr
|| !bitmap_bit_p (old_irr
, son
->index
))
4493 && bitmap_bit_p (all_region_blocks
, son
->index
))
4494 bitmap_set_bit (new_irr
, son
->index
);
4498 while (!bbs
.is_empty ());
4500 BITMAP_FREE (all_region_blocks
);
4505 ipa_tm_decrement_clone_counts (basic_block bb
, bool for_clone
)
4507 gimple_stmt_iterator gsi
;
4509 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4511 gimple stmt
= gsi_stmt (gsi
);
4512 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4514 tree fndecl
= gimple_call_fndecl (stmt
);
4517 struct tm_ipa_cg_data
*d
;
4519 struct cgraph_node
*tnode
;
4521 if (is_tm_ending_fndecl (fndecl
))
4523 if (find_tm_replacement_function (fndecl
))
4526 tnode
= cgraph_node::get (fndecl
);
4527 d
= get_cg_data (&tnode
, true);
4529 pcallers
= (for_clone
? &d
->tm_callers_clone
4530 : &d
->tm_callers_normal
);
4532 gcc_assert (*pcallers
> 0);
4539 /* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions,
4540 as well as other irrevocable actions such as inline assembly. Mark all
4541 such blocks as irrevocable and decrement the number of calls to
4542 transactional clones. Return true if, for the transactional clone, the
4543 entire function is irrevocable. */
4546 ipa_tm_scan_irr_function (struct cgraph_node
*node
, bool for_clone
)
4548 struct tm_ipa_cg_data
*d
;
4549 bitmap new_irr
, old_irr
;
4552 /* Builtin operators (operator new, and such). */
4553 if (DECL_STRUCT_FUNCTION (node
->decl
) == NULL
4554 || DECL_STRUCT_FUNCTION (node
->decl
)->cfg
== NULL
)
4557 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
4558 calculate_dominance_info (CDI_DOMINATORS
);
4560 d
= get_cg_data (&node
, true);
4561 auto_vec
<basic_block
, 10> queue
;
4562 new_irr
= BITMAP_ALLOC (&tm_obstack
);
4564 /* Scan each tm region, propagating irrevocable status through the tree. */
4567 old_irr
= d
->irrevocable_blocks_clone
;
4568 queue
.quick_push (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)));
4569 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
, NULL
))
4571 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
4574 ret
= bitmap_bit_p (new_irr
,
4575 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
))->index
);
4580 struct tm_region
*region
;
4582 old_irr
= d
->irrevocable_blocks_normal
;
4583 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
4585 queue
.quick_push (region
->entry_block
);
4586 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
,
4587 region
->exit_blocks
))
4588 ipa_tm_propagate_irr (region
->entry_block
, new_irr
, old_irr
,
4589 region
->exit_blocks
);
4593 /* If we found any new irrevocable blocks, reduce the call count for
4594 transactional clones within the irrevocable blocks. Save the new
4595 set of irrevocable blocks for next time. */
4596 if (!bitmap_empty_p (new_irr
))
4598 bitmap_iterator bmi
;
4601 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4602 ipa_tm_decrement_clone_counts (BASIC_BLOCK_FOR_FN (cfun
, i
),
4607 bitmap_ior_into (old_irr
, new_irr
);
4608 BITMAP_FREE (new_irr
);
4611 d
->irrevocable_blocks_clone
= new_irr
;
4613 d
->irrevocable_blocks_normal
= new_irr
;
4615 if (dump_file
&& new_irr
)
4618 bitmap_iterator bmi
;
4621 dname
= lang_hooks
.decl_printable_name (current_function_decl
, 2);
4622 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4623 fprintf (dump_file
, "%s: bb %d goes irrevocable\n", dname
, i
);
4627 BITMAP_FREE (new_irr
);
4634 /* Return true if, for the transactional clone of NODE, any call
4635 may enter irrevocable mode. */
4638 ipa_tm_mayenterirr_function (struct cgraph_node
*node
)
4640 struct tm_ipa_cg_data
*d
;
4644 d
= get_cg_data (&node
, true);
4646 flags
= flags_from_decl_or_type (decl
);
4648 /* Handle some TM builtins. Ordinarily these aren't actually generated
4649 at this point, but handling these functions when written in by the
4650 user makes it easier to build unit tests. */
4651 if (flags
& ECF_TM_BUILTIN
)
4654 /* Filter out all functions that are marked. */
4655 if (flags
& ECF_TM_PURE
)
4657 if (is_tm_safe (decl
))
4659 if (is_tm_irrevocable (decl
))
4661 if (is_tm_callable (decl
))
4663 if (find_tm_replacement_function (decl
))
4666 /* If we aren't seeing the final version of the function we don't
4667 know what it will contain at runtime. */
4668 if (node
->get_availability () < AVAIL_AVAILABLE
)
4671 /* If the function must go irrevocable, then of course true. */
4672 if (d
->is_irrevocable
)
4675 /* If there are any blocks marked irrevocable, then the function
4676 as a whole may enter irrevocable. */
4677 if (d
->irrevocable_blocks_clone
)
4680 /* We may have previously marked this function as tm_may_enter_irr;
4681 see pass_diagnose_tm_blocks. */
4682 if (node
->local
.tm_may_enter_irr
)
4685 /* Recurse on the main body for aliases. In general, this will
4686 result in one of the bits above being set so that we will not
4687 have to recurse next time. */
4689 return ipa_tm_mayenterirr_function (cgraph_node::get (node
->thunk
.alias
));
4691 /* What remains is unmarked local functions without items that force
4692 the function to go irrevocable. */
4696 /* Diagnose calls from transaction_safe functions to unmarked
4697 functions that are determined to not be safe. */
4700 ipa_tm_diagnose_tm_safe (struct cgraph_node
*node
)
4702 struct cgraph_edge
*e
;
4704 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4705 if (!is_tm_callable (e
->callee
->decl
)
4706 && e
->callee
->local
.tm_may_enter_irr
)
4707 error_at (gimple_location (e
->call_stmt
),
4708 "unsafe function call %qD within "
4709 "%<transaction_safe%> function", e
->callee
->decl
);
4712 /* Diagnose call from atomic transactions to unmarked functions
4713 that are determined to not be safe. */
4716 ipa_tm_diagnose_transaction (struct cgraph_node
*node
,
4717 struct tm_region
*all_tm_regions
)
4719 struct tm_region
*r
;
4721 for (r
= all_tm_regions
; r
; r
= r
->next
)
4722 if (gimple_transaction_subcode (r
->get_transaction_stmt ())
4725 /* Atomic transactions can be nested inside relaxed. */
4727 ipa_tm_diagnose_transaction (node
, r
->inner
);
4731 vec
<basic_block
> bbs
;
4732 gimple_stmt_iterator gsi
;
4736 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
,
4737 r
->irr_blocks
, NULL
, false);
4739 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
4740 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4742 gimple stmt
= gsi_stmt (gsi
);
4745 if (gimple_code (stmt
) == GIMPLE_ASM
)
4747 error_at (gimple_location (stmt
),
4748 "asm not allowed in atomic transaction");
4752 if (!is_gimple_call (stmt
))
4754 fndecl
= gimple_call_fndecl (stmt
);
4756 /* Indirect function calls have been diagnosed already. */
4760 /* Stop at the end of the transaction. */
4761 if (is_tm_ending_fndecl (fndecl
))
4763 if (bitmap_bit_p (r
->exit_blocks
, bb
->index
))
4768 /* Marked functions have been diagnosed already. */
4769 if (is_tm_pure_call (stmt
))
4771 if (is_tm_callable (fndecl
))
4774 if (cgraph_node::local_info (fndecl
)->tm_may_enter_irr
)
4775 error_at (gimple_location (stmt
),
4776 "unsafe function call %qD within "
4777 "atomic transaction", fndecl
);
4784 /* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in
4785 OLD_DECL. The returned value is a freshly malloced pointer that
4786 should be freed by the caller. */
4789 tm_mangle (tree old_asm_id
)
4791 const char *old_asm_name
;
4794 struct demangle_component
*dc
;
4797 /* Determine if the symbol is already a valid C++ mangled name. Do this
4798 even for C, which might be interfacing with C++ code via appropriately
4799 ugly identifiers. */
4800 /* ??? We could probably do just as well checking for "_Z" and be done. */
4801 old_asm_name
= IDENTIFIER_POINTER (old_asm_id
);
4802 dc
= cplus_demangle_v3_components (old_asm_name
, DMGL_NO_OPTS
, &alloc
);
4809 sprintf (length
, "%u", IDENTIFIER_LENGTH (old_asm_id
));
4810 tm_name
= concat ("_ZGTt", length
, old_asm_name
, NULL
);
4814 old_asm_name
+= 2; /* Skip _Z */
4818 case DEMANGLE_COMPONENT_TRANSACTION_CLONE
:
4819 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE
:
4820 /* Don't play silly games, you! */
4823 case DEMANGLE_COMPONENT_HIDDEN_ALIAS
:
4824 /* I'd really like to know if we can ever be passed one of
4825 these from the C++ front end. The Logical Thing would
4826 seem that hidden-alias should be outer-most, so that we
4827 get hidden-alias of a transaction-clone and not vice-versa. */
4835 tm_name
= concat ("_ZGTt", old_asm_name
, NULL
);
4839 new_asm_id
= get_identifier (tm_name
);
4846 ipa_tm_mark_force_output_node (struct cgraph_node
*node
)
4848 node
->mark_force_output ();
4849 node
->analyzed
= true;
4853 ipa_tm_mark_forced_by_abi_node (struct cgraph_node
*node
)
4855 node
->forced_by_abi
= true;
4856 node
->analyzed
= true;
4859 /* Callback data for ipa_tm_create_version_alias. */
4860 struct create_version_alias_info
4862 struct cgraph_node
*old_node
;
4866 /* A subroutine of ipa_tm_create_version, called via
4867 cgraph_for_node_and_aliases. Create new tm clones for each of
4868 the existing aliases. */
4870 ipa_tm_create_version_alias (struct cgraph_node
*node
, void *data
)
4872 struct create_version_alias_info
*info
4873 = (struct create_version_alias_info
*)data
;
4874 tree old_decl
, new_decl
, tm_name
;
4875 struct cgraph_node
*new_node
;
4877 if (!node
->cpp_implicit_alias
)
4880 old_decl
= node
->decl
;
4881 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4882 new_decl
= build_decl (DECL_SOURCE_LOCATION (old_decl
),
4883 TREE_CODE (old_decl
), tm_name
,
4884 TREE_TYPE (old_decl
));
4886 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4887 SET_DECL_RTL (new_decl
, NULL
);
4889 /* Based loosely on C++'s make_alias_for(). */
4890 TREE_PUBLIC (new_decl
) = TREE_PUBLIC (old_decl
);
4891 DECL_CONTEXT (new_decl
) = DECL_CONTEXT (old_decl
);
4892 DECL_LANG_SPECIFIC (new_decl
) = DECL_LANG_SPECIFIC (old_decl
);
4893 TREE_READONLY (new_decl
) = TREE_READONLY (old_decl
);
4894 DECL_EXTERNAL (new_decl
) = 0;
4895 DECL_ARTIFICIAL (new_decl
) = 1;
4896 TREE_ADDRESSABLE (new_decl
) = 1;
4897 TREE_USED (new_decl
) = 1;
4898 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4900 /* Perform the same remapping to the comdat group. */
4901 if (DECL_ONE_ONLY (new_decl
))
4902 varpool_node::get (new_decl
)->set_comdat_group
4903 (tm_mangle (decl_comdat_group_id (old_decl
)));
4905 new_node
= cgraph_node::create_same_body_alias (new_decl
, info
->new_decl
);
4906 new_node
->tm_clone
= true;
4907 new_node
->externally_visible
= info
->old_node
->externally_visible
;
4908 new_node
->no_reorder
= info
->old_node
->no_reorder
;
4909 /* ?? Do not traverse aliases here. */
4910 get_cg_data (&node
, false)->clone
= new_node
;
4912 record_tm_clone_pair (old_decl
, new_decl
);
4914 if (info
->old_node
->force_output
4915 || info
->old_node
->ref_list
.first_referring ())
4916 ipa_tm_mark_force_output_node (new_node
);
4917 if (info
->old_node
->forced_by_abi
)
4918 ipa_tm_mark_forced_by_abi_node (new_node
);
4922 /* Create a copy of the function (possibly declaration only) of OLD_NODE,
4923 appropriate for the transactional clone. */
4926 ipa_tm_create_version (struct cgraph_node
*old_node
)
4928 tree new_decl
, old_decl
, tm_name
;
4929 struct cgraph_node
*new_node
;
4931 old_decl
= old_node
->decl
;
4932 new_decl
= copy_node (old_decl
);
4934 /* DECL_ASSEMBLER_NAME needs to be set before we call
4935 cgraph_copy_node_for_versioning below, because cgraph_node will
4936 fill the assembler_name_hash. */
4937 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4938 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4939 SET_DECL_RTL (new_decl
, NULL
);
4940 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4942 /* Perform the same remapping to the comdat group. */
4943 if (DECL_ONE_ONLY (new_decl
))
4944 varpool_node::get (new_decl
)->set_comdat_group
4945 (tm_mangle (DECL_COMDAT_GROUP (old_decl
)));
4947 gcc_assert (!old_node
->ipa_transforms_to_apply
.exists ());
4948 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
4949 new_node
->local
.local
= false;
4950 new_node
->externally_visible
= old_node
->externally_visible
;
4951 new_node
->lowered
= true;
4952 new_node
->tm_clone
= 1;
4953 if (!old_node
->implicit_section
)
4954 new_node
->set_section (old_node
->get_section ());
4955 get_cg_data (&old_node
, true)->clone
= new_node
;
4957 if (old_node
->get_availability () >= AVAIL_INTERPOSABLE
)
4959 /* Remap extern inline to static inline. */
4960 /* ??? Is it worth trying to use make_decl_one_only? */
4961 if (DECL_DECLARED_INLINE_P (new_decl
) && DECL_EXTERNAL (new_decl
))
4963 DECL_EXTERNAL (new_decl
) = 0;
4964 TREE_PUBLIC (new_decl
) = 0;
4965 DECL_WEAK (new_decl
) = 0;
4968 tree_function_versioning (old_decl
, new_decl
,
4973 record_tm_clone_pair (old_decl
, new_decl
);
4975 symtab
->call_cgraph_insertion_hooks (new_node
);
4976 if (old_node
->force_output
4977 || old_node
->ref_list
.first_referring ())
4978 ipa_tm_mark_force_output_node (new_node
);
4979 if (old_node
->forced_by_abi
)
4980 ipa_tm_mark_forced_by_abi_node (new_node
);
4982 /* Do the same thing, but for any aliases of the original node. */
4984 struct create_version_alias_info data
;
4985 data
.old_node
= old_node
;
4986 data
.new_decl
= new_decl
;
4987 old_node
->call_for_symbol_thunks_and_aliases (ipa_tm_create_version_alias
,
4992 /* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */
4995 ipa_tm_insert_irr_call (struct cgraph_node
*node
, struct tm_region
*region
,
4998 gimple_stmt_iterator gsi
;
5001 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
5003 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
),
5004 1, build_int_cst (NULL_TREE
, MODE_SERIALIRREVOCABLE
));
5006 split_block_after_labels (bb
);
5007 gsi
= gsi_after_labels (bb
);
5008 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
5010 node
->create_edge (cgraph_node::get_create
5011 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
)),
5013 compute_call_stmt_bb_frequency (node
->decl
,
5017 /* Construct a call to TM_GETTMCLONE and insert it before GSI. */
5020 ipa_tm_insert_gettmclone_call (struct cgraph_node
*node
,
5021 struct tm_region
*region
,
5022 gimple_stmt_iterator
*gsi
, gcall
*stmt
)
5024 tree gettm_fn
, ret
, old_fn
, callfn
;
5029 old_fn
= gimple_call_fn (stmt
);
5031 if (TREE_CODE (old_fn
) == ADDR_EXPR
)
5033 tree fndecl
= TREE_OPERAND (old_fn
, 0);
5034 tree clone
= get_tm_clone_pair (fndecl
);
5036 /* By transforming the call into a TM_GETTMCLONE, we are
5037 technically taking the address of the original function and
5038 its clone. Explain this so inlining will know this function
5040 cgraph_node::get (fndecl
)->mark_address_taken () ;
5042 cgraph_node::get (clone
)->mark_address_taken ();
5045 safe
= is_tm_safe (TREE_TYPE (old_fn
));
5046 gettm_fn
= builtin_decl_explicit (safe
? BUILT_IN_TM_GETTMCLONE_SAFE
5047 : BUILT_IN_TM_GETTMCLONE_IRR
);
5048 ret
= create_tmp_var (ptr_type_node
);
5051 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
5053 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */
5054 if (TREE_CODE (old_fn
) == OBJ_TYPE_REF
)
5055 old_fn
= OBJ_TYPE_REF_EXPR (old_fn
);
5057 g
= gimple_build_call (gettm_fn
, 1, old_fn
);
5058 ret
= make_ssa_name (ret
, g
);
5059 gimple_call_set_lhs (g
, ret
);
5061 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
5063 node
->create_edge (cgraph_node::get_create (gettm_fn
), g
, 0,
5064 compute_call_stmt_bb_frequency (node
->decl
,
5067 /* Cast return value from tm_gettmclone* into appropriate function
5069 callfn
= create_tmp_var (TREE_TYPE (old_fn
));
5070 g2
= gimple_build_assign (callfn
,
5071 fold_build1 (NOP_EXPR
, TREE_TYPE (callfn
), ret
));
5072 callfn
= make_ssa_name (callfn
, g2
);
5073 gimple_assign_set_lhs (g2
, callfn
);
5074 gsi_insert_before (gsi
, g2
, GSI_SAME_STMT
);
5076 /* ??? This is a hack to preserve the NOTHROW bit on the call,
5077 which we would have derived from the decl. Failure to save
5078 this bit means we might have to split the basic block. */
5079 if (gimple_call_nothrow_p (stmt
))
5080 gimple_call_set_nothrow (stmt
, true);
5082 gimple_call_set_fn (stmt
, callfn
);
5084 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS
5085 for a call statement. Fix it. */
5087 tree lhs
= gimple_call_lhs (stmt
);
5088 tree rettype
= TREE_TYPE (gimple_call_fntype (stmt
));
5090 && !useless_type_conversion_p (TREE_TYPE (lhs
), rettype
))
5094 temp
= create_tmp_reg (rettype
);
5095 gimple_call_set_lhs (stmt
, temp
);
5097 g2
= gimple_build_assign (lhs
,
5098 fold_build1 (VIEW_CONVERT_EXPR
,
5099 TREE_TYPE (lhs
), temp
));
5100 gsi_insert_after (gsi
, g2
, GSI_SAME_STMT
);
5105 cgraph_edge
*e
= cgraph_node::get (current_function_decl
)->get_edge (stmt
);
5106 if (e
&& e
->indirect_info
)
5107 e
->indirect_info
->polymorphic
= false;
5112 /* Helper function for ipa_tm_transform_calls*. Given a call
5113 statement in GSI which resides inside transaction REGION, redirect
5114 the call to either its wrapper function, or its clone. */
5117 ipa_tm_transform_calls_redirect (struct cgraph_node
*node
,
5118 struct tm_region
*region
,
5119 gimple_stmt_iterator
*gsi
,
5120 bool *need_ssa_rename_p
)
5122 gcall
*stmt
= as_a
<gcall
*> (gsi_stmt (*gsi
));
5123 struct cgraph_node
*new_node
;
5124 struct cgraph_edge
*e
= node
->get_edge (stmt
);
5125 tree fndecl
= gimple_call_fndecl (stmt
);
5127 /* For indirect calls, pass the address through the runtime. */
5130 *need_ssa_rename_p
|=
5131 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5135 /* Handle some TM builtins. Ordinarily these aren't actually generated
5136 at this point, but handling these functions when written in by the
5137 user makes it easier to build unit tests. */
5138 if (flags_from_decl_or_type (fndecl
) & ECF_TM_BUILTIN
)
5141 /* Fixup recursive calls inside clones. */
5142 /* ??? Why did cgraph_copy_node_for_versioning update the call edges
5143 for recursion but not update the call statements themselves? */
5144 if (e
->caller
== e
->callee
&& decl_is_tm_clone (current_function_decl
))
5146 gimple_call_set_fndecl (stmt
, current_function_decl
);
5150 /* If there is a replacement, use it. */
5151 fndecl
= find_tm_replacement_function (fndecl
);
5154 new_node
= cgraph_node::get_create (fndecl
);
5156 /* ??? Mark all transaction_wrap functions tm_may_enter_irr.
5158 We can't do this earlier in record_tm_replacement because
5159 cgraph_remove_unreachable_nodes is called before we inject
5160 references to the node. Further, we can't do this in some
5161 nice central place in ipa_tm_execute because we don't have
5162 the exact list of wrapper functions that would be used.
5163 Marking more wrappers than necessary results in the creation
5164 of unnecessary cgraph_nodes, which can cause some of the
5165 other IPA passes to crash.
5167 We do need to mark these nodes so that we get the proper
5168 result in expand_call_tm. */
5169 /* ??? This seems broken. How is it that we're marking the
5170 CALLEE as may_enter_irr? Surely we should be marking the
5171 CALLER. Also note that find_tm_replacement_function also
5172 contains mappings into the TM runtime, e.g. memcpy. These
5173 we know won't go irrevocable. */
5174 new_node
->local
.tm_may_enter_irr
= 1;
5178 struct tm_ipa_cg_data
*d
;
5179 struct cgraph_node
*tnode
= e
->callee
;
5181 d
= get_cg_data (&tnode
, true);
5182 new_node
= d
->clone
;
5184 /* As we've already skipped pure calls and appropriate builtins,
5185 and we've already marked irrevocable blocks, if we can't come
5186 up with a static replacement, then ask the runtime. */
5187 if (new_node
== NULL
)
5189 *need_ssa_rename_p
|=
5190 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5194 fndecl
= new_node
->decl
;
5197 e
->redirect_callee (new_node
);
5198 gimple_call_set_fndecl (stmt
, fndecl
);
5201 /* Helper function for ipa_tm_transform_calls. For a given BB,
5202 install calls to tm_irrevocable when IRR_BLOCKS are reached,
5203 redirect other calls to the generated transactional clone. */
5206 ipa_tm_transform_calls_1 (struct cgraph_node
*node
, struct tm_region
*region
,
5207 basic_block bb
, bitmap irr_blocks
)
5209 gimple_stmt_iterator gsi
;
5210 bool need_ssa_rename
= false;
5212 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5214 ipa_tm_insert_irr_call (node
, region
, bb
);
5218 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5220 gimple stmt
= gsi_stmt (gsi
);
5222 if (!is_gimple_call (stmt
))
5224 if (is_tm_pure_call (stmt
))
5227 /* Redirect edges to the appropriate replacement or clone. */
5228 ipa_tm_transform_calls_redirect (node
, region
, &gsi
, &need_ssa_rename
);
5231 return need_ssa_rename
;
5234 /* Walk the CFG for REGION, beginning at BB. Install calls to
5235 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to
5236 the generated transactional clone. */
5239 ipa_tm_transform_calls (struct cgraph_node
*node
, struct tm_region
*region
,
5240 basic_block bb
, bitmap irr_blocks
)
5242 bool need_ssa_rename
= false;
5245 auto_vec
<basic_block
> queue
;
5246 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
5248 queue
.safe_push (bb
);
5254 ipa_tm_transform_calls_1 (node
, region
, bb
, irr_blocks
);
5256 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5259 if (region
&& bitmap_bit_p (region
->exit_blocks
, bb
->index
))
5262 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
5263 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
5265 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
5266 queue
.safe_push (e
->dest
);
5269 while (!queue
.is_empty ());
5271 BITMAP_FREE (visited_blocks
);
5273 return need_ssa_rename
;
5276 /* Transform the calls within the TM regions within NODE. */
5279 ipa_tm_transform_transaction (struct cgraph_node
*node
)
5281 struct tm_ipa_cg_data
*d
;
5282 struct tm_region
*region
;
5283 bool need_ssa_rename
= false;
5285 d
= get_cg_data (&node
, true);
5287 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5288 calculate_dominance_info (CDI_DOMINATORS
);
5290 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
5292 /* If we're sure to go irrevocable, don't transform anything. */
5293 if (d
->irrevocable_blocks_normal
5294 && bitmap_bit_p (d
->irrevocable_blocks_normal
,
5295 region
->entry_block
->index
))
5297 transaction_subcode_ior (region
, GTMA_DOES_GO_IRREVOCABLE
5298 | GTMA_MAY_ENTER_IRREVOCABLE
5299 | GTMA_HAS_NO_INSTRUMENTATION
);
5304 ipa_tm_transform_calls (node
, region
, region
->entry_block
,
5305 d
->irrevocable_blocks_normal
);
5308 if (need_ssa_rename
)
5309 update_ssa (TODO_update_ssa_only_virtuals
);
5314 /* Transform the calls within the transactional clone of NODE. */
5317 ipa_tm_transform_clone (struct cgraph_node
*node
)
5319 struct tm_ipa_cg_data
*d
;
5320 bool need_ssa_rename
;
5322 d
= get_cg_data (&node
, true);
5324 /* If this function makes no calls and has no irrevocable blocks,
5325 then there's nothing to do. */
5326 /* ??? Remove non-aborting top-level transactions. */
5327 if (!node
->callees
&& !node
->indirect_calls
&& !d
->irrevocable_blocks_clone
)
5330 push_cfun (DECL_STRUCT_FUNCTION (d
->clone
->decl
));
5331 calculate_dominance_info (CDI_DOMINATORS
);
5334 ipa_tm_transform_calls (d
->clone
, NULL
,
5335 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
5336 d
->irrevocable_blocks_clone
);
5338 if (need_ssa_rename
)
5339 update_ssa (TODO_update_ssa_only_virtuals
);
5344 /* Main entry point for the transactional memory IPA pass. */
5347 ipa_tm_execute (void)
5349 cgraph_node_queue tm_callees
= cgraph_node_queue ();
5350 /* List of functions that will go irrevocable. */
5351 cgraph_node_queue irr_worklist
= cgraph_node_queue ();
5353 struct cgraph_node
*node
;
5354 struct tm_ipa_cg_data
*d
;
5355 enum availability a
;
5358 #ifdef ENABLE_CHECKING
5359 cgraph_node::verify_cgraph_nodes ();
5362 bitmap_obstack_initialize (&tm_obstack
);
5363 initialize_original_copy_tables ();
5365 /* For all local functions marked tm_callable, queue them. */
5366 FOR_EACH_DEFINED_FUNCTION (node
)
5367 if (is_tm_callable (node
->decl
)
5368 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5370 d
= get_cg_data (&node
, true);
5371 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5374 /* For all local reachable functions... */
5375 FOR_EACH_DEFINED_FUNCTION (node
)
5377 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5379 /* ... marked tm_pure, record that fact for the runtime by
5380 indicating that the pure function is its own tm_callable.
5381 No need to do this if the function's address can't be taken. */
5382 if (is_tm_pure (node
->decl
))
5384 if (!node
->local
.local
)
5385 record_tm_clone_pair (node
->decl
, node
->decl
);
5389 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5390 calculate_dominance_info (CDI_DOMINATORS
);
5392 tm_region_init (NULL
);
5395 d
= get_cg_data (&node
, true);
5397 /* Scan for calls that are in each transaction, and
5398 generate the uninstrumented code path. */
5399 ipa_tm_scan_calls_transaction (d
, &tm_callees
);
5401 /* Put it in the worklist so we can scan the function
5402 later (ipa_tm_scan_irr_function) and mark the
5403 irrevocable blocks. */
5404 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5405 d
->want_irr_scan_normal
= true;
5411 /* For every local function on the callee list, scan as if we will be
5412 creating a transactional clone, queueing all new functions we find
5414 for (i
= 0; i
< tm_callees
.length (); ++i
)
5416 node
= tm_callees
[i
];
5417 a
= node
->get_availability ();
5418 d
= get_cg_data (&node
, true);
5420 /* Put it in the worklist so we can scan the function later
5421 (ipa_tm_scan_irr_function) and mark the irrevocable
5423 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5425 /* Some callees cannot be arbitrarily cloned. These will always be
5426 irrevocable. Mark these now, so that we need not scan them. */
5427 if (is_tm_irrevocable (node
->decl
))
5428 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5429 else if (a
<= AVAIL_NOT_AVAILABLE
5430 && !is_tm_safe_or_pure (node
->decl
))
5431 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5432 else if (a
>= AVAIL_INTERPOSABLE
)
5434 if (!tree_versionable_function_p (node
->decl
))
5435 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5436 else if (!d
->is_irrevocable
)
5438 /* If this is an alias, make sure its base is queued as well.
5439 we need not scan the callees now, as the base will do. */
5442 node
= cgraph_node::get (node
->thunk
.alias
);
5443 d
= get_cg_data (&node
, true);
5444 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5448 /* Add all nodes called by this function into
5449 tm_callees as well. */
5450 ipa_tm_scan_calls_clone (node
, &tm_callees
);
5455 /* Iterate scans until no more work to be done. Prefer not to use
5456 vec::pop because the worklist tends to follow a breadth-first
5457 search of the callgraph, which should allow convergance with a
5458 minimum number of scans. But we also don't want the worklist
5459 array to grow without bound, so we shift the array up periodically. */
5460 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5462 if (i
> 256 && i
== irr_worklist
.length () / 8)
5464 irr_worklist
.block_remove (0, i
);
5468 node
= irr_worklist
[i
];
5469 d
= get_cg_data (&node
, true);
5470 d
->in_worklist
= false;
5472 if (d
->want_irr_scan_normal
)
5474 d
->want_irr_scan_normal
= false;
5475 ipa_tm_scan_irr_function (node
, false);
5477 if (d
->in_callee_queue
&& ipa_tm_scan_irr_function (node
, true))
5478 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5481 /* For every function on the callee list, collect the tm_may_enter_irr
5483 irr_worklist
.truncate (0);
5484 for (i
= 0; i
< tm_callees
.length (); ++i
)
5486 node
= tm_callees
[i
];
5487 if (ipa_tm_mayenterirr_function (node
))
5489 d
= get_cg_data (&node
, true);
5490 gcc_assert (d
->in_worklist
== false);
5491 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5495 /* Propagate the tm_may_enter_irr bit to callers until stable. */
5496 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5498 struct cgraph_node
*caller
;
5499 struct cgraph_edge
*e
;
5500 struct ipa_ref
*ref
;
5502 if (i
> 256 && i
== irr_worklist
.length () / 8)
5504 irr_worklist
.block_remove (0, i
);
5508 node
= irr_worklist
[i
];
5509 d
= get_cg_data (&node
, true);
5510 d
->in_worklist
= false;
5511 node
->local
.tm_may_enter_irr
= true;
5513 /* Propagate back to normal callers. */
5514 for (e
= node
->callers
; e
; e
= e
->next_caller
)
5517 if (!is_tm_safe_or_pure (caller
->decl
)
5518 && !caller
->local
.tm_may_enter_irr
)
5520 d
= get_cg_data (&caller
, true);
5521 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5525 /* Propagate back to referring aliases as well. */
5526 FOR_EACH_ALIAS (node
, ref
)
5528 caller
= dyn_cast
<cgraph_node
*> (ref
->referring
);
5529 if (!caller
->local
.tm_may_enter_irr
)
5531 /* ?? Do not traverse aliases here. */
5532 d
= get_cg_data (&caller
, false);
5533 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5538 /* Now validate all tm_safe functions, and all atomic regions in
5540 FOR_EACH_DEFINED_FUNCTION (node
)
5542 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5544 d
= get_cg_data (&node
, true);
5545 if (is_tm_safe (node
->decl
))
5546 ipa_tm_diagnose_tm_safe (node
);
5547 else if (d
->all_tm_regions
)
5548 ipa_tm_diagnose_transaction (node
, d
->all_tm_regions
);
5551 /* Create clones. Do those that are not irrevocable and have a
5552 positive call count. Do those publicly visible functions that
5553 the user directed us to clone. */
5554 for (i
= 0; i
< tm_callees
.length (); ++i
)
5558 node
= tm_callees
[i
];
5559 if (node
->cpp_implicit_alias
)
5562 a
= node
->get_availability ();
5563 d
= get_cg_data (&node
, true);
5565 if (a
<= AVAIL_NOT_AVAILABLE
)
5566 doit
= is_tm_callable (node
->decl
);
5567 else if (a
<= AVAIL_AVAILABLE
&& is_tm_callable (node
->decl
))
5569 else if (!d
->is_irrevocable
5570 && d
->tm_callers_normal
+ d
->tm_callers_clone
> 0)
5574 ipa_tm_create_version (node
);
5577 /* Redirect calls to the new clones, and insert irrevocable marks. */
5578 for (i
= 0; i
< tm_callees
.length (); ++i
)
5580 node
= tm_callees
[i
];
5583 d
= get_cg_data (&node
, true);
5585 ipa_tm_transform_clone (node
);
5588 FOR_EACH_DEFINED_FUNCTION (node
)
5590 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5592 d
= get_cg_data (&node
, true);
5593 if (d
->all_tm_regions
)
5594 ipa_tm_transform_transaction (node
);
5597 /* Free and clear all data structures. */
5598 tm_callees
.release ();
5599 irr_worklist
.release ();
5600 bitmap_obstack_release (&tm_obstack
);
5601 free_original_copy_tables ();
5603 FOR_EACH_FUNCTION (node
)
5606 #ifdef ENABLE_CHECKING
5607 cgraph_node::verify_cgraph_nodes ();
5615 const pass_data pass_data_ipa_tm
=
5617 SIMPLE_IPA_PASS
, /* type */
5619 OPTGROUP_NONE
, /* optinfo_flags */
5620 TV_TRANS_MEM
, /* tv_id */
5621 ( PROP_ssa
| PROP_cfg
), /* properties_required */
5622 0, /* properties_provided */
5623 0, /* properties_destroyed */
5624 0, /* todo_flags_start */
5625 0, /* todo_flags_finish */
5628 class pass_ipa_tm
: public simple_ipa_opt_pass
5631 pass_ipa_tm (gcc::context
*ctxt
)
5632 : simple_ipa_opt_pass (pass_data_ipa_tm
, ctxt
)
5635 /* opt_pass methods: */
5636 virtual bool gate (function
*) { return flag_tm
; }
5637 virtual unsigned int execute (function
*) { return ipa_tm_execute (); }
5639 }; // class pass_ipa_tm
5643 simple_ipa_opt_pass
*
5644 make_pass_ipa_tm (gcc::context
*ctxt
)
5646 return new pass_ipa_tm (ctxt
);
5649 #include "gt-trans-mem.h"