1 /* Passes for transactional memory support.
2 Copyright (C) 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
25 #include "tree-flow.h"
26 #include "tree-pass.h"
27 #include "tree-inline.h"
28 #include "diagnostic-core.h"
31 #include "trans-mem.h"
34 #include "langhooks.h"
35 #include "tree-pretty-print.h"
36 #include "gimple-pretty-print.h"
39 #define PROB_VERY_UNLIKELY (REG_BR_PROB_BASE / 2000 - 1)
40 #define PROB_ALWAYS (REG_BR_PROB_BASE)
42 #define A_RUNINSTRUMENTEDCODE 0x0001
43 #define A_RUNUNINSTRUMENTEDCODE 0x0002
44 #define A_SAVELIVEVARIABLES 0x0004
45 #define A_RESTORELIVEVARIABLES 0x0008
46 #define A_ABORTTRANSACTION 0x0010
48 #define AR_USERABORT 0x0001
49 #define AR_USERRETRY 0x0002
50 #define AR_TMCONFLICT 0x0004
51 #define AR_EXCEPTIONBLOCKABORT 0x0008
52 #define AR_OUTERABORT 0x0010
54 #define MODE_SERIALIRREVOCABLE 0x0000
57 /* The representation of a transaction changes several times during the
58 lowering process. In the beginning, in the front-end we have the
59 GENERIC tree TRANSACTION_EXPR. For example,
67 During initial gimplification (gimplify.c) the TRANSACTION_EXPR node is
68 trivially replaced with a GIMPLE_TRANSACTION node.
70 During pass_lower_tm, we examine the body of transactions looking
71 for aborts. Transactions that do not contain an abort may be
72 merged into an outer transaction. We also add a TRY-FINALLY node
73 to arrange for the transaction to be committed on any exit.
75 [??? Think about how this arrangement affects throw-with-commit
76 and throw-with-abort operations. In this case we want the TRY to
77 handle gotos, but not to catch any exceptions because the transaction
78 will already be closed.]
80 GIMPLE_TRANSACTION [label=NULL] {
87 __builtin___tm_abort ();
89 __builtin___tm_commit ();
93 During pass_lower_eh, we create EH regions for the transactions,
94 intermixed with the regular EH stuff. This gives us a nice persistent
95 mapping (all the way through rtl) from transactional memory operation
96 back to the transaction, which allows us to get the abnormal edges
97 correct to model transaction aborts and restarts:
99 GIMPLE_TRANSACTION [label=over]
105 __builtin___tm_abort ();
106 __builtin___tm_commit ();
109 This is the end of all_lowering_passes, and so is what is present
110 during the IPA passes, and through all of the optimization passes.
112 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all
113 functions and mark functions for cloning.
115 At the end of gimple optimization, before exiting SSA form,
116 pass_tm_edges replaces statements that perform transactional
117 memory operations with the appropriate TM builtins, and swap
118 out function calls with their transactional clones. At this
119 point we introduce the abnormal transaction restart edges and
120 complete lowering of the GIMPLE_TRANSACTION node.
122 x = __builtin___tm_start (MAY_ABORT);
124 if (x & abort_transaction)
127 t0 = __builtin___tm_load (global);
129 __builtin___tm_store (&global, t1);
131 __builtin___tm_abort ();
132 __builtin___tm_commit ();
137 /* Return the attributes we want to examine for X, or NULL if it's not
138 something we examine. We look at function types, but allow pointers
139 to function types and function decls and peek through. */
142 get_attrs_for (const_tree x
)
144 switch (TREE_CODE (x
))
147 return TYPE_ATTRIBUTES (TREE_TYPE (x
));
154 if (TREE_CODE (x
) != POINTER_TYPE
)
160 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
166 return TYPE_ATTRIBUTES (x
);
170 /* Return true if X has been marked TM_PURE. */
173 is_tm_pure (const_tree x
)
177 switch (TREE_CODE (x
))
188 if (TREE_CODE (x
) != POINTER_TYPE
)
194 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
199 flags
= flags_from_decl_or_type (x
);
200 return (flags
& ECF_TM_PURE
) != 0;
203 /* Return true if X has been marked TM_IRREVOCABLE. */
206 is_tm_irrevocable (tree x
)
208 tree attrs
= get_attrs_for (x
);
210 if (attrs
&& lookup_attribute ("transaction_unsafe", attrs
))
213 /* A call to the irrevocable builtin is by definition,
215 if (TREE_CODE (x
) == ADDR_EXPR
)
216 x
= TREE_OPERAND (x
, 0);
217 if (TREE_CODE (x
) == FUNCTION_DECL
218 && DECL_BUILT_IN_CLASS (x
) == BUILT_IN_NORMAL
219 && DECL_FUNCTION_CODE (x
) == BUILT_IN_TM_IRREVOCABLE
)
225 /* Return true if X has been marked TM_SAFE. */
228 is_tm_safe (const_tree x
)
232 tree attrs
= get_attrs_for (x
);
235 if (lookup_attribute ("transaction_safe", attrs
))
237 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
244 /* Return true if CALL is const, or tm_pure. */
247 is_tm_pure_call (gimple call
)
249 tree fn
= gimple_call_fn (call
);
251 if (TREE_CODE (fn
) == ADDR_EXPR
)
253 fn
= TREE_OPERAND (fn
, 0);
254 gcc_assert (TREE_CODE (fn
) == FUNCTION_DECL
);
259 return is_tm_pure (fn
);
262 /* Return true if X has been marked TM_CALLABLE. */
265 is_tm_callable (tree x
)
267 tree attrs
= get_attrs_for (x
);
270 if (lookup_attribute ("transaction_callable", attrs
))
272 if (lookup_attribute ("transaction_safe", attrs
))
274 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
280 /* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */
283 is_tm_may_cancel_outer (tree x
)
285 tree attrs
= get_attrs_for (x
);
287 return lookup_attribute ("transaction_may_cancel_outer", attrs
) != NULL
;
291 /* Return true for built in functions that "end" a transaction. */
294 is_tm_ending_fndecl (tree fndecl
)
296 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
297 switch (DECL_FUNCTION_CODE (fndecl
))
299 case BUILT_IN_TM_COMMIT
:
300 case BUILT_IN_TM_COMMIT_EH
:
301 case BUILT_IN_TM_ABORT
:
302 case BUILT_IN_TM_IRREVOCABLE
:
311 /* Return true if STMT is a TM load. */
314 is_tm_load (gimple stmt
)
318 if (gimple_code (stmt
) != GIMPLE_CALL
)
321 fndecl
= gimple_call_fndecl (stmt
);
322 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
323 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl
)));
326 /* Same as above, but for simple TM loads, that is, not the
327 after-write, after-read, etc optimized variants. */
330 is_tm_simple_load (gimple stmt
)
334 if (gimple_code (stmt
) != GIMPLE_CALL
)
337 fndecl
= gimple_call_fndecl (stmt
);
338 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
340 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
341 return (fcode
== BUILT_IN_TM_LOAD_1
342 || fcode
== BUILT_IN_TM_LOAD_2
343 || fcode
== BUILT_IN_TM_LOAD_4
344 || fcode
== BUILT_IN_TM_LOAD_8
345 || fcode
== BUILT_IN_TM_LOAD_FLOAT
346 || fcode
== BUILT_IN_TM_LOAD_DOUBLE
347 || fcode
== BUILT_IN_TM_LOAD_LDOUBLE
348 || fcode
== BUILT_IN_TM_LOAD_M64
349 || fcode
== BUILT_IN_TM_LOAD_M128
350 || fcode
== BUILT_IN_TM_LOAD_M256
);
355 /* Return true if STMT is a TM store. */
358 is_tm_store (gimple stmt
)
362 if (gimple_code (stmt
) != GIMPLE_CALL
)
365 fndecl
= gimple_call_fndecl (stmt
);
366 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
367 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl
)));
370 /* Same as above, but for simple TM stores, that is, not the
371 after-write, after-read, etc optimized variants. */
374 is_tm_simple_store (gimple stmt
)
378 if (gimple_code (stmt
) != GIMPLE_CALL
)
381 fndecl
= gimple_call_fndecl (stmt
);
382 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
384 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
385 return (fcode
== BUILT_IN_TM_STORE_1
386 || fcode
== BUILT_IN_TM_STORE_2
387 || fcode
== BUILT_IN_TM_STORE_4
388 || fcode
== BUILT_IN_TM_STORE_8
389 || fcode
== BUILT_IN_TM_STORE_FLOAT
390 || fcode
== BUILT_IN_TM_STORE_DOUBLE
391 || fcode
== BUILT_IN_TM_STORE_LDOUBLE
392 || fcode
== BUILT_IN_TM_STORE_M64
393 || fcode
== BUILT_IN_TM_STORE_M128
394 || fcode
== BUILT_IN_TM_STORE_M256
);
399 /* Return true if FNDECL is BUILT_IN_TM_ABORT. */
402 is_tm_abort (tree fndecl
)
405 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
406 && DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_TM_ABORT
);
409 /* Build a GENERIC tree for a user abort. This is called by front ends
410 while transforming the __tm_abort statement. */
413 build_tm_abort_call (location_t loc
, bool is_outer
)
415 return build_call_expr_loc (loc
, builtin_decl_explicit (BUILT_IN_TM_ABORT
), 1,
416 build_int_cst (integer_type_node
,
418 | (is_outer
? AR_OUTERABORT
: 0)));
421 /* Common gateing function for several of the TM passes. */
429 /* Map for aribtrary function replacement under TM, as created
430 by the tm_wrap attribute. */
432 static GTY((if_marked ("tree_map_marked_p"), param_is (struct tree_map
)))
436 record_tm_replacement (tree from
, tree to
)
438 struct tree_map
**slot
, *h
;
440 /* Do not inline wrapper functions that will get replaced in the TM
443 Suppose you have foo() that will get replaced into tmfoo(). Make
444 sure the inliner doesn't try to outsmart us and inline foo()
445 before we get a chance to do the TM replacement. */
446 DECL_UNINLINABLE (from
) = 1;
448 if (tm_wrap_map
== NULL
)
449 tm_wrap_map
= htab_create_ggc (32, tree_map_hash
, tree_map_eq
, 0);
451 h
= ggc_alloc_tree_map ();
452 h
->hash
= htab_hash_pointer (from
);
456 slot
= (struct tree_map
**)
457 htab_find_slot_with_hash (tm_wrap_map
, h
, h
->hash
, INSERT
);
461 /* Return a TM-aware replacement function for DECL. */
464 find_tm_replacement_function (tree fndecl
)
468 struct tree_map
*h
, in
;
470 in
.base
.from
= fndecl
;
471 in
.hash
= htab_hash_pointer (fndecl
);
472 h
= (struct tree_map
*) htab_find_with_hash (tm_wrap_map
, &in
, in
.hash
);
477 /* ??? We may well want TM versions of most of the common <string.h>
478 functions. For now, we've already these two defined. */
479 /* Adjust expand_call_tm() attributes as necessary for the cases
481 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
482 switch (DECL_FUNCTION_CODE (fndecl
))
484 case BUILT_IN_MEMCPY
:
485 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY
);
486 case BUILT_IN_MEMMOVE
:
487 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
);
488 case BUILT_IN_MEMSET
:
489 return builtin_decl_explicit (BUILT_IN_TM_MEMSET
);
497 /* When appropriate, record TM replacement for memory allocation functions.
499 FROM is the FNDECL to wrap. */
501 tm_malloc_replacement (tree from
)
506 if (TREE_CODE (from
) != FUNCTION_DECL
)
509 /* If we have a previous replacement, the user must be explicitly
510 wrapping malloc/calloc/free. They better know what they're
512 if (find_tm_replacement_function (from
))
515 str
= IDENTIFIER_POINTER (DECL_NAME (from
));
517 if (!strcmp (str
, "malloc"))
518 to
= builtin_decl_explicit (BUILT_IN_TM_MALLOC
);
519 else if (!strcmp (str
, "calloc"))
520 to
= builtin_decl_explicit (BUILT_IN_TM_CALLOC
);
521 else if (!strcmp (str
, "free"))
522 to
= builtin_decl_explicit (BUILT_IN_TM_FREE
);
526 TREE_NOTHROW (to
) = 0;
528 record_tm_replacement (from
, to
);
531 /* Diagnostics for tm_safe functions/regions. Called by the front end
532 once we've lowered the function to high-gimple. */
534 /* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq.
535 Process exactly one statement. WI->INFO is set to non-null when in
536 the context of a tm_safe function, and null for a __transaction block. */
538 #define DIAG_TM_OUTER 1
539 #define DIAG_TM_SAFE 2
540 #define DIAG_TM_RELAXED 4
544 unsigned int summary_flags
: 8;
545 unsigned int block_flags
: 8;
546 unsigned int func_flags
: 8;
547 unsigned int saw_unsafe
: 1;
548 unsigned int saw_volatile
: 1;
552 /* Tree callback function for diagnose_tm pass. */
555 diagnose_tm_1_op (tree
*tp
, int *walk_subtrees ATTRIBUTE_UNUSED
,
558 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
559 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
560 enum tree_code code
= TREE_CODE (*tp
);
562 if ((code
== VAR_DECL
563 || code
== RESULT_DECL
564 || code
== PARM_DECL
)
565 && d
->block_flags
& (DIAG_TM_SAFE
| DIAG_TM_RELAXED
)
566 && TREE_THIS_VOLATILE (TREE_TYPE (*tp
))
570 error_at (gimple_location (d
->stmt
),
571 "invalid volatile use of %qD inside transaction",
579 diagnose_tm_1 (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
580 struct walk_stmt_info
*wi
)
582 gimple stmt
= gsi_stmt (*gsi
);
583 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
585 /* Save stmt for use in leaf analysis. */
588 switch (gimple_code (stmt
))
592 tree fn
= gimple_call_fn (stmt
);
594 if ((d
->summary_flags
& DIAG_TM_OUTER
) == 0
595 && is_tm_may_cancel_outer (fn
))
596 error_at (gimple_location (stmt
),
597 "%<transaction_may_cancel_outer%> function call not within"
598 " outer transaction or %<transaction_may_cancel_outer%>");
600 if (d
->summary_flags
& DIAG_TM_SAFE
)
602 bool is_safe
, direct_call_p
;
605 if (TREE_CODE (fn
) == ADDR_EXPR
606 && TREE_CODE (TREE_OPERAND (fn
, 0)) == FUNCTION_DECL
)
608 direct_call_p
= true;
609 replacement
= TREE_OPERAND (fn
, 0);
610 replacement
= find_tm_replacement_function (replacement
);
616 direct_call_p
= false;
617 replacement
= NULL_TREE
;
620 if (is_tm_safe_or_pure (fn
))
622 else if (is_tm_callable (fn
) || is_tm_irrevocable (fn
))
624 /* A function explicitly marked transaction_callable as
625 opposed to transaction_safe is being defined to be
626 unsafe as part of its ABI, regardless of its contents. */
629 else if (direct_call_p
)
631 if (flags_from_decl_or_type (fn
) & ECF_TM_BUILTIN
)
633 else if (replacement
)
635 /* ??? At present we've been considering replacements
636 merely transaction_callable, and therefore might
637 enter irrevocable. The tm_wrap attribute has not
638 yet made it into the new language spec. */
643 /* ??? Diagnostics for unmarked direct calls moved into
644 the IPA pass. Section 3.2 of the spec details how
645 functions not marked should be considered "implicitly
646 safe" based on having examined the function body. */
652 /* An unmarked indirect call. Consider it unsafe even
653 though optimization may yet figure out how to inline. */
659 if (TREE_CODE (fn
) == ADDR_EXPR
)
660 fn
= TREE_OPERAND (fn
, 0);
661 if (d
->block_flags
& DIAG_TM_SAFE
)
662 error_at (gimple_location (stmt
),
663 "unsafe function call %qD within "
664 "atomic transaction", fn
);
666 error_at (gimple_location (stmt
),
667 "unsafe function call %qD within "
668 "%<transaction_safe%> function", fn
);
675 /* ??? We ought to come up with a way to add attributes to
676 asm statements, and then add "transaction_safe" to it.
677 Either that or get the language spec to resurrect __tm_waiver. */
678 if (d
->block_flags
& DIAG_TM_SAFE
)
679 error_at (gimple_location (stmt
),
680 "asm not allowed in atomic transaction");
681 else if (d
->func_flags
& DIAG_TM_SAFE
)
682 error_at (gimple_location (stmt
),
683 "asm not allowed in %<transaction_safe%> function");
685 d
->saw_unsafe
= true;
688 case GIMPLE_TRANSACTION
:
690 unsigned char inner_flags
= DIAG_TM_SAFE
;
692 if (gimple_transaction_subcode (stmt
) & GTMA_IS_RELAXED
)
694 if (d
->block_flags
& DIAG_TM_SAFE
)
695 error_at (gimple_location (stmt
),
696 "relaxed transaction in atomic transaction");
697 else if (d
->func_flags
& DIAG_TM_SAFE
)
698 error_at (gimple_location (stmt
),
699 "relaxed transaction in %<transaction_safe%> function");
701 d
->saw_unsafe
= true;
702 inner_flags
= DIAG_TM_RELAXED
;
704 else if (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
)
707 error_at (gimple_location (stmt
),
708 "outer transaction in transaction");
709 else if (d
->func_flags
& DIAG_TM_OUTER
)
710 error_at (gimple_location (stmt
),
711 "outer transaction in "
712 "%<transaction_may_cancel_outer%> function");
713 else if (d
->func_flags
& DIAG_TM_SAFE
)
714 error_at (gimple_location (stmt
),
715 "outer transaction in %<transaction_safe%> function");
717 d
->saw_unsafe
= true;
718 inner_flags
|= DIAG_TM_OUTER
;
721 *handled_ops_p
= true;
722 if (gimple_transaction_body (stmt
))
724 struct walk_stmt_info wi_inner
;
725 struct diagnose_tm d_inner
;
727 memset (&d_inner
, 0, sizeof (d_inner
));
728 d_inner
.func_flags
= d
->func_flags
;
729 d_inner
.block_flags
= d
->block_flags
| inner_flags
;
730 d_inner
.summary_flags
= d_inner
.func_flags
| d_inner
.block_flags
;
732 memset (&wi_inner
, 0, sizeof (wi_inner
));
733 wi_inner
.info
= &d_inner
;
735 walk_gimple_seq (gimple_transaction_body (stmt
),
736 diagnose_tm_1
, diagnose_tm_1_op
, &wi_inner
);
738 d
->saw_unsafe
|= d_inner
.saw_unsafe
;
751 diagnose_tm_blocks (void)
753 struct walk_stmt_info wi
;
754 struct diagnose_tm d
;
756 memset (&d
, 0, sizeof (d
));
757 if (is_tm_may_cancel_outer (current_function_decl
))
758 d
.func_flags
= DIAG_TM_OUTER
| DIAG_TM_SAFE
;
759 else if (is_tm_safe (current_function_decl
))
760 d
.func_flags
= DIAG_TM_SAFE
;
761 d
.summary_flags
= d
.func_flags
;
763 memset (&wi
, 0, sizeof (wi
));
766 walk_gimple_seq (gimple_body (current_function_decl
),
767 diagnose_tm_1
, diagnose_tm_1_op
, &wi
);
769 /* If we saw something other than a call that makes this function
770 unsafe, remember it so that the IPA pass only needs to scan calls. */
771 if (d
.saw_unsafe
&& !is_tm_safe_or_pure (current_function_decl
))
772 cgraph_local_info (current_function_decl
)->tm_may_enter_irr
= 1;
777 struct gimple_opt_pass pass_diagnose_tm_blocks
=
781 "*diagnose_tm_blocks", /* name */
783 diagnose_tm_blocks
, /* execute */
786 0, /* static_pass_number */
787 TV_TRANS_MEM
, /* tv_id */
788 PROP_gimple_any
, /* properties_required */
789 0, /* properties_provided */
790 0, /* properties_destroyed */
791 0, /* todo_flags_start */
792 0, /* todo_flags_finish */
796 /* Instead of instrumenting thread private memory, we save the
797 addresses in a log which we later use to save/restore the addresses
798 upon transaction start/restart.
800 The log is keyed by address, where each element contains individual
801 statements among different code paths that perform the store.
803 This log is later used to generate either plain save/restore of the
804 addresses upon transaction start/restart, or calls to the ITM_L*
807 So for something like:
809 struct large { int x[1000]; };
810 struct large lala = { 0 };
816 We can either save/restore:
819 trxn = _ITM_startTransaction ();
820 if (trxn & a_saveLiveVariables)
821 tmp_lala1 = lala.x[i];
822 else if (a & a_restoreLiveVariables)
823 lala.x[i] = tmp_lala1;
825 or use the logging functions:
828 trxn = _ITM_startTransaction ();
829 _ITM_LU4 (&lala.x[i]);
831 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as
832 far up the dominator tree to shadow all of the writes to a given
833 location (thus reducing the total number of logging calls), but not
834 so high as to be called on a path that does not perform a
837 /* One individual log entry. We may have multiple statements for the
838 same location if neither dominate each other (on different
840 typedef struct tm_log_entry
842 /* Address to save. */
844 /* Entry block for the transaction this address occurs in. */
845 basic_block entry_block
;
846 /* Dominating statements the store occurs in. */
848 /* Initially, while we are building the log, we place a nonzero
849 value here to mean that this address *will* be saved with a
850 save/restore sequence. Later, when generating the save sequence
851 we place the SSA temp generated here. */
855 /* The actual log. */
856 static htab_t tm_log
;
858 /* Addresses to log with a save/restore sequence. These should be in
860 static VEC(tree
,heap
) *tm_log_save_addresses
;
862 /* Map for an SSA_NAME originally pointing to a non aliased new piece
863 of memory (malloc, alloc, etc). */
864 static htab_t tm_new_mem_hash
;
866 enum thread_memory_type
870 mem_transaction_local
,
874 typedef struct tm_new_mem_map
876 /* SSA_NAME being dereferenced. */
878 enum thread_memory_type local_new_memory
;
881 /* Htab support. Return hash value for a `tm_log_entry'. */
883 tm_log_hash (const void *p
)
885 const struct tm_log_entry
*log
= (const struct tm_log_entry
*) p
;
886 return iterative_hash_expr (log
->addr
, 0);
889 /* Htab support. Return true if two log entries are the same. */
891 tm_log_eq (const void *p1
, const void *p2
)
893 const struct tm_log_entry
*log1
= (const struct tm_log_entry
*) p1
;
894 const struct tm_log_entry
*log2
= (const struct tm_log_entry
*) p2
;
898 rth: I suggest that we get rid of the component refs etc.
899 I.e. resolve the reference to base + offset.
901 We may need to actually finish a merge with mainline for this,
902 since we'd like to be presented with Richi's MEM_REF_EXPRs more
903 often than not. But in the meantime your tm_log_entry could save
904 the results of get_inner_reference.
906 See: g++.dg/tm/pr46653.C
909 /* Special case plain equality because operand_equal_p() below will
910 return FALSE if the addresses are equal but they have
911 side-effects (e.g. a volatile address). */
912 if (log1
->addr
== log2
->addr
)
915 return operand_equal_p (log1
->addr
, log2
->addr
, 0);
918 /* Htab support. Free one tm_log_entry. */
920 tm_log_free (void *p
)
922 struct tm_log_entry
*lp
= (struct tm_log_entry
*) p
;
923 VEC_free (gimple
, heap
, lp
->stmts
);
927 /* Initialize logging data structures. */
931 tm_log
= htab_create (10, tm_log_hash
, tm_log_eq
, tm_log_free
);
932 tm_new_mem_hash
= htab_create (5, struct_ptr_hash
, struct_ptr_eq
, free
);
933 tm_log_save_addresses
= VEC_alloc (tree
, heap
, 5);
936 /* Free logging data structures. */
940 htab_delete (tm_log
);
941 htab_delete (tm_new_mem_hash
);
942 VEC_free (tree
, heap
, tm_log_save_addresses
);
945 /* Return true if MEM is a transaction invariant memory for the TM
946 region starting at REGION_ENTRY_BLOCK. */
948 transaction_invariant_address_p (const_tree mem
, basic_block region_entry_block
)
950 if ((TREE_CODE (mem
) == INDIRECT_REF
|| TREE_CODE (mem
) == MEM_REF
)
951 && TREE_CODE (TREE_OPERAND (mem
, 0)) == SSA_NAME
)
955 def_bb
= gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem
, 0)));
956 return def_bb
!= region_entry_block
957 && dominated_by_p (CDI_DOMINATORS
, region_entry_block
, def_bb
);
960 mem
= strip_invariant_refs (mem
);
961 return mem
&& (CONSTANT_CLASS_P (mem
) || decl_address_invariant_p (mem
));
964 /* Given an address ADDR in STMT, find it in the memory log or add it,
965 making sure to keep only the addresses highest in the dominator
968 ENTRY_BLOCK is the entry_block for the transaction.
970 If we find the address in the log, make sure it's either the same
971 address, or an equivalent one that dominates ADDR.
973 If we find the address, but neither ADDR dominates the found
974 address, nor the found one dominates ADDR, we're on different
975 execution paths. Add it.
977 If known, ENTRY_BLOCK is the entry block for the region, otherwise
980 tm_log_add (basic_block entry_block
, tree addr
, gimple stmt
)
983 struct tm_log_entry l
, *lp
;
986 slot
= htab_find_slot (tm_log
, &l
, INSERT
);
989 tree type
= TREE_TYPE (addr
);
991 lp
= XNEW (struct tm_log_entry
);
995 /* Small invariant addresses can be handled as save/restores. */
997 && transaction_invariant_address_p (lp
->addr
, entry_block
)
998 && TYPE_SIZE_UNIT (type
) != NULL
999 && host_integerp (TYPE_SIZE_UNIT (type
), 1)
1000 && (tree_low_cst (TYPE_SIZE_UNIT (type
), 1)
1001 < PARAM_VALUE (PARAM_TM_MAX_AGGREGATE_SIZE
))
1002 /* We must be able to copy this type normally. I.e., no
1003 special constructors and the like. */
1004 && !TREE_ADDRESSABLE (type
))
1006 lp
->save_var
= create_tmp_var (TREE_TYPE (lp
->addr
), "tm_save");
1007 add_referenced_var (lp
->save_var
);
1009 lp
->entry_block
= entry_block
;
1010 /* Save addresses separately in dominator order so we don't
1011 get confused by overlapping addresses in the save/restore
1013 VEC_safe_push (tree
, heap
, tm_log_save_addresses
, lp
->addr
);
1017 /* Use the logging functions. */
1018 lp
->stmts
= VEC_alloc (gimple
, heap
, 5);
1019 VEC_quick_push (gimple
, lp
->stmts
, stmt
);
1020 lp
->save_var
= NULL
;
1028 lp
= (struct tm_log_entry
*) *slot
;
1030 /* If we're generating a save/restore sequence, we don't care
1031 about statements. */
1035 for (i
= 0; VEC_iterate (gimple
, lp
->stmts
, i
, oldstmt
); ++i
)
1037 if (stmt
== oldstmt
)
1039 /* We already have a store to the same address, higher up the
1040 dominator tree. Nothing to do. */
1041 if (dominated_by_p (CDI_DOMINATORS
,
1042 gimple_bb (stmt
), gimple_bb (oldstmt
)))
1044 /* We should be processing blocks in dominator tree order. */
1045 gcc_assert (!dominated_by_p (CDI_DOMINATORS
,
1046 gimple_bb (oldstmt
), gimple_bb (stmt
)));
1048 /* Store is on a different code path. */
1049 VEC_safe_push (gimple
, heap
, lp
->stmts
, stmt
);
1053 /* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME
1054 result, insert the new statements before GSI. */
1057 gimplify_addr (gimple_stmt_iterator
*gsi
, tree x
)
1059 if (TREE_CODE (x
) == TARGET_MEM_REF
)
1060 x
= tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x
)), x
);
1062 x
= build_fold_addr_expr (x
);
1063 return force_gimple_operand_gsi (gsi
, x
, true, NULL
, true, GSI_SAME_STMT
);
1066 /* Instrument one address with the logging functions.
1067 ADDR is the address to save.
1068 STMT is the statement before which to place it. */
1070 tm_log_emit_stmt (tree addr
, gimple stmt
)
1072 tree type
= TREE_TYPE (addr
);
1073 tree size
= TYPE_SIZE_UNIT (type
);
1074 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
1076 enum built_in_function code
= BUILT_IN_TM_LOG
;
1078 if (type
== float_type_node
)
1079 code
= BUILT_IN_TM_LOG_FLOAT
;
1080 else if (type
== double_type_node
)
1081 code
= BUILT_IN_TM_LOG_DOUBLE
;
1082 else if (type
== long_double_type_node
)
1083 code
= BUILT_IN_TM_LOG_LDOUBLE
;
1084 else if (host_integerp (size
, 1))
1086 unsigned int n
= tree_low_cst (size
, 1);
1090 code
= BUILT_IN_TM_LOG_1
;
1093 code
= BUILT_IN_TM_LOG_2
;
1096 code
= BUILT_IN_TM_LOG_4
;
1099 code
= BUILT_IN_TM_LOG_8
;
1102 code
= BUILT_IN_TM_LOG
;
1103 if (TREE_CODE (type
) == VECTOR_TYPE
)
1105 if (n
== 8 && builtin_decl_explicit (BUILT_IN_TM_LOG_M64
))
1106 code
= BUILT_IN_TM_LOG_M64
;
1107 else if (n
== 16 && builtin_decl_explicit (BUILT_IN_TM_LOG_M128
))
1108 code
= BUILT_IN_TM_LOG_M128
;
1109 else if (n
== 32 && builtin_decl_explicit (BUILT_IN_TM_LOG_M256
))
1110 code
= BUILT_IN_TM_LOG_M256
;
1116 addr
= gimplify_addr (&gsi
, addr
);
1117 if (code
== BUILT_IN_TM_LOG
)
1118 log
= gimple_build_call (builtin_decl_explicit (code
), 2, addr
, size
);
1120 log
= gimple_build_call (builtin_decl_explicit (code
), 1, addr
);
1121 gsi_insert_before (&gsi
, log
, GSI_SAME_STMT
);
1124 /* Go through the log and instrument address that must be instrumented
1125 with the logging functions. Leave the save/restore addresses for
1131 struct tm_log_entry
*lp
;
1133 FOR_EACH_HTAB_ELEMENT (tm_log
, lp
, tm_log_entry_t
, hi
)
1140 fprintf (dump_file
, "TM thread private mem logging: ");
1141 print_generic_expr (dump_file
, lp
->addr
, 0);
1142 fprintf (dump_file
, "\n");
1148 fprintf (dump_file
, "DUMPING to variable\n");
1154 fprintf (dump_file
, "DUMPING with logging functions\n");
1155 for (i
= 0; VEC_iterate (gimple
, lp
->stmts
, i
, stmt
); ++i
)
1156 tm_log_emit_stmt (lp
->addr
, stmt
);
1161 /* Emit the save sequence for the corresponding addresses in the log.
1162 ENTRY_BLOCK is the entry block for the transaction.
1163 BB is the basic block to insert the code in. */
1165 tm_log_emit_saves (basic_block entry_block
, basic_block bb
)
1168 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1170 struct tm_log_entry l
, *lp
;
1172 for (i
= 0; i
< VEC_length (tree
, tm_log_save_addresses
); ++i
)
1174 l
.addr
= VEC_index (tree
, tm_log_save_addresses
, i
);
1175 lp
= (struct tm_log_entry
*) *htab_find_slot (tm_log
, &l
, NO_INSERT
);
1176 gcc_assert (lp
->save_var
!= NULL
);
1178 /* We only care about variables in the current transaction. */
1179 if (lp
->entry_block
!= entry_block
)
1182 stmt
= gimple_build_assign (lp
->save_var
, unshare_expr (lp
->addr
));
1184 /* Make sure we can create an SSA_NAME for this type. For
1185 instance, aggregates aren't allowed, in which case the system
1186 will create a VOP for us and everything will just work. */
1187 if (is_gimple_reg_type (TREE_TYPE (lp
->save_var
)))
1189 lp
->save_var
= make_ssa_name (lp
->save_var
, stmt
);
1190 gimple_assign_set_lhs (stmt
, lp
->save_var
);
1193 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
1197 /* Emit the restore sequence for the corresponding addresses in the log.
1198 ENTRY_BLOCK is the entry block for the transaction.
1199 BB is the basic block to insert the code in. */
1201 tm_log_emit_restores (basic_block entry_block
, basic_block bb
)
1204 struct tm_log_entry l
, *lp
;
1205 gimple_stmt_iterator gsi
;
1208 for (i
= VEC_length (tree
, tm_log_save_addresses
) - 1; i
>= 0; i
--)
1210 l
.addr
= VEC_index (tree
, tm_log_save_addresses
, i
);
1211 lp
= (struct tm_log_entry
*) *htab_find_slot (tm_log
, &l
, NO_INSERT
);
1212 gcc_assert (lp
->save_var
!= NULL
);
1214 /* We only care about variables in the current transaction. */
1215 if (lp
->entry_block
!= entry_block
)
1218 /* Restores are in LIFO order from the saves in case we have
1220 gsi
= gsi_start_bb (bb
);
1222 stmt
= gimple_build_assign (unshare_expr (lp
->addr
), lp
->save_var
);
1223 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1227 /* Emit the checks for performing either a save or a restore sequence.
1229 TRXN_PROP is either A_SAVELIVEVARIABLES or A_RESTORELIVEVARIABLES.
1231 The code sequence is inserted in a new basic block created in
1232 END_BB which is inserted between BEFORE_BB and the destination of
1235 STATUS is the return value from _ITM_beginTransaction.
1236 ENTRY_BLOCK is the entry block for the transaction.
1237 EMITF is a callback to emit the actual save/restore code.
1239 The basic block containing the conditional checking for TRXN_PROP
1242 tm_log_emit_save_or_restores (basic_block entry_block
,
1245 void (*emitf
)(basic_block
, basic_block
),
1246 basic_block before_bb
,
1248 basic_block
*end_bb
)
1250 basic_block cond_bb
, code_bb
;
1251 gimple cond_stmt
, stmt
;
1252 gimple_stmt_iterator gsi
;
1254 int old_flags
= fallthru_edge
->flags
;
1256 cond_bb
= create_empty_bb (before_bb
);
1257 code_bb
= create_empty_bb (cond_bb
);
1258 *end_bb
= create_empty_bb (code_bb
);
1259 redirect_edge_pred (fallthru_edge
, *end_bb
);
1260 fallthru_edge
->flags
= EDGE_FALLTHRU
;
1261 make_edge (before_bb
, cond_bb
, old_flags
);
1263 set_immediate_dominator (CDI_DOMINATORS
, cond_bb
, before_bb
);
1264 set_immediate_dominator (CDI_DOMINATORS
, code_bb
, cond_bb
);
1266 gsi
= gsi_last_bb (cond_bb
);
1268 /* t1 = status & A_{property}. */
1269 t1
= make_rename_temp (TREE_TYPE (status
), NULL
);
1270 t2
= build_int_cst (TREE_TYPE (status
), trxn_prop
);
1271 stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
, status
, t2
);
1272 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1275 t2
= build_int_cst (TREE_TYPE (status
), 0);
1276 cond_stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
1277 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
1279 emitf (entry_block
, code_bb
);
1281 make_edge (cond_bb
, code_bb
, EDGE_TRUE_VALUE
);
1282 make_edge (cond_bb
, *end_bb
, EDGE_FALSE_VALUE
);
1283 make_edge (code_bb
, *end_bb
, EDGE_FALLTHRU
);
1288 static tree
lower_sequence_tm (gimple_stmt_iterator
*, bool *,
1289 struct walk_stmt_info
*);
1290 static tree
lower_sequence_no_tm (gimple_stmt_iterator
*, bool *,
1291 struct walk_stmt_info
*);
1293 /* Evaluate an address X being dereferenced and determine if it
1294 originally points to a non aliased new chunk of memory (malloc,
1297 Return MEM_THREAD_LOCAL if it points to a thread-local address.
1298 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address.
1299 Return MEM_NON_LOCAL otherwise.
1301 ENTRY_BLOCK is the entry block to the transaction containing the
1302 dereference of X. */
1303 static enum thread_memory_type
1304 thread_private_new_memory (basic_block entry_block
, tree x
)
1307 enum tree_code code
;
1309 tm_new_mem_map_t elt
, *elt_p
;
1311 enum thread_memory_type retval
= mem_transaction_local
;
1314 || TREE_CODE (x
) != SSA_NAME
1315 /* Possible uninitialized use, or a function argument. In
1316 either case, we don't care. */
1317 || SSA_NAME_IS_DEFAULT_DEF (x
))
1318 return mem_non_local
;
1320 /* Look in cache first. */
1322 slot
= htab_find_slot (tm_new_mem_hash
, &elt
, INSERT
);
1323 elt_p
= (tm_new_mem_map_t
*) *slot
;
1325 return elt_p
->local_new_memory
;
1327 /* Optimistically assume the memory is transaction local during
1328 processing. This catches recursion into this variable. */
1329 *slot
= elt_p
= XNEW (tm_new_mem_map_t
);
1331 elt_p
->local_new_memory
= mem_transaction_local
;
1333 /* Search DEF chain to find the original definition of this address. */
1336 if (ptr_deref_may_alias_global_p (x
))
1338 /* Address escapes. This is not thread-private. */
1339 retval
= mem_non_local
;
1340 goto new_memory_ret
;
1343 stmt
= SSA_NAME_DEF_STMT (x
);
1345 /* If the malloc call is outside the transaction, this is
1347 if (retval
!= mem_thread_local
1348 && !dominated_by_p (CDI_DOMINATORS
, gimple_bb (stmt
), entry_block
))
1349 retval
= mem_thread_local
;
1351 if (is_gimple_assign (stmt
))
1353 code
= gimple_assign_rhs_code (stmt
);
1354 /* x = foo ==> foo */
1355 if (code
== SSA_NAME
)
1356 x
= gimple_assign_rhs1 (stmt
);
1357 /* x = foo + n ==> foo */
1358 else if (code
== POINTER_PLUS_EXPR
)
1359 x
= gimple_assign_rhs1 (stmt
);
1360 /* x = (cast*) foo ==> foo */
1361 else if (code
== VIEW_CONVERT_EXPR
|| code
== NOP_EXPR
)
1362 x
= gimple_assign_rhs1 (stmt
);
1365 retval
= mem_non_local
;
1366 goto new_memory_ret
;
1371 if (gimple_code (stmt
) == GIMPLE_PHI
)
1374 enum thread_memory_type mem
;
1375 tree phi_result
= gimple_phi_result (stmt
);
1377 /* If any of the ancestors are non-local, we are sure to
1378 be non-local. Otherwise we can avoid doing anything
1379 and inherit what has already been generated. */
1381 for (i
= 0; i
< gimple_phi_num_args (stmt
); ++i
)
1383 tree op
= PHI_ARG_DEF (stmt
, i
);
1385 /* Exclude self-assignment. */
1386 if (phi_result
== op
)
1389 mem
= thread_private_new_memory (entry_block
, op
);
1390 if (mem
== mem_non_local
)
1393 goto new_memory_ret
;
1395 retval
= MIN (retval
, mem
);
1397 goto new_memory_ret
;
1402 while (TREE_CODE (x
) == SSA_NAME
);
1404 if (stmt
&& is_gimple_call (stmt
) && gimple_call_flags (stmt
) & ECF_MALLOC
)
1405 /* Thread-local or transaction-local. */
1408 retval
= mem_non_local
;
1411 elt_p
->local_new_memory
= retval
;
1415 /* Determine whether X has to be instrumented using a read
1418 ENTRY_BLOCK is the entry block for the region where stmt resides
1419 in. NULL if unknown.
1421 STMT is the statement in which X occurs in. It is used for thread
1422 private memory instrumentation. If no TPM instrumentation is
1423 desired, STMT should be null. */
1425 requires_barrier (basic_block entry_block
, tree x
, gimple stmt
)
1428 while (handled_component_p (x
))
1429 x
= TREE_OPERAND (x
, 0);
1431 switch (TREE_CODE (x
))
1436 enum thread_memory_type ret
;
1438 ret
= thread_private_new_memory (entry_block
, TREE_OPERAND (x
, 0));
1439 if (ret
== mem_non_local
)
1441 if (stmt
&& ret
== mem_thread_local
)
1442 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */
1443 tm_log_add (entry_block
, orig
, stmt
);
1445 /* Transaction-locals require nothing at all. For malloc, a
1446 transaction restart frees the memory and we reallocate.
1447 For alloca, the stack pointer gets reset by the retry and
1452 case TARGET_MEM_REF
:
1453 if (TREE_CODE (TMR_BASE (x
)) != ADDR_EXPR
)
1455 x
= TREE_OPERAND (TMR_BASE (x
), 0);
1456 if (TREE_CODE (x
) == PARM_DECL
)
1458 gcc_assert (TREE_CODE (x
) == VAR_DECL
);
1464 if (DECL_BY_REFERENCE (x
))
1466 /* ??? This value is a pointer, but aggregate_value_p has been
1467 jigged to return true which confuses needs_to_live_in_memory.
1468 This ought to be cleaned up generically.
1470 FIXME: Verify this still happens after the next mainline
1471 merge. Testcase ie g++.dg/tm/pr47554.C.
1476 if (is_global_var (x
))
1477 return !TREE_READONLY (x
);
1478 if (/* FIXME: This condition should actually go below in the
1479 tm_log_add() call, however is_call_clobbered() depends on
1480 aliasing info which is not available during
1481 gimplification. Since requires_barrier() gets called
1482 during lower_sequence_tm/gimplification, leave the call
1483 to needs_to_live_in_memory until we eliminate
1484 lower_sequence_tm altogether. */
1485 needs_to_live_in_memory (x
)
1487 || ptr_deref_may_alias_global_p (x
))
1491 /* For local memory that doesn't escape (aka thread private
1492 memory), we can either save the value at the beginning of
1493 the transaction and restore on restart, or call a tm
1494 function to dynamically save and restore on restart
1497 tm_log_add (entry_block
, orig
, stmt
);
1506 /* Mark the GIMPLE_ASSIGN statement as appropriate for being inside
1507 a transaction region. */
1510 examine_assign_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1512 gimple stmt
= gsi_stmt (*gsi
);
1514 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_rhs1 (stmt
), NULL
))
1515 *state
|= GTMA_HAVE_LOAD
;
1516 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_lhs (stmt
), NULL
))
1517 *state
|= GTMA_HAVE_STORE
;
1520 /* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */
1523 examine_call_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1525 gimple stmt
= gsi_stmt (*gsi
);
1528 if (is_tm_pure_call (stmt
))
1531 /* Check if this call is a transaction abort. */
1532 fn
= gimple_call_fndecl (stmt
);
1533 if (is_tm_abort (fn
))
1534 *state
|= GTMA_HAVE_ABORT
;
1536 /* Note that something may happen. */
1537 *state
|= GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
;
1540 /* Lower a GIMPLE_TRANSACTION statement. */
1543 lower_transaction (gimple_stmt_iterator
*gsi
, struct walk_stmt_info
*wi
)
1545 gimple g
, stmt
= gsi_stmt (*gsi
);
1546 unsigned int *outer_state
= (unsigned int *) wi
->info
;
1547 unsigned int this_state
= 0;
1548 struct walk_stmt_info this_wi
;
1550 /* First, lower the body. The scanning that we do inside gives
1551 us some idea of what we're dealing with. */
1552 memset (&this_wi
, 0, sizeof (this_wi
));
1553 this_wi
.info
= (void *) &this_state
;
1554 walk_gimple_seq (gimple_transaction_body (stmt
),
1555 lower_sequence_tm
, NULL
, &this_wi
);
1557 /* If there was absolutely nothing transaction related inside the
1558 transaction, we may elide it. Likewise if this is a nested
1559 transaction and does not contain an abort. */
1561 || (!(this_state
& GTMA_HAVE_ABORT
) && outer_state
!= NULL
))
1564 *outer_state
|= this_state
;
1566 gsi_insert_seq_before (gsi
, gimple_transaction_body (stmt
),
1568 gimple_transaction_set_body (stmt
, NULL
);
1570 gsi_remove (gsi
, true);
1571 wi
->removed_stmt
= true;
1575 /* Wrap the body of the transaction in a try-finally node so that
1576 the commit call is always properly called. */
1577 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT
), 0);
1578 if (flag_exceptions
)
1581 gimple_seq n_seq
, e_seq
;
1583 n_seq
= gimple_seq_alloc_with_stmt (g
);
1584 e_seq
= gimple_seq_alloc ();
1586 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER
),
1587 1, integer_zero_node
);
1588 ptr
= create_tmp_var (ptr_type_node
, NULL
);
1589 gimple_call_set_lhs (g
, ptr
);
1590 gimple_seq_add_stmt (&e_seq
, g
);
1592 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH
),
1594 gimple_seq_add_stmt (&e_seq
, g
);
1596 g
= gimple_build_eh_else (n_seq
, e_seq
);
1599 g
= gimple_build_try (gimple_transaction_body (stmt
),
1600 gimple_seq_alloc_with_stmt (g
), GIMPLE_TRY_FINALLY
);
1601 gsi_insert_after (gsi
, g
, GSI_CONTINUE_LINKING
);
1603 gimple_transaction_set_body (stmt
, NULL
);
1605 /* If the transaction calls abort or if this is an outer transaction,
1606 add an "over" label afterwards. */
1607 if ((this_state
& (GTMA_HAVE_ABORT
))
1608 || (gimple_transaction_subcode(stmt
) & GTMA_IS_OUTER
))
1610 tree label
= create_artificial_label (UNKNOWN_LOCATION
);
1611 gimple_transaction_set_label (stmt
, label
);
1612 gsi_insert_after (gsi
, gimple_build_label (label
), GSI_CONTINUE_LINKING
);
1615 /* Record the set of operations found for use later. */
1616 this_state
|= gimple_transaction_subcode (stmt
) & GTMA_DECLARATION_MASK
;
1617 gimple_transaction_set_subcode (stmt
, this_state
);
1620 /* Iterate through the statements in the sequence, lowering them all
1621 as appropriate for being in a transaction. */
1624 lower_sequence_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1625 struct walk_stmt_info
*wi
)
1627 unsigned int *state
= (unsigned int *) wi
->info
;
1628 gimple stmt
= gsi_stmt (*gsi
);
1630 *handled_ops_p
= true;
1631 switch (gimple_code (stmt
))
1634 /* Only memory reads/writes need to be instrumented. */
1635 if (gimple_assign_single_p (stmt
))
1636 examine_assign_tm (state
, gsi
);
1640 examine_call_tm (state
, gsi
);
1644 *state
|= GTMA_MAY_ENTER_IRREVOCABLE
;
1647 case GIMPLE_TRANSACTION
:
1648 lower_transaction (gsi
, wi
);
1652 *handled_ops_p
= !gimple_has_substatements (stmt
);
1659 /* Iterate through the statements in the sequence, lowering them all
1660 as appropriate for being outside of a transaction. */
1663 lower_sequence_no_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1664 struct walk_stmt_info
* wi
)
1666 gimple stmt
= gsi_stmt (*gsi
);
1668 if (gimple_code (stmt
) == GIMPLE_TRANSACTION
)
1670 *handled_ops_p
= true;
1671 lower_transaction (gsi
, wi
);
1674 *handled_ops_p
= !gimple_has_substatements (stmt
);
1679 /* Main entry point for flattening GIMPLE_TRANSACTION constructs. After
1680 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has
1681 been moved out, and all the data required for constructing a proper
1682 CFG has been recorded. */
1685 execute_lower_tm (void)
1687 struct walk_stmt_info wi
;
1689 /* Transactional clones aren't created until a later pass. */
1690 gcc_assert (!decl_is_tm_clone (current_function_decl
));
1692 memset (&wi
, 0, sizeof (wi
));
1693 walk_gimple_seq (gimple_body (current_function_decl
),
1694 lower_sequence_no_tm
, NULL
, &wi
);
1699 struct gimple_opt_pass pass_lower_tm
=
1703 "tmlower", /* name */
1705 execute_lower_tm
, /* execute */
1708 0, /* static_pass_number */
1709 TV_TRANS_MEM
, /* tv_id */
1710 PROP_gimple_lcf
, /* properties_required */
1711 0, /* properties_provided */
1712 0, /* properties_destroyed */
1713 0, /* todo_flags_start */
1714 TODO_dump_func
/* todo_flags_finish */
1718 /* Collect region information for each transaction. */
1722 /* Link to the next unnested transaction. */
1723 struct tm_region
*next
;
1725 /* Link to the next inner transaction. */
1726 struct tm_region
*inner
;
1728 /* Link to the next outer transaction. */
1729 struct tm_region
*outer
;
1731 /* The GIMPLE_TRANSACTION statement beginning this transaction. */
1732 gimple transaction_stmt
;
1734 /* The entry block to this region. */
1735 basic_block entry_block
;
1737 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK.
1738 These blocks are still a part of the region (i.e., the border is
1739 inclusive). Note that this set is only complete for paths in the CFG
1740 starting at ENTRY_BLOCK, and that there is no exit block recorded for
1741 the edge to the "over" label. */
1744 /* The set of all blocks that have an TM_IRREVOCABLE call. */
1748 /* True if there are pending edge statements to be committed for the
1749 current function being scanned in the tmmark pass. */
1750 bool pending_edge_inserts_p
;
1752 static struct tm_region
*all_tm_regions
;
1753 static bitmap_obstack tm_obstack
;
1756 /* A subroutine of tm_region_init. Record the existance of the
1757 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */
1759 static struct tm_region
*
1760 tm_region_init_0 (struct tm_region
*outer
, basic_block bb
, gimple stmt
)
1762 struct tm_region
*region
;
1764 region
= (struct tm_region
*)
1765 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1769 region
->next
= outer
->inner
;
1770 outer
->inner
= region
;
1774 region
->next
= all_tm_regions
;
1775 all_tm_regions
= region
;
1777 region
->inner
= NULL
;
1778 region
->outer
= outer
;
1780 region
->transaction_stmt
= stmt
;
1782 /* There are either one or two edges out of the block containing
1783 the GIMPLE_TRANSACTION, one to the actual region and one to the
1784 "over" label if the region contains an abort. The former will
1785 always be the one marked FALLTHRU. */
1786 region
->entry_block
= FALLTHRU_EDGE (bb
)->dest
;
1788 region
->exit_blocks
= BITMAP_ALLOC (&tm_obstack
);
1789 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1794 /* A subroutine of tm_region_init. Record all the exit and
1795 irrevocable blocks in BB into the region's exit_blocks and
1796 irr_blocks bitmaps. Returns the new region being scanned. */
1798 static struct tm_region
*
1799 tm_region_init_1 (struct tm_region
*region
, basic_block bb
)
1801 gimple_stmt_iterator gsi
;
1805 || (!region
->irr_blocks
&& !region
->exit_blocks
))
1808 /* Check to see if this is the end of a region by seeing if it
1809 contains a call to __builtin_tm_commit{,_eh}. Note that the
1810 outermost region for DECL_IS_TM_CLONE need not collect this. */
1811 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
1814 if (gimple_code (g
) == GIMPLE_CALL
)
1816 tree fn
= gimple_call_fndecl (g
);
1817 if (fn
&& DECL_BUILT_IN_CLASS (fn
) == BUILT_IN_NORMAL
)
1819 if ((DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT
1820 || DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT_EH
)
1821 && region
->exit_blocks
)
1823 bitmap_set_bit (region
->exit_blocks
, bb
->index
);
1824 region
= region
->outer
;
1827 if (DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_IRREVOCABLE
)
1828 bitmap_set_bit (region
->irr_blocks
, bb
->index
);
1835 /* Collect all of the transaction regions within the current function
1836 and record them in ALL_TM_REGIONS. The REGION parameter may specify
1837 an "outermost" region for use by tm clones. */
1840 tm_region_init (struct tm_region
*region
)
1846 VEC(basic_block
, heap
) *queue
= NULL
;
1847 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
1848 struct tm_region
*old_region
;
1850 all_tm_regions
= region
;
1851 bb
= single_succ (ENTRY_BLOCK_PTR
);
1853 VEC_safe_push (basic_block
, heap
, queue
, bb
);
1854 gcc_assert (!bb
->aux
); /* FIXME: Remove me. */
1858 bb
= VEC_pop (basic_block
, queue
);
1859 region
= (struct tm_region
*)bb
->aux
;
1862 /* Record exit and irrevocable blocks. */
1863 region
= tm_region_init_1 (region
, bb
);
1865 /* Check for the last statement in the block beginning a new region. */
1867 old_region
= region
;
1868 if (g
&& gimple_code (g
) == GIMPLE_TRANSACTION
)
1869 region
= tm_region_init_0 (region
, bb
, g
);
1871 /* Process subsequent blocks. */
1872 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1873 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
1875 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
1876 VEC_safe_push (basic_block
, heap
, queue
, e
->dest
);
1877 gcc_assert (!e
->dest
->aux
); /* FIXME: Remove me. */
1879 /* If the current block started a new region, make sure that only
1880 the entry block of the new region is associated with this region.
1881 Other successors are still part of the old region. */
1882 if (old_region
!= region
&& e
->dest
!= region
->entry_block
)
1883 e
->dest
->aux
= old_region
;
1885 e
->dest
->aux
= region
;
1888 while (!VEC_empty (basic_block
, queue
));
1889 VEC_free (basic_block
, heap
, queue
);
1890 BITMAP_FREE (visited_blocks
);
1893 /* The "gate" function for all transactional memory expansion and optimization
1894 passes. We collect region information for each top-level transaction, and
1895 if we don't find any, we skip all of the TM passes. Each region will have
1896 all of the exit blocks recorded, and the originating statement. */
1904 calculate_dominance_info (CDI_DOMINATORS
);
1905 bitmap_obstack_initialize (&tm_obstack
);
1907 /* If the function is a TM_CLONE, then the entire function is the region. */
1908 if (decl_is_tm_clone (current_function_decl
))
1910 struct tm_region
*region
= (struct tm_region
*)
1911 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1912 memset (region
, 0, sizeof (*region
));
1913 region
->entry_block
= single_succ (ENTRY_BLOCK_PTR
);
1914 /* For a clone, the entire function is the region. But even if
1915 we don't need to record any exit blocks, we may need to
1916 record irrevocable blocks. */
1917 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1919 tm_region_init (region
);
1923 tm_region_init (NULL
);
1925 /* If we didn't find any regions, cleanup and skip the whole tree
1926 of tm-related optimizations. */
1927 if (all_tm_regions
== NULL
)
1929 bitmap_obstack_release (&tm_obstack
);
1937 struct gimple_opt_pass pass_tm_init
=
1941 "*tminit", /* name */
1942 gate_tm_init
, /* gate */
1946 0, /* static_pass_number */
1947 TV_TRANS_MEM
, /* tv_id */
1948 PROP_ssa
| PROP_cfg
, /* properties_required */
1949 0, /* properties_provided */
1950 0, /* properties_destroyed */
1951 0, /* todo_flags_start */
1952 0, /* todo_flags_finish */
1956 /* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region
1957 represented by STATE. */
1960 transaction_subcode_ior (struct tm_region
*region
, unsigned flags
)
1962 if (region
&& region
->transaction_stmt
)
1964 flags
|= gimple_transaction_subcode (region
->transaction_stmt
);
1965 gimple_transaction_set_subcode (region
->transaction_stmt
, flags
);
1969 /* Construct a memory load in a transactional context. Return the
1970 gimple statement performing the load, or NULL if there is no
1971 TM_LOAD builtin of the appropriate size to do the load.
1973 LOC is the location to use for the new statement(s). */
1976 build_tm_load (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
1978 enum built_in_function code
= END_BUILTINS
;
1979 tree t
, type
= TREE_TYPE (rhs
), decl
;
1982 if (type
== float_type_node
)
1983 code
= BUILT_IN_TM_LOAD_FLOAT
;
1984 else if (type
== double_type_node
)
1985 code
= BUILT_IN_TM_LOAD_DOUBLE
;
1986 else if (type
== long_double_type_node
)
1987 code
= BUILT_IN_TM_LOAD_LDOUBLE
;
1988 else if (TYPE_SIZE_UNIT (type
) != NULL
1989 && host_integerp (TYPE_SIZE_UNIT (type
), 1))
1991 switch (tree_low_cst (TYPE_SIZE_UNIT (type
), 1))
1994 code
= BUILT_IN_TM_LOAD_1
;
1997 code
= BUILT_IN_TM_LOAD_2
;
2000 code
= BUILT_IN_TM_LOAD_4
;
2003 code
= BUILT_IN_TM_LOAD_8
;
2008 if (code
== END_BUILTINS
)
2010 decl
= targetm
.vectorize
.builtin_tm_load (type
);
2015 decl
= builtin_decl_explicit (code
);
2017 t
= gimplify_addr (gsi
, rhs
);
2018 gcall
= gimple_build_call (decl
, 1, t
);
2019 gimple_set_location (gcall
, loc
);
2021 t
= TREE_TYPE (TREE_TYPE (decl
));
2022 if (useless_type_conversion_p (type
, t
))
2024 gimple_call_set_lhs (gcall
, lhs
);
2025 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2032 temp
= make_rename_temp (t
, NULL
);
2033 gimple_call_set_lhs (gcall
, temp
);
2034 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2036 t
= fold_build1 (VIEW_CONVERT_EXPR
, type
, temp
);
2037 g
= gimple_build_assign (lhs
, t
);
2038 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2045 /* Similarly for storing TYPE in a transactional context. */
2048 build_tm_store (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2050 enum built_in_function code
= END_BUILTINS
;
2051 tree t
, fn
, type
= TREE_TYPE (rhs
), simple_type
;
2054 if (type
== float_type_node
)
2055 code
= BUILT_IN_TM_STORE_FLOAT
;
2056 else if (type
== double_type_node
)
2057 code
= BUILT_IN_TM_STORE_DOUBLE
;
2058 else if (type
== long_double_type_node
)
2059 code
= BUILT_IN_TM_STORE_LDOUBLE
;
2060 else if (TYPE_SIZE_UNIT (type
) != NULL
2061 && host_integerp (TYPE_SIZE_UNIT (type
), 1))
2063 switch (tree_low_cst (TYPE_SIZE_UNIT (type
), 1))
2066 code
= BUILT_IN_TM_STORE_1
;
2069 code
= BUILT_IN_TM_STORE_2
;
2072 code
= BUILT_IN_TM_STORE_4
;
2075 code
= BUILT_IN_TM_STORE_8
;
2080 if (code
== END_BUILTINS
)
2082 fn
= targetm
.vectorize
.builtin_tm_store (type
);
2087 fn
= builtin_decl_explicit (code
);
2089 simple_type
= TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn
))));
2091 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
2093 /* Handle the easy initialization to zero. */
2094 if (CONSTRUCTOR_ELTS (rhs
) == 0)
2095 rhs
= build_int_cst (simple_type
, 0);
2098 /* ...otherwise punt to the caller and probably use
2099 BUILT_IN_TM_MEMMOVE, because we can't wrap a
2100 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce
2105 else if (!useless_type_conversion_p (simple_type
, type
))
2110 temp
= make_rename_temp (simple_type
, NULL
);
2111 t
= fold_build1 (VIEW_CONVERT_EXPR
, simple_type
, rhs
);
2112 g
= gimple_build_assign (temp
, t
);
2113 gimple_set_location (g
, loc
);
2114 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2119 t
= gimplify_addr (gsi
, lhs
);
2120 gcall
= gimple_build_call (fn
, 2, t
, rhs
);
2121 gimple_set_location (gcall
, loc
);
2122 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2128 /* Expand an assignment statement into transactional builtins. */
2131 expand_assign_tm (struct tm_region
*region
, gimple_stmt_iterator
*gsi
)
2133 gimple stmt
= gsi_stmt (*gsi
);
2134 location_t loc
= gimple_location (stmt
);
2135 tree lhs
= gimple_assign_lhs (stmt
);
2136 tree rhs
= gimple_assign_rhs1 (stmt
);
2137 bool store_p
= requires_barrier (region
->entry_block
, lhs
, NULL
);
2138 bool load_p
= requires_barrier (region
->entry_block
, rhs
, NULL
);
2139 gimple gcall
= NULL
;
2141 if (!load_p
&& !store_p
)
2143 /* Add thread private addresses to log if applicable. */
2144 requires_barrier (region
->entry_block
, lhs
, stmt
);
2149 gsi_remove (gsi
, true);
2151 if (load_p
&& !store_p
)
2153 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2154 gcall
= build_tm_load (loc
, lhs
, rhs
, gsi
);
2156 else if (store_p
&& !load_p
)
2158 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2159 gcall
= build_tm_store (loc
, lhs
, rhs
, gsi
);
2163 tree lhs_addr
, rhs_addr
;
2166 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2168 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2170 /* ??? Figure out if there's any possible overlap between the LHS
2171 and the RHS and if not, use MEMCPY. */
2172 lhs_addr
= gimplify_addr (gsi
, lhs
);
2173 rhs_addr
= gimplify_addr (gsi
, rhs
);
2174 gcall
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
),
2175 3, lhs_addr
, rhs_addr
,
2176 TYPE_SIZE_UNIT (TREE_TYPE (lhs
)));
2177 gimple_set_location (gcall
, loc
);
2178 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2181 /* Now that we have the load/store in its instrumented form, add
2182 thread private addresses to the log if applicable. */
2184 requires_barrier (region
->entry_block
, lhs
, gcall
);
2186 /* add_stmt_to_tm_region (region, gcall); */
2190 /* Expand a call statement as appropriate for a transaction. That is,
2191 either verify that the call does not affect the transaction, or
2192 redirect the call to a clone that handles transactions, or change
2193 the transaction state to IRREVOCABLE. Return true if the call is
2194 one of the builtins that end a transaction. */
2197 expand_call_tm (struct tm_region
*region
,
2198 gimple_stmt_iterator
*gsi
)
2200 gimple stmt
= gsi_stmt (*gsi
);
2201 tree lhs
= gimple_call_lhs (stmt
);
2203 struct cgraph_node
*node
;
2204 bool retval
= false;
2206 fn_decl
= gimple_call_fndecl (stmt
);
2208 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMCPY
)
2209 || fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
))
2210 transaction_subcode_ior (region
, GTMA_HAVE_STORE
| GTMA_HAVE_LOAD
);
2211 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMSET
))
2212 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2214 if (is_tm_pure_call (stmt
))
2218 retval
= is_tm_ending_fndecl (fn_decl
);
2221 /* Assume all non-const/pure calls write to memory, except
2222 transaction ending builtins. */
2223 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2226 /* For indirect calls, we already generated a call into the runtime. */
2229 tree fn
= gimple_call_fn (stmt
);
2231 /* We are guaranteed never to go irrevocable on a safe or pure
2232 call, and the pure call was handled above. */
2233 if (is_tm_safe (fn
))
2236 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2241 node
= cgraph_get_node (fn_decl
);
2242 if (node
->local
.tm_may_enter_irr
)
2243 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2245 if (is_tm_abort (fn_decl
))
2247 transaction_subcode_ior (region
, GTMA_HAVE_ABORT
);
2251 /* Instrument the store if needed.
2253 If the assignment happens inside the function call (return slot
2254 optimization), there is no instrumentation to be done, since
2255 the callee should have done the right thing. */
2256 if (lhs
&& requires_barrier (region
->entry_block
, lhs
, stmt
)
2257 && !gimple_call_return_slot_opt_p (stmt
))
2259 tree tmp
= make_rename_temp (TREE_TYPE (lhs
), NULL
);
2260 location_t loc
= gimple_location (stmt
);
2261 edge fallthru_edge
= NULL
;
2263 /* Remember if the call was going to throw. */
2264 if (stmt_can_throw_internal (stmt
))
2268 basic_block bb
= gimple_bb (stmt
);
2270 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2271 if (e
->flags
& EDGE_FALLTHRU
)
2278 gimple_call_set_lhs (stmt
, tmp
);
2280 stmt
= gimple_build_assign (lhs
, tmp
);
2281 gimple_set_location (stmt
, loc
);
2283 /* We cannot throw in the middle of a BB. If the call was going
2284 to throw, place the instrumentation on the fallthru edge, so
2285 the call remains the last statement in the block. */
2288 gimple_seq fallthru_seq
= gimple_seq_alloc_with_stmt (stmt
);
2289 gimple_stmt_iterator fallthru_gsi
= gsi_start (fallthru_seq
);
2290 expand_assign_tm (region
, &fallthru_gsi
);
2291 gsi_insert_seq_on_edge (fallthru_edge
, fallthru_seq
);
2292 pending_edge_inserts_p
= true;
2296 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
2297 expand_assign_tm (region
, gsi
);
2300 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2307 /* Expand all statements in BB as appropriate for being inside
2311 expand_block_tm (struct tm_region
*region
, basic_block bb
)
2313 gimple_stmt_iterator gsi
;
2315 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2317 gimple stmt
= gsi_stmt (gsi
);
2318 switch (gimple_code (stmt
))
2321 /* Only memory reads/writes need to be instrumented. */
2322 if (gimple_assign_single_p (stmt
))
2324 expand_assign_tm (region
, &gsi
);
2330 if (expand_call_tm (region
, &gsi
))
2340 if (!gsi_end_p (gsi
))
2345 /* Return the list of basic-blocks in REGION.
2347 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks
2348 following a TM_IRREVOCABLE call. */
2350 static VEC (basic_block
, heap
) *
2351 get_tm_region_blocks (basic_block entry_block
,
2354 bitmap all_region_blocks
,
2355 bool stop_at_irrevocable_p
)
2357 VEC(basic_block
, heap
) *bbs
= NULL
;
2361 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
2364 VEC_safe_push (basic_block
, heap
, bbs
, entry_block
);
2365 bitmap_set_bit (visited_blocks
, entry_block
->index
);
2369 basic_block bb
= VEC_index (basic_block
, bbs
, i
++);
2372 bitmap_bit_p (exit_blocks
, bb
->index
))
2375 if (stop_at_irrevocable_p
2377 && bitmap_bit_p (irr_blocks
, bb
->index
))
2380 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2381 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2383 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2384 VEC_safe_push (basic_block
, heap
, bbs
, e
->dest
);
2387 while (i
< VEC_length (basic_block
, bbs
));
2389 if (all_region_blocks
)
2390 bitmap_ior_into (all_region_blocks
, visited_blocks
);
2392 BITMAP_FREE (visited_blocks
);
2396 /* Entry point to the MARK phase of TM expansion. Here we replace
2397 transactional memory statements with calls to builtins, and function
2398 calls with their transactional clones (if available). But we don't
2399 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */
2402 execute_tm_mark (void)
2404 struct tm_region
*region
;
2406 VEC (basic_block
, heap
) *queue
;
2409 queue
= VEC_alloc (basic_block
, heap
, 10);
2410 pending_edge_inserts_p
= false;
2412 for (region
= all_tm_regions
; region
; region
= region
->next
)
2415 /* If we have a transaction... */
2416 if (region
->exit_blocks
)
2418 unsigned int subcode
2419 = gimple_transaction_subcode (region
->transaction_stmt
);
2421 /* Collect a new SUBCODE set, now that optimizations are done... */
2422 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2423 subcode
&= (GTMA_DECLARATION_MASK
| GTMA_DOES_GO_IRREVOCABLE
2424 | GTMA_MAY_ENTER_IRREVOCABLE
);
2426 subcode
&= GTMA_DECLARATION_MASK
;
2427 gimple_transaction_set_subcode (region
->transaction_stmt
, subcode
);
2430 queue
= get_tm_region_blocks (region
->entry_block
,
2431 region
->exit_blocks
,
2434 /*stop_at_irr_p=*/true);
2435 for (i
= 0; VEC_iterate (basic_block
, queue
, i
, bb
); ++i
)
2436 expand_block_tm (region
, bb
);
2437 VEC_free (basic_block
, heap
, queue
);
2442 if (pending_edge_inserts_p
)
2443 gsi_commit_edge_inserts ();
2447 struct gimple_opt_pass pass_tm_mark
=
2451 "tmmark", /* name */
2453 execute_tm_mark
, /* execute */
2456 0, /* static_pass_number */
2457 TV_TRANS_MEM
, /* tv_id */
2458 PROP_ssa
| PROP_cfg
, /* properties_required */
2459 0, /* properties_provided */
2460 0, /* properties_destroyed */
2461 0, /* todo_flags_start */
2464 | TODO_dump_func
, /* todo_flags_finish */
2468 /* Create an abnormal call edge from BB to the first block of the region
2469 represented by STATE. Also record the edge in the TM_RESTART map. */
2472 make_tm_edge (gimple stmt
, basic_block bb
, struct tm_region
*region
)
2475 struct tm_restart_node
*n
, dummy
;
2477 if (cfun
->gimple_df
->tm_restart
== NULL
)
2478 cfun
->gimple_df
->tm_restart
= htab_create_ggc (31, struct_ptr_hash
,
2479 struct_ptr_eq
, ggc_free
);
2482 dummy
.label_or_list
= gimple_block_label (region
->entry_block
);
2483 slot
= htab_find_slot (cfun
->gimple_df
->tm_restart
, &dummy
, INSERT
);
2484 n
= (struct tm_restart_node
*) *slot
;
2487 n
= ggc_alloc_tm_restart_node ();
2492 tree old
= n
->label_or_list
;
2493 if (TREE_CODE (old
) == LABEL_DECL
)
2494 old
= tree_cons (NULL
, old
, NULL
);
2495 n
->label_or_list
= tree_cons (NULL
, dummy
.label_or_list
, old
);
2498 make_edge (bb
, region
->entry_block
, EDGE_ABNORMAL
| EDGE_ABNORMAL_CALL
);
2502 /* Split block BB as necessary for every builtin function we added, and
2503 wire up the abnormal back edges implied by the transaction restart. */
2506 expand_block_edges (struct tm_region
*region
, basic_block bb
)
2508 gimple_stmt_iterator gsi
;
2510 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2512 gimple stmt
= gsi_stmt (gsi
);
2514 /* ??? TM_COMMIT (and any other tm builtin function) in a nested
2515 transaction has an abnormal edge back to the outer-most transaction
2516 (there are no nested retries), while a TM_ABORT also has an abnormal
2517 backedge to the inner-most transaction. We haven't actually saved
2518 the inner-most transaction here. We should be able to get to it
2519 via the region_nr saved on STMT, and read the transaction_stmt from
2520 that, and find the first region block from there. */
2521 /* ??? Shouldn't we split for any non-pure, non-irrevocable function? */
2522 if (gimple_code (stmt
) == GIMPLE_CALL
2523 && (gimple_call_flags (stmt
) & ECF_TM_BUILTIN
) != 0)
2525 if (gsi_one_before_end_p (gsi
))
2526 make_tm_edge (stmt
, bb
, region
);
2529 edge e
= split_block (bb
, stmt
);
2530 make_tm_edge (stmt
, bb
, region
);
2532 gsi
= gsi_start_bb (bb
);
2535 /* Delete any tail-call annotation that may have been added.
2536 The tail-call pass may have mis-identified the commit as being
2537 a candidate because we had not yet added this restart edge. */
2538 gimple_call_set_tail (stmt
, false);
2545 /* Expand the GIMPLE_TRANSACTION statement into the STM library call. */
2548 expand_transaction (struct tm_region
*region
)
2550 tree status
, tm_start
;
2551 basic_block atomic_bb
, slice_bb
;
2552 gimple_stmt_iterator gsi
;
2557 tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2558 status
= make_rename_temp (TREE_TYPE (TREE_TYPE (tm_start
)), "tm_state");
2560 /* ??? There are plenty of bits here we're not computing. */
2561 subcode
= gimple_transaction_subcode (region
->transaction_stmt
);
2562 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2563 flags
= PR_DOESGOIRREVOCABLE
| PR_UNINSTRUMENTEDCODE
;
2565 flags
= PR_INSTRUMENTEDCODE
;
2566 if ((subcode
& GTMA_MAY_ENTER_IRREVOCABLE
) == 0)
2567 flags
|= PR_HASNOIRREVOCABLE
;
2568 /* If the transaction does not have an abort in lexical scope and is not
2569 marked as an outer transaction, then it will never abort. */
2570 if ((subcode
& GTMA_HAVE_ABORT
) == 0
2571 && (subcode
& GTMA_IS_OUTER
) == 0)
2572 flags
|= PR_HASNOABORT
;
2573 if ((subcode
& GTMA_HAVE_STORE
) == 0)
2574 flags
|= PR_READONLY
;
2575 t2
= build_int_cst (TREE_TYPE (status
), flags
);
2576 g
= gimple_build_call (tm_start
, 1, t2
);
2577 gimple_call_set_lhs (g
, status
);
2578 gimple_set_location (g
, gimple_location (region
->transaction_stmt
));
2580 atomic_bb
= gimple_bb (region
->transaction_stmt
);
2582 if (!VEC_empty (tree
, tm_log_save_addresses
))
2583 tm_log_emit_saves (region
->entry_block
, atomic_bb
);
2585 gsi
= gsi_last_bb (atomic_bb
);
2586 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
2587 gsi_remove (&gsi
, true);
2589 if (!VEC_empty (tree
, tm_log_save_addresses
))
2590 region
->entry_block
=
2591 tm_log_emit_save_or_restores (region
->entry_block
,
2592 A_RESTORELIVEVARIABLES
,
2594 tm_log_emit_restores
,
2596 FALLTHRU_EDGE (atomic_bb
),
2599 slice_bb
= atomic_bb
;
2601 /* If we have an ABORT statement, create a test following the start
2602 call to perform the abort. */
2603 if (gimple_transaction_label (region
->transaction_stmt
))
2606 basic_block test_bb
;
2608 test_bb
= create_empty_bb (slice_bb
);
2609 if (VEC_empty (tree
, tm_log_save_addresses
))
2610 region
->entry_block
= test_bb
;
2611 gsi
= gsi_last_bb (test_bb
);
2613 t1
= make_rename_temp (TREE_TYPE (status
), NULL
);
2614 t2
= build_int_cst (TREE_TYPE (status
), A_ABORTTRANSACTION
);
2615 g
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
, status
, t2
);
2616 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
2618 t2
= build_int_cst (TREE_TYPE (status
), 0);
2619 g
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2620 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
2622 e
= FALLTHRU_EDGE (slice_bb
);
2623 redirect_edge_pred (e
, test_bb
);
2624 e
->flags
= EDGE_FALSE_VALUE
;
2625 e
->probability
= PROB_ALWAYS
- PROB_VERY_UNLIKELY
;
2627 e
= BRANCH_EDGE (atomic_bb
);
2628 redirect_edge_pred (e
, test_bb
);
2629 e
->flags
= EDGE_TRUE_VALUE
;
2630 e
->probability
= PROB_VERY_UNLIKELY
;
2632 e
= make_edge (slice_bb
, test_bb
, EDGE_FALLTHRU
);
2635 /* If we've no abort, but we do have PHIs at the beginning of the atomic
2636 region, that means we've a loop at the beginning of the atomic region
2637 that shares the first block. This can cause problems with the abnormal
2638 edges we're about to add for the transaction restart. Solve this by
2639 adding a new empty block to receive the abnormal edges. */
2640 else if (phi_nodes (region
->entry_block
))
2643 basic_block empty_bb
;
2645 region
->entry_block
= empty_bb
= create_empty_bb (atomic_bb
);
2647 e
= FALLTHRU_EDGE (atomic_bb
);
2648 redirect_edge_pred (e
, empty_bb
);
2650 e
= make_edge (atomic_bb
, empty_bb
, EDGE_FALLTHRU
);
2653 /* The GIMPLE_TRANSACTION statement no longer exists. */
2654 region
->transaction_stmt
= NULL
;
2657 static void expand_regions (struct tm_region
*);
2659 /* Helper function for expand_regions. Expand REGION and recurse to
2660 the inner region. */
2663 expand_regions_1 (struct tm_region
*region
)
2665 if (region
->exit_blocks
)
2669 VEC (basic_block
, heap
) *queue
;
2671 /* Collect the set of blocks in this region. Do this before
2672 splitting edges, so that we don't have to play with the
2673 dominator tree in the middle. */
2674 queue
= get_tm_region_blocks (region
->entry_block
,
2675 region
->exit_blocks
,
2678 /*stop_at_irr_p=*/false);
2679 expand_transaction (region
);
2680 for (i
= 0; VEC_iterate (basic_block
, queue
, i
, bb
); ++i
)
2681 expand_block_edges (region
, bb
);
2682 VEC_free (basic_block
, heap
, queue
);
2685 expand_regions (region
->inner
);
2688 /* Expand regions starting at REGION. */
2691 expand_regions (struct tm_region
*region
)
2695 expand_regions_1 (region
);
2696 region
= region
->next
;
2700 /* Entry point to the final expansion of transactional nodes. */
2703 execute_tm_edges (void)
2705 expand_regions (all_tm_regions
);
2708 /* We've got to release the dominance info now, to indicate that it
2709 must be rebuilt completely. Otherwise we'll crash trying to update
2710 the SSA web in the TODO section following this pass. */
2711 free_dominance_info (CDI_DOMINATORS
);
2712 bitmap_obstack_release (&tm_obstack
);
2713 all_tm_regions
= NULL
;
2718 struct gimple_opt_pass pass_tm_edges
=
2722 "tmedge", /* name */
2724 execute_tm_edges
, /* execute */
2727 0, /* static_pass_number */
2728 TV_TRANS_MEM
, /* tv_id */
2729 PROP_ssa
| PROP_cfg
, /* properties_required */
2730 0, /* properties_provided */
2731 0, /* properties_destroyed */
2732 0, /* todo_flags_start */
2735 | TODO_dump_func
, /* todo_flags_finish */
2739 /* A unique TM memory operation. */
2740 typedef struct tm_memop
2742 /* Unique ID that all memory operations to the same location have. */
2743 unsigned int value_id
;
2744 /* Address of load/store. */
2748 /* Sets for solving data flow equations in the memory optimization pass. */
2749 struct tm_memopt_bitmaps
2751 /* Stores available to this BB upon entry. Basically, stores that
2752 dominate this BB. */
2753 bitmap store_avail_in
;
2754 /* Stores available at the end of this BB. */
2755 bitmap store_avail_out
;
2756 bitmap store_antic_in
;
2757 bitmap store_antic_out
;
2758 /* Reads available to this BB upon entry. Basically, reads that
2759 dominate this BB. */
2760 bitmap read_avail_in
;
2761 /* Reads available at the end of this BB. */
2762 bitmap read_avail_out
;
2763 /* Reads performed in this BB. */
2765 /* Writes performed in this BB. */
2768 /* Temporary storage for pass. */
2769 /* Is the current BB in the worklist? */
2770 bool avail_in_worklist_p
;
2771 /* Have we visited this BB? */
2775 static bitmap_obstack tm_memopt_obstack
;
2777 /* Unique counter for TM loads and stores. Loads and stores of the
2778 same address get the same ID. */
2779 static unsigned int tm_memopt_value_id
;
2780 static htab_t tm_memopt_value_numbers
;
2782 #define STORE_AVAIL_IN(BB) \
2783 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in
2784 #define STORE_AVAIL_OUT(BB) \
2785 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out
2786 #define STORE_ANTIC_IN(BB) \
2787 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in
2788 #define STORE_ANTIC_OUT(BB) \
2789 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out
2790 #define READ_AVAIL_IN(BB) \
2791 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in
2792 #define READ_AVAIL_OUT(BB) \
2793 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out
2794 #define READ_LOCAL(BB) \
2795 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local
2796 #define STORE_LOCAL(BB) \
2797 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local
2798 #define AVAIL_IN_WORKLIST_P(BB) \
2799 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p
2800 #define BB_VISITED_P(BB) \
2801 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p
2803 /* Htab support. Return a hash value for a `tm_memop'. */
2805 tm_memop_hash (const void *p
)
2807 const struct tm_memop
*mem
= (const struct tm_memop
*) p
;
2808 tree addr
= mem
->addr
;
2809 /* We drill down to the SSA_NAME/DECL for the hash, but equality is
2810 actually done with operand_equal_p (see tm_memop_eq). */
2811 if (TREE_CODE (addr
) == ADDR_EXPR
)
2812 addr
= TREE_OPERAND (addr
, 0);
2813 return iterative_hash_expr (addr
, 0);
2816 /* Htab support. Return true if two tm_memop's are the same. */
2818 tm_memop_eq (const void *p1
, const void *p2
)
2820 const struct tm_memop
*mem1
= (const struct tm_memop
*) p1
;
2821 const struct tm_memop
*mem2
= (const struct tm_memop
*) p2
;
2823 return operand_equal_p (mem1
->addr
, mem2
->addr
, 0);
2826 /* Given a TM load/store in STMT, return the value number for the address
2830 tm_memopt_value_number (gimple stmt
, enum insert_option op
)
2832 struct tm_memop tmpmem
, *mem
;
2835 gcc_assert (is_tm_load (stmt
) || is_tm_store (stmt
));
2836 tmpmem
.addr
= gimple_call_arg (stmt
, 0);
2837 slot
= htab_find_slot (tm_memopt_value_numbers
, &tmpmem
, op
);
2839 mem
= (struct tm_memop
*) *slot
;
2840 else if (op
== INSERT
)
2842 mem
= XNEW (struct tm_memop
);
2844 mem
->value_id
= tm_memopt_value_id
++;
2845 mem
->addr
= tmpmem
.addr
;
2849 return mem
->value_id
;
2852 /* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */
2855 tm_memopt_accumulate_memops (basic_block bb
)
2857 gimple_stmt_iterator gsi
;
2859 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
2861 gimple stmt
= gsi_stmt (gsi
);
2865 if (is_tm_store (stmt
))
2866 bits
= STORE_LOCAL (bb
);
2867 else if (is_tm_load (stmt
))
2868 bits
= READ_LOCAL (bb
);
2872 loc
= tm_memopt_value_number (stmt
, INSERT
);
2873 bitmap_set_bit (bits
, loc
);
2876 fprintf (dump_file
, "TM memopt (%s): value num=%d, BB=%d, addr=",
2877 is_tm_load (stmt
) ? "LOAD" : "STORE", loc
,
2878 gimple_bb (stmt
)->index
);
2879 print_generic_expr (dump_file
, gimple_call_arg (stmt
, 0), 0);
2880 fprintf (dump_file
, "\n");
2885 /* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */
2888 dump_tm_memopt_set (const char *set_name
, bitmap bits
)
2892 const char *comma
= "";
2894 fprintf (dump_file
, "TM memopt: %s: [", set_name
);
2895 EXECUTE_IF_SET_IN_BITMAP (bits
, 0, i
, bi
)
2898 struct tm_memop
*mem
;
2900 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */
2901 FOR_EACH_HTAB_ELEMENT (tm_memopt_value_numbers
, mem
, tm_memop_t
, hi
)
2902 if (mem
->value_id
== i
)
2904 gcc_assert (mem
->value_id
== i
);
2905 fprintf (dump_file
, "%s", comma
);
2907 print_generic_expr (dump_file
, mem
->addr
, 0);
2909 fprintf (dump_file
, "]\n");
2912 /* Prettily dump all of the memopt sets in BLOCKS. */
2915 dump_tm_memopt_sets (VEC (basic_block
, heap
) *blocks
)
2920 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
2922 fprintf (dump_file
, "------------BB %d---------\n", bb
->index
);
2923 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb
));
2924 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb
));
2925 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb
));
2926 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb
));
2927 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb
));
2928 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb
));
2932 /* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */
2935 tm_memopt_compute_avin (basic_block bb
)
2940 /* Seed with the AVOUT of any predecessor. */
2941 for (ix
= 0; ix
< EDGE_COUNT (bb
->preds
); ix
++)
2943 e
= EDGE_PRED (bb
, ix
);
2944 /* Make sure we have already visited this BB, and is thus
2947 If e->src->aux is NULL, this predecessor is actually on an
2948 enclosing transaction. We only care about the current
2949 transaction, so ignore it. */
2950 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
2952 bitmap_copy (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
2953 bitmap_copy (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
2958 for (; ix
< EDGE_COUNT (bb
->preds
); ix
++)
2960 e
= EDGE_PRED (bb
, ix
);
2961 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
2963 bitmap_and_into (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
2964 bitmap_and_into (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
2968 BB_VISITED_P (bb
) = true;
2971 /* Compute the STORE_ANTIC_IN for the basic block BB. */
2974 tm_memopt_compute_antin (basic_block bb
)
2979 /* Seed with the ANTIC_OUT of any successor. */
2980 for (ix
= 0; ix
< EDGE_COUNT (bb
->succs
); ix
++)
2982 e
= EDGE_SUCC (bb
, ix
);
2983 /* Make sure we have already visited this BB, and is thus
2985 if (BB_VISITED_P (e
->dest
))
2987 bitmap_copy (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
2992 for (; ix
< EDGE_COUNT (bb
->succs
); ix
++)
2994 e
= EDGE_SUCC (bb
, ix
);
2995 if (BB_VISITED_P (e
->dest
))
2996 bitmap_and_into (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
2999 BB_VISITED_P (bb
) = true;
3002 /* Compute the AVAIL sets for every basic block in BLOCKS.
3004 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows:
3006 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb])
3007 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors])
3009 This is basically what we do in lcm's compute_available(), but here
3010 we calculate two sets of sets (one for STOREs and one for READs),
3011 and we work on a region instead of the entire CFG.
3013 REGION is the TM region.
3014 BLOCKS are the basic blocks in the region. */
3017 tm_memopt_compute_available (struct tm_region
*region
,
3018 VEC (basic_block
, heap
) *blocks
)
3021 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3022 unsigned int qlen
, i
;
3026 /* Allocate a worklist array/queue. Entries are only added to the
3027 list if they were not already on the list. So the size is
3028 bounded by the number of basic blocks in the region. */
3029 qlen
= VEC_length (basic_block
, blocks
) - 1;
3030 qin
= qout
= worklist
=
3031 XNEWVEC (basic_block
, qlen
);
3033 /* Put every block in the region on the worklist. */
3034 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
3036 /* Seed AVAIL_OUT with the LOCAL set. */
3037 bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_LOCAL (bb
));
3038 bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_LOCAL (bb
));
3040 AVAIL_IN_WORKLIST_P (bb
) = true;
3041 /* No need to insert the entry block, since it has an AVIN of
3042 null, and an AVOUT that has already been seeded in. */
3043 if (bb
!= region
->entry_block
)
3047 /* The entry block has been initialized with the local sets. */
3048 BB_VISITED_P (region
->entry_block
) = true;
3051 qend
= &worklist
[qlen
];
3053 /* Iterate until the worklist is empty. */
3056 /* Take the first entry off the worklist. */
3063 /* This block can be added to the worklist again if necessary. */
3064 AVAIL_IN_WORKLIST_P (bb
) = false;
3065 tm_memopt_compute_avin (bb
);
3067 /* Note: We do not add the LOCAL sets here because we already
3068 seeded the AVAIL_OUT sets with them. */
3069 changed
= bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_AVAIL_IN (bb
));
3070 changed
|= bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_AVAIL_IN (bb
));
3072 && (region
->exit_blocks
== NULL
3073 || !bitmap_bit_p (region
->exit_blocks
, bb
->index
)))
3074 /* If the out state of this block changed, then we need to add
3075 its successors to the worklist if they are not already in. */
3076 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3077 if (!AVAIL_IN_WORKLIST_P (e
->dest
) && e
->dest
!= EXIT_BLOCK_PTR
)
3080 AVAIL_IN_WORKLIST_P (e
->dest
) = true;
3091 dump_tm_memopt_sets (blocks
);
3094 /* Compute ANTIC sets for every basic block in BLOCKS.
3096 We compute STORE_ANTIC_OUT as follows:
3098 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb])
3099 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors])
3101 REGION is the TM region.
3102 BLOCKS are the basic blocks in the region. */
3105 tm_memopt_compute_antic (struct tm_region
*region
,
3106 VEC (basic_block
, heap
) *blocks
)
3109 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3114 /* Allocate a worklist array/queue. Entries are only added to the
3115 list if they were not already on the list. So the size is
3116 bounded by the number of basic blocks in the region. */
3117 qin
= qout
= worklist
=
3118 XNEWVEC (basic_block
, VEC_length (basic_block
, blocks
));
3120 for (qlen
= 0, i
= VEC_length (basic_block
, blocks
) - 1; i
>= 0; --i
)
3122 bb
= VEC_index (basic_block
, blocks
, i
);
3124 /* Seed ANTIC_OUT with the LOCAL set. */
3125 bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_LOCAL (bb
));
3127 /* Put every block in the region on the worklist. */
3128 AVAIL_IN_WORKLIST_P (bb
) = true;
3129 /* No need to insert exit blocks, since their ANTIC_IN is NULL,
3130 and their ANTIC_OUT has already been seeded in. */
3131 if (region
->exit_blocks
3132 && !bitmap_bit_p (region
->exit_blocks
, bb
->index
))
3139 /* The exit blocks have been initialized with the local sets. */
3140 if (region
->exit_blocks
)
3144 EXECUTE_IF_SET_IN_BITMAP (region
->exit_blocks
, 0, i
, bi
)
3145 BB_VISITED_P (BASIC_BLOCK (i
)) = true;
3149 qend
= &worklist
[qlen
];
3151 /* Iterate until the worklist is empty. */
3154 /* Take the first entry off the worklist. */
3161 /* This block can be added to the worklist again if necessary. */
3162 AVAIL_IN_WORKLIST_P (bb
) = false;
3163 tm_memopt_compute_antin (bb
);
3165 /* Note: We do not add the LOCAL sets here because we already
3166 seeded the ANTIC_OUT sets with them. */
3167 if (bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_ANTIC_IN (bb
))
3168 && bb
!= region
->entry_block
)
3169 /* If the out state of this block changed, then we need to add
3170 its predecessors to the worklist if they are not already in. */
3171 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
3172 if (!AVAIL_IN_WORKLIST_P (e
->src
))
3175 AVAIL_IN_WORKLIST_P (e
->src
) = true;
3186 dump_tm_memopt_sets (blocks
);
3189 /* Offsets of load variants from TM_LOAD. For example,
3190 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*.
3191 See gtm-builtins.def. */
3192 #define TRANSFORM_RAR 1
3193 #define TRANSFORM_RAW 2
3194 #define TRANSFORM_RFW 3
3195 /* Offsets of store variants from TM_STORE. */
3196 #define TRANSFORM_WAR 1
3197 #define TRANSFORM_WAW 2
3199 /* Inform about a load/store optimization. */
3202 dump_tm_memopt_transform (gimple stmt
)
3206 fprintf (dump_file
, "TM memopt: transforming: ");
3207 print_gimple_stmt (dump_file
, stmt
, 0, 0);
3208 fprintf (dump_file
, "\n");
3212 /* Perform a read/write optimization. Replaces the TM builtin in STMT
3213 by a builtin that is OFFSET entries down in the builtins table in
3214 gtm-builtins.def. */
3217 tm_memopt_transform_stmt (unsigned int offset
,
3219 gimple_stmt_iterator
*gsi
)
3221 tree fn
= gimple_call_fn (stmt
);
3222 gcc_assert (TREE_CODE (fn
) == ADDR_EXPR
);
3223 TREE_OPERAND (fn
, 0)
3224 = builtin_decl_explicit ((enum built_in_function
)
3225 (DECL_FUNCTION_CODE (TREE_OPERAND (fn
, 0))
3227 gimple_call_set_fn (stmt
, fn
);
3228 gsi_replace (gsi
, stmt
, true);
3229 dump_tm_memopt_transform (stmt
);
3232 /* Perform the actual TM memory optimization transformations in the
3233 basic blocks in BLOCKS. */
3236 tm_memopt_transform_blocks (VEC (basic_block
, heap
) *blocks
)
3240 gimple_stmt_iterator gsi
;
3242 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
3244 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3246 gimple stmt
= gsi_stmt (gsi
);
3247 bitmap read_avail
= READ_AVAIL_IN (bb
);
3248 bitmap store_avail
= STORE_AVAIL_IN (bb
);
3249 bitmap store_antic
= STORE_ANTIC_OUT (bb
);
3252 if (is_tm_simple_load (stmt
))
3254 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3255 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3256 tm_memopt_transform_stmt (TRANSFORM_RAW
, stmt
, &gsi
);
3257 else if (store_antic
&& bitmap_bit_p (store_antic
, loc
))
3259 tm_memopt_transform_stmt (TRANSFORM_RFW
, stmt
, &gsi
);
3260 bitmap_set_bit (store_avail
, loc
);
3262 else if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3263 tm_memopt_transform_stmt (TRANSFORM_RAR
, stmt
, &gsi
);
3265 bitmap_set_bit (read_avail
, loc
);
3267 else if (is_tm_simple_store (stmt
))
3269 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3270 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3271 tm_memopt_transform_stmt (TRANSFORM_WAW
, stmt
, &gsi
);
3274 if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3275 tm_memopt_transform_stmt (TRANSFORM_WAR
, stmt
, &gsi
);
3276 bitmap_set_bit (store_avail
, loc
);
3283 /* Return a new set of bitmaps for a BB. */
3285 static struct tm_memopt_bitmaps
*
3286 tm_memopt_init_sets (void)
3288 struct tm_memopt_bitmaps
*b
3289 = XOBNEW (&tm_memopt_obstack
.obstack
, struct tm_memopt_bitmaps
);
3290 b
->store_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3291 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3292 b
->store_antic_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3293 b
->store_antic_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3294 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3295 b
->read_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3296 b
->read_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3297 b
->read_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3298 b
->store_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3302 /* Free sets computed for each BB. */
3305 tm_memopt_free_sets (VEC (basic_block
, heap
) *blocks
)
3310 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
3314 /* Clear the visited bit for every basic block in BLOCKS. */
3317 tm_memopt_clear_visited (VEC (basic_block
, heap
) *blocks
)
3322 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
3323 BB_VISITED_P (bb
) = false;
3326 /* Replace TM load/stores with hints for the runtime. We handle
3327 things like read-after-write, write-after-read, read-after-read,
3328 read-for-write, etc. */
3331 execute_tm_memopt (void)
3333 struct tm_region
*region
;
3334 VEC (basic_block
, heap
) *bbs
;
3336 tm_memopt_value_id
= 0;
3337 tm_memopt_value_numbers
= htab_create (10, tm_memop_hash
, tm_memop_eq
, free
);
3339 for (region
= all_tm_regions
; region
; region
= region
->next
)
3341 /* All the TM stores/loads in the current region. */
3345 bitmap_obstack_initialize (&tm_memopt_obstack
);
3347 /* Save all BBs for the current region. */
3348 bbs
= get_tm_region_blocks (region
->entry_block
,
3349 region
->exit_blocks
,
3354 /* Collect all the memory operations. */
3355 for (i
= 0; VEC_iterate (basic_block
, bbs
, i
, bb
); ++i
)
3357 bb
->aux
= tm_memopt_init_sets ();
3358 tm_memopt_accumulate_memops (bb
);
3361 /* Solve data flow equations and transform each block accordingly. */
3362 tm_memopt_clear_visited (bbs
);
3363 tm_memopt_compute_available (region
, bbs
);
3364 tm_memopt_clear_visited (bbs
);
3365 tm_memopt_compute_antic (region
, bbs
);
3366 tm_memopt_transform_blocks (bbs
);
3368 tm_memopt_free_sets (bbs
);
3369 VEC_free (basic_block
, heap
, bbs
);
3370 bitmap_obstack_release (&tm_memopt_obstack
);
3371 htab_empty (tm_memopt_value_numbers
);
3374 htab_delete (tm_memopt_value_numbers
);
3379 gate_tm_memopt (void)
3381 return flag_tm
&& optimize
> 0;
3384 struct gimple_opt_pass pass_tm_memopt
=
3388 "tmmemopt", /* name */
3389 gate_tm_memopt
, /* gate */
3390 execute_tm_memopt
, /* execute */
3393 0, /* static_pass_number */
3394 TV_TRANS_MEM
, /* tv_id */
3395 PROP_ssa
| PROP_cfg
, /* properties_required */
3396 0, /* properties_provided */
3397 0, /* properties_destroyed */
3398 0, /* todo_flags_start */
3399 TODO_dump_func
, /* todo_flags_finish */
3404 /* Interprocedual analysis for the creation of transactional clones.
3405 The aim of this pass is to find which functions are referenced in
3406 a non-irrevocable transaction context, and for those over which
3407 we have control (or user directive), create a version of the
3408 function which uses only the transactional interface to reference
3409 protected memories. This analysis proceeds in several steps:
3411 (1) Collect the set of all possible transactional clones:
3413 (a) For all local public functions marked tm_callable, push
3414 it onto the tm_callee queue.
3416 (b) For all local functions, scan for calls in transaction blocks.
3417 Push the caller and callee onto the tm_caller and tm_callee
3418 queues. Count the number of callers for each callee.
3420 (c) For each local function on the callee list, assume we will
3421 create a transactional clone. Push *all* calls onto the
3422 callee queues; count the number of clone callers separately
3423 to the number of original callers.
3425 (2) Propagate irrevocable status up the dominator tree:
3427 (a) Any external function on the callee list that is not marked
3428 tm_callable is irrevocable. Push all callers of such onto
3431 (b) For each function on the worklist, mark each block that
3432 contains an irrevocable call. Use the AND operator to
3433 propagate that mark up the dominator tree.
3435 (c) If we reach the entry block for a possible transactional
3436 clone, then the transactional clone is irrevocable, and
3437 we should not create the clone after all. Push all
3438 callers onto the worklist.
3440 (d) Place tm_irrevocable calls at the beginning of the relevant
3441 blocks. Special case here is the entry block for the entire
3442 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for
3443 the library to begin the region in serial mode. Decrement
3444 the call count for all callees in the irrevocable region.
3446 (3) Create the transactional clones:
3448 Any tm_callee that still has a non-zero call count is cloned.
3451 /* This structure is stored in the AUX field of each cgraph_node. */
3452 struct tm_ipa_cg_data
3454 /* The clone of the function that got created. */
3455 struct cgraph_node
*clone
;
3457 /* The tm regions in the normal function. */
3458 struct tm_region
*all_tm_regions
;
3460 /* The blocks of the normal/clone functions that contain irrevocable
3461 calls, or blocks that are post-dominated by irrevocable calls. */
3462 bitmap irrevocable_blocks_normal
;
3463 bitmap irrevocable_blocks_clone
;
3465 /* The blocks of the normal function that are involved in transactions. */
3466 bitmap transaction_blocks_normal
;
3468 /* The number of callers to the transactional clone of this function
3469 from normal and transactional clones respectively. */
3470 unsigned tm_callers_normal
;
3471 unsigned tm_callers_clone
;
3473 /* True if all calls to this function's transactional clone
3474 are irrevocable. Also automatically true if the function
3475 has no transactional clone. */
3476 bool is_irrevocable
;
3478 /* Flags indicating the presence of this function in various queues. */
3479 bool in_callee_queue
;
3482 /* Flags indicating the kind of scan desired while in the worklist. */
3483 bool want_irr_scan_normal
;
3486 typedef struct cgraph_node
*cgraph_node_p
;
3488 DEF_VEC_P (cgraph_node_p
);
3489 DEF_VEC_ALLOC_P (cgraph_node_p
, heap
);
3491 typedef VEC (cgraph_node_p
, heap
) *cgraph_node_queue
;
3493 /* Return the ipa data associated with NODE, allocating zeroed memory
3496 static struct tm_ipa_cg_data
*
3497 get_cg_data (struct cgraph_node
*node
)
3499 struct tm_ipa_cg_data
*d
= (struct tm_ipa_cg_data
*) node
->aux
;
3503 d
= (struct tm_ipa_cg_data
*)
3504 obstack_alloc (&tm_obstack
.obstack
, sizeof (*d
));
3505 node
->aux
= (void *) d
;
3506 memset (d
, 0, sizeof (*d
));
3512 /* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that
3513 it is already present. */
3516 maybe_push_queue (struct cgraph_node
*node
,
3517 cgraph_node_queue
*queue_p
, bool *in_queue_p
)
3522 VEC_safe_push (cgraph_node_p
, heap
, *queue_p
, node
);
3526 /* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone.
3527 Queue all callees within block BB. */
3530 ipa_tm_scan_calls_block (cgraph_node_queue
*callees_p
,
3531 basic_block bb
, bool for_clone
)
3533 gimple_stmt_iterator gsi
;
3535 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3537 gimple stmt
= gsi_stmt (gsi
);
3538 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
3540 tree fndecl
= gimple_call_fndecl (stmt
);
3543 struct tm_ipa_cg_data
*d
;
3545 struct cgraph_node
*node
;
3547 if (is_tm_ending_fndecl (fndecl
))
3549 if (find_tm_replacement_function (fndecl
))
3552 node
= cgraph_get_node (fndecl
);
3553 gcc_assert (node
!= NULL
);
3554 d
= get_cg_data (node
);
3556 pcallers
= (for_clone
? &d
->tm_callers_clone
3557 : &d
->tm_callers_normal
);
3560 maybe_push_queue (node
, callees_p
, &d
->in_callee_queue
);
3566 /* Scan all calls in NODE that are within a transaction region,
3567 and push the resulting nodes into the callee queue. */
3570 ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data
*d
,
3571 cgraph_node_queue
*callees_p
)
3573 struct tm_region
*r
;
3575 d
->transaction_blocks_normal
= BITMAP_ALLOC (&tm_obstack
);
3576 d
->all_tm_regions
= all_tm_regions
;
3578 for (r
= all_tm_regions
; r
; r
= r
->next
)
3580 VEC (basic_block
, heap
) *bbs
;
3584 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
, NULL
,
3585 d
->transaction_blocks_normal
, false);
3587 FOR_EACH_VEC_ELT (basic_block
, bbs
, i
, bb
)
3588 ipa_tm_scan_calls_block (callees_p
, bb
, false);
3590 VEC_free (basic_block
, heap
, bbs
);
3594 /* Scan all calls in NODE as if this is the transactional clone,
3595 and push the destinations into the callee queue. */
3598 ipa_tm_scan_calls_clone (struct cgraph_node
*node
,
3599 cgraph_node_queue
*callees_p
)
3601 struct function
*fn
= DECL_STRUCT_FUNCTION (node
->decl
);
3604 FOR_EACH_BB_FN (bb
, fn
)
3605 ipa_tm_scan_calls_block (callees_p
, bb
, true);
3608 /* The function NODE has been detected to be irrevocable. Push all
3609 of its callers onto WORKLIST for the purpose of re-scanning them. */
3612 ipa_tm_note_irrevocable (struct cgraph_node
*node
,
3613 cgraph_node_queue
*worklist_p
)
3615 struct tm_ipa_cg_data
*d
= get_cg_data (node
);
3616 struct cgraph_edge
*e
;
3618 d
->is_irrevocable
= true;
3620 for (e
= node
->callers
; e
; e
= e
->next_caller
)
3624 /* Don't examine recursive calls. */
3625 if (e
->caller
== node
)
3627 /* Even if we think we can go irrevocable, believe the user
3629 if (is_tm_safe_or_pure (e
->caller
->decl
))
3632 d
= get_cg_data (e
->caller
);
3634 /* Check if the callee is in a transactional region. If so,
3635 schedule the function for normal re-scan as well. */
3636 bb
= gimple_bb (e
->call_stmt
);
3637 gcc_assert (bb
!= NULL
);
3638 if (d
->transaction_blocks_normal
3639 && bitmap_bit_p (d
->transaction_blocks_normal
, bb
->index
))
3640 d
->want_irr_scan_normal
= true;
3642 maybe_push_queue (e
->caller
, worklist_p
, &d
->in_worklist
);
3646 /* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement
3647 within the block is irrevocable. */
3650 ipa_tm_scan_irr_block (basic_block bb
)
3652 gimple_stmt_iterator gsi
;
3655 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3657 gimple stmt
= gsi_stmt (gsi
);
3658 switch (gimple_code (stmt
))
3661 if (is_tm_pure_call (stmt
))
3664 fn
= gimple_call_fn (stmt
);
3666 /* Functions with the attribute are by definition irrevocable. */
3667 if (is_tm_irrevocable (fn
))
3670 /* For direct function calls, go ahead and check for replacement
3671 functions, or transitive irrevocable functions. For indirect
3672 functions, we'll ask the runtime. */
3673 if (TREE_CODE (fn
) == ADDR_EXPR
)
3675 struct tm_ipa_cg_data
*d
;
3677 fn
= TREE_OPERAND (fn
, 0);
3678 if (is_tm_ending_fndecl (fn
))
3680 if (find_tm_replacement_function (fn
))
3683 d
= get_cg_data (cgraph_get_node (fn
));
3684 if (d
->is_irrevocable
)
3690 /* ??? The Approved Method of indicating that an inline
3691 assembly statement is not relevant to the transaction
3692 is to wrap it in a __tm_waiver block. This is not
3693 yet implemented, so we can't check for it. */
3704 /* For each of the blocks seeded witin PQUEUE, walk the CFG looking
3705 for new irrevocable blocks, marking them in NEW_IRR. Don't bother
3706 scanning past OLD_IRR or EXIT_BLOCKS. */
3709 ipa_tm_scan_irr_blocks (VEC (basic_block
, heap
) **pqueue
, bitmap new_irr
,
3710 bitmap old_irr
, bitmap exit_blocks
)
3712 bool any_new_irr
= false;
3715 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
3719 basic_block bb
= VEC_pop (basic_block
, *pqueue
);
3721 /* Don't re-scan blocks we know already are irrevocable. */
3722 if (old_irr
&& bitmap_bit_p (old_irr
, bb
->index
))
3725 if (ipa_tm_scan_irr_block (bb
))
3727 bitmap_set_bit (new_irr
, bb
->index
);
3730 else if (exit_blocks
== NULL
|| !bitmap_bit_p (exit_blocks
, bb
->index
))
3732 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3733 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
3735 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
3736 VEC_safe_push (basic_block
, heap
, *pqueue
, e
->dest
);
3740 while (!VEC_empty (basic_block
, *pqueue
));
3742 BITMAP_FREE (visited_blocks
);
3747 /* Propagate the irrevocable property both up and down the dominator tree.
3748 BB is the current block being scanned; EXIT_BLOCKS are the edges of the
3749 TM regions; OLD_IRR are the results of a previous scan of the dominator
3750 tree which has been fully propagated; NEW_IRR is the set of new blocks
3751 which are gaining the irrevocable property during the current scan. */
3754 ipa_tm_propagate_irr (basic_block entry_block
, bitmap new_irr
,
3755 bitmap old_irr
, bitmap exit_blocks
)
3757 VEC (basic_block
, heap
) *bbs
;
3758 bitmap all_region_blocks
;
3760 /* If this block is in the old set, no need to rescan. */
3761 if (old_irr
&& bitmap_bit_p (old_irr
, entry_block
->index
))
3764 all_region_blocks
= BITMAP_ALLOC (&tm_obstack
);
3765 bbs
= get_tm_region_blocks (entry_block
, exit_blocks
, NULL
,
3766 all_region_blocks
, false);
3769 basic_block bb
= VEC_pop (basic_block
, bbs
);
3770 bool this_irr
= bitmap_bit_p (new_irr
, bb
->index
);
3771 bool all_son_irr
= false;
3775 /* Propagate up. If my children are, I am too, but we must have
3776 at least one child that is. */
3779 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3781 if (!bitmap_bit_p (new_irr
, e
->dest
->index
))
3783 all_son_irr
= false;
3791 /* Add block to new_irr if it hasn't already been processed. */
3792 if (!old_irr
|| !bitmap_bit_p (old_irr
, bb
->index
))
3794 bitmap_set_bit (new_irr
, bb
->index
);
3800 /* Propagate down to everyone we immediately dominate. */
3804 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
3806 son
= next_dom_son (CDI_DOMINATORS
, son
))
3808 /* Make sure block is actually in a TM region, and it
3809 isn't already in old_irr. */
3810 if ((!old_irr
|| !bitmap_bit_p (old_irr
, son
->index
))
3811 && bitmap_bit_p (all_region_blocks
, son
->index
))
3812 bitmap_set_bit (new_irr
, son
->index
);
3816 while (!VEC_empty (basic_block
, bbs
));
3818 BITMAP_FREE (all_region_blocks
);
3819 VEC_free (basic_block
, heap
, bbs
);
3823 ipa_tm_decrement_clone_counts (basic_block bb
, bool for_clone
)
3825 gimple_stmt_iterator gsi
;
3827 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3829 gimple stmt
= gsi_stmt (gsi
);
3830 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
3832 tree fndecl
= gimple_call_fndecl (stmt
);
3835 struct tm_ipa_cg_data
*d
;
3838 if (is_tm_ending_fndecl (fndecl
))
3840 if (find_tm_replacement_function (fndecl
))
3843 d
= get_cg_data (cgraph_get_node (fndecl
));
3844 pcallers
= (for_clone
? &d
->tm_callers_clone
3845 : &d
->tm_callers_normal
);
3847 gcc_assert (*pcallers
> 0);
3854 /* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions,
3855 as well as other irrevocable actions such as inline assembly. Mark all
3856 such blocks as irrevocable and decrement the number of calls to
3857 transactional clones. Return true if, for the transactional clone, the
3858 entire function is irrevocable. */
3861 ipa_tm_scan_irr_function (struct cgraph_node
*node
, bool for_clone
)
3863 struct tm_ipa_cg_data
*d
;
3864 bitmap new_irr
, old_irr
;
3865 VEC (basic_block
, heap
) *queue
;
3868 current_function_decl
= node
->decl
;
3869 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
3870 calculate_dominance_info (CDI_DOMINATORS
);
3872 d
= get_cg_data (node
);
3873 queue
= VEC_alloc (basic_block
, heap
, 10);
3874 new_irr
= BITMAP_ALLOC (&tm_obstack
);
3876 /* Scan each tm region, propagating irrevocable status through the tree. */
3879 old_irr
= d
->irrevocable_blocks_clone
;
3880 VEC_quick_push (basic_block
, queue
, single_succ (ENTRY_BLOCK_PTR
));
3881 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
, NULL
))
3883 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR
), new_irr
,
3885 ret
= bitmap_bit_p (new_irr
, single_succ (ENTRY_BLOCK_PTR
)->index
);
3890 struct tm_region
*region
;
3892 old_irr
= d
->irrevocable_blocks_normal
;
3893 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
3895 VEC_quick_push (basic_block
, queue
, region
->entry_block
);
3896 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
,
3897 region
->exit_blocks
))
3898 ipa_tm_propagate_irr (region
->entry_block
, new_irr
, old_irr
,
3899 region
->exit_blocks
);
3903 /* If we found any new irrevocable blocks, reduce the call count for
3904 transactional clones within the irrevocable blocks. Save the new
3905 set of irrevocable blocks for next time. */
3906 if (!bitmap_empty_p (new_irr
))
3908 bitmap_iterator bmi
;
3911 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
3912 ipa_tm_decrement_clone_counts (BASIC_BLOCK (i
), for_clone
);
3916 bitmap_ior_into (old_irr
, new_irr
);
3917 BITMAP_FREE (new_irr
);
3920 d
->irrevocable_blocks_clone
= new_irr
;
3922 d
->irrevocable_blocks_normal
= new_irr
;
3924 if (dump_file
&& new_irr
)
3927 bitmap_iterator bmi
;
3930 dname
= lang_hooks
.decl_printable_name (current_function_decl
, 2);
3931 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
3932 fprintf (dump_file
, "%s: bb %d goes irrevocable\n", dname
, i
);
3936 BITMAP_FREE (new_irr
);
3938 VEC_free (basic_block
, heap
, queue
);
3940 current_function_decl
= NULL
;
3945 /* Return true if, for the transactional clone of NODE, any call
3946 may enter irrevocable mode. */
3949 ipa_tm_mayenterirr_function (struct cgraph_node
*node
)
3951 struct tm_ipa_cg_data
*d
= get_cg_data (node
);
3952 tree decl
= node
->decl
;
3953 unsigned flags
= flags_from_decl_or_type (decl
);
3955 /* Handle some TM builtins. Ordinarily these aren't actually generated
3956 at this point, but handling these functions when written in by the
3957 user makes it easier to build unit tests. */
3958 if (flags
& ECF_TM_BUILTIN
)
3961 /* Filter out all functions that are marked. */
3962 if (flags
& ECF_TM_PURE
)
3964 if (is_tm_safe (decl
))
3966 if (is_tm_irrevocable (decl
))
3968 if (is_tm_callable (decl
))
3970 if (find_tm_replacement_function (decl
))
3973 /* If we aren't seeing the final version of the function we don't
3974 know what it will contain at runtime. */
3975 if (cgraph_function_body_availability (node
) < AVAIL_AVAILABLE
)
3978 /* If the function must go irrevocable, then of course true. */
3979 if (d
->is_irrevocable
)
3982 /* If there are any blocks marked irrevocable, then the function
3983 as a whole may enter irrevocable. */
3984 if (d
->irrevocable_blocks_clone
)
3987 /* We may have previously marked this function as tm_may_enter_irr;
3988 see pass_diagnose_tm_blocks. */
3989 if (node
->local
.tm_may_enter_irr
)
3992 /* Recurse on the main body for aliases. In general, this will
3993 result in one of the bits above being set so that we will not
3994 have to recurse next time. */
3996 return ipa_tm_mayenterirr_function (cgraph_get_node (node
->thunk
.alias
));
3998 /* What remains is unmarked local functions without items that force
3999 the function to go irrevocable. */
4003 /* Diagnose calls from transaction_safe functions to unmarked
4004 functions that are determined to not be safe. */
4007 ipa_tm_diagnose_tm_safe (struct cgraph_node
*node
)
4009 struct cgraph_edge
*e
;
4011 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4012 if (!is_tm_callable (e
->callee
->decl
)
4013 && e
->callee
->local
.tm_may_enter_irr
)
4014 error_at (gimple_location (e
->call_stmt
),
4015 "unsafe function call %qD within "
4016 "%<transaction_safe%> function", e
->callee
->decl
);
4019 /* Diagnose call from atomic transactions to unmarked functions
4020 that are determined to not be safe. */
4023 ipa_tm_diagnose_transaction (struct cgraph_node
*node
,
4024 struct tm_region
*all_tm_regions
)
4026 struct tm_region
*r
;
4028 for (r
= all_tm_regions
; r
; r
= r
->next
)
4029 if (gimple_transaction_subcode (r
->transaction_stmt
) & GTMA_IS_RELAXED
)
4031 /* Atomic transactions can be nested inside relaxed. */
4033 ipa_tm_diagnose_transaction (node
, r
->inner
);
4037 VEC (basic_block
, heap
) *bbs
;
4038 gimple_stmt_iterator gsi
;
4042 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
,
4043 r
->irr_blocks
, NULL
, false);
4045 for (i
= 0; VEC_iterate (basic_block
, bbs
, i
, bb
); ++i
)
4046 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4048 gimple stmt
= gsi_stmt (gsi
);
4051 if (gimple_code (stmt
) == GIMPLE_ASM
)
4053 error_at (gimple_location (stmt
),
4054 "asm not allowed in atomic transaction");
4058 if (!is_gimple_call (stmt
))
4060 fndecl
= gimple_call_fndecl (stmt
);
4062 /* Indirect function calls have been diagnosed already. */
4066 /* Stop at the end of the transaction. */
4067 if (is_tm_ending_fndecl (fndecl
))
4069 if (bitmap_bit_p (r
->exit_blocks
, bb
->index
))
4074 /* Marked functions have been diagnosed already. */
4075 if (is_tm_pure_call (stmt
))
4077 if (is_tm_callable (fndecl
))
4080 if (cgraph_local_info (fndecl
)->tm_may_enter_irr
)
4081 error_at (gimple_location (stmt
),
4082 "unsafe function call %qD within "
4083 "atomic transaction", fndecl
);
4086 VEC_free (basic_block
, heap
, bbs
);
4090 /* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in
4091 OLD_DECL. The returned value is a freshly malloced pointer that
4092 should be freed by the caller. */
4095 tm_mangle (tree old_asm_id
)
4097 const char *old_asm_name
;
4100 struct demangle_component
*dc
;
4103 /* Determine if the symbol is already a valid C++ mangled name. Do this
4104 even for C, which might be interfacing with C++ code via appropriately
4105 ugly identifiers. */
4106 /* ??? We could probably do just as well checking for "_Z" and be done. */
4107 old_asm_name
= IDENTIFIER_POINTER (old_asm_id
);
4108 dc
= cplus_demangle_v3_components (old_asm_name
, DMGL_NO_OPTS
, &alloc
);
4115 sprintf (length
, "%u", IDENTIFIER_LENGTH (old_asm_id
));
4116 tm_name
= concat ("_ZGTt", length
, old_asm_name
, NULL
);
4120 old_asm_name
+= 2; /* Skip _Z */
4124 case DEMANGLE_COMPONENT_TRANSACTION_CLONE
:
4125 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE
:
4126 /* Don't play silly games, you! */
4129 case DEMANGLE_COMPONENT_HIDDEN_ALIAS
:
4130 /* I'd really like to know if we can ever be passed one of
4131 these from the C++ front end. The Logical Thing would
4132 seem that hidden-alias should be outer-most, so that we
4133 get hidden-alias of a transaction-clone and not vice-versa. */
4141 tm_name
= concat ("_ZGTt", old_asm_name
, NULL
);
4145 new_asm_id
= get_identifier (tm_name
);
4152 ipa_tm_mark_needed_node (struct cgraph_node
*node
)
4154 cgraph_mark_needed_node (node
);
4155 /* ??? function_and_variable_visibility will reset
4156 the needed bit, without actually checking. */
4160 /* Callback data for ipa_tm_create_version_alias. */
4161 struct create_version_alias_info
4163 struct cgraph_node
*old_node
;
4167 /* A subrontine of ipa_tm_create_version, called via
4168 cgraph_for_node_and_aliases. Create new tm clones for each of
4169 the existing aliases. */
4171 ipa_tm_create_version_alias (struct cgraph_node
*node
, void *data
)
4173 struct create_version_alias_info
*info
4174 = (struct create_version_alias_info
*)data
;
4175 tree old_decl
, new_decl
, tm_name
;
4176 struct cgraph_node
*new_node
;
4178 if (!node
->same_body_alias
)
4181 old_decl
= node
->decl
;
4182 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4183 new_decl
= build_decl (DECL_SOURCE_LOCATION (old_decl
),
4184 TREE_CODE (old_decl
), tm_name
,
4185 TREE_TYPE (old_decl
));
4187 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4188 SET_DECL_RTL (new_decl
, NULL
);
4190 /* Based loosely on C++'s make_alias_for(). */
4191 TREE_PUBLIC (new_decl
) = TREE_PUBLIC (old_decl
);
4192 DECL_CONTEXT (new_decl
) = NULL
;
4193 TREE_READONLY (new_decl
) = TREE_READONLY (old_decl
);
4194 DECL_EXTERNAL (new_decl
) = 0;
4195 DECL_ARTIFICIAL (new_decl
) = 1;
4196 TREE_ADDRESSABLE (new_decl
) = 1;
4197 TREE_USED (new_decl
) = 1;
4198 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4200 /* Perform the same remapping to the comdat group. */
4201 if (DECL_COMDAT (new_decl
))
4202 DECL_COMDAT_GROUP (new_decl
) = tm_mangle (DECL_COMDAT_GROUP (old_decl
));
4204 new_node
= cgraph_same_body_alias (NULL
, new_decl
, info
->new_decl
);
4205 new_node
->tm_clone
= true;
4206 get_cg_data (node
)->clone
= new_node
;
4208 record_tm_clone_pair (old_decl
, new_decl
);
4210 if (info
->old_node
->needed
)
4211 ipa_tm_mark_needed_node (new_node
);
4215 /* Create a copy of the function (possibly declaration only) of OLD_NODE,
4216 appropriate for the transactional clone. */
4219 ipa_tm_create_version (struct cgraph_node
*old_node
)
4221 tree new_decl
, old_decl
, tm_name
;
4222 struct cgraph_node
*new_node
;
4224 old_decl
= old_node
->decl
;
4225 new_decl
= copy_node (old_decl
);
4227 /* DECL_ASSEMBLER_NAME needs to be set before we call
4228 cgraph_copy_node_for_versioning below, because cgraph_node will
4229 fill the assembler_name_hash. */
4230 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4231 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4232 SET_DECL_RTL (new_decl
, NULL
);
4233 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4235 /* Perform the same remapping to the comdat group. */
4236 if (DECL_COMDAT (new_decl
))
4237 DECL_COMDAT_GROUP (new_decl
) = tm_mangle (DECL_COMDAT_GROUP (old_decl
));
4239 new_node
= cgraph_copy_node_for_versioning (old_node
, new_decl
, NULL
, NULL
);
4240 new_node
->lowered
= true;
4241 new_node
->tm_clone
= 1;
4242 get_cg_data (old_node
)->clone
= new_node
;
4244 if (cgraph_function_body_availability (old_node
) >= AVAIL_OVERWRITABLE
)
4246 /* Remap extern inline to static inline. */
4247 /* ??? Is it worth trying to use make_decl_one_only? */
4248 if (DECL_DECLARED_INLINE_P (new_decl
) && DECL_EXTERNAL (new_decl
))
4250 DECL_EXTERNAL (new_decl
) = 0;
4251 TREE_PUBLIC (new_decl
) = 0;
4254 tree_function_versioning (old_decl
, new_decl
, NULL
, false, NULL
,
4258 record_tm_clone_pair (old_decl
, new_decl
);
4260 cgraph_call_function_insertion_hooks (new_node
);
4261 if (old_node
->needed
)
4262 ipa_tm_mark_needed_node (new_node
);
4264 /* Do the same thing, but for any aliases of the original node. */
4266 struct create_version_alias_info data
;
4267 data
.old_node
= old_node
;
4268 data
.new_decl
= new_decl
;
4269 cgraph_for_node_and_aliases (old_node
, ipa_tm_create_version_alias
,
4274 /* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */
4277 ipa_tm_insert_irr_call (struct cgraph_node
*node
, struct tm_region
*region
,
4280 gimple_stmt_iterator gsi
;
4283 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4285 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
),
4286 1, build_int_cst (NULL_TREE
, MODE_SERIALIRREVOCABLE
));
4288 split_block_after_labels (bb
);
4289 gsi
= gsi_after_labels (bb
);
4290 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
4292 cgraph_create_edge (node
,
4293 cgraph_get_create_node
4294 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
)),
4296 compute_call_stmt_bb_frequency (node
->decl
,
4300 /* Construct a call to TM_GETTMCLONE and insert it before GSI. */
4303 ipa_tm_insert_gettmclone_call (struct cgraph_node
*node
,
4304 struct tm_region
*region
,
4305 gimple_stmt_iterator
*gsi
, gimple stmt
)
4307 tree gettm_fn
, ret
, old_fn
, callfn
;
4311 old_fn
= gimple_call_fn (stmt
);
4313 if (TREE_CODE (old_fn
) == ADDR_EXPR
)
4315 tree fndecl
= TREE_OPERAND (old_fn
, 0);
4316 tree clone
= get_tm_clone_pair (fndecl
);
4318 /* By transforming the call into a TM_GETTMCLONE, we are
4319 technically taking the address of the original function and
4320 its clone. Explain this so inlining will know this function
4322 cgraph_mark_address_taken_node (cgraph_get_node (fndecl
));
4324 cgraph_mark_address_taken_node (cgraph_get_node (clone
));
4327 safe
= is_tm_safe (TREE_TYPE (old_fn
));
4328 gettm_fn
= builtin_decl_explicit (safe
? BUILT_IN_TM_GETTMCLONE_SAFE
4329 : BUILT_IN_TM_GETTMCLONE_IRR
);
4330 ret
= create_tmp_var (ptr_type_node
, NULL
);
4331 add_referenced_var (ret
);
4334 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4336 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */
4337 if (TREE_CODE (old_fn
) == OBJ_TYPE_REF
)
4338 old_fn
= OBJ_TYPE_REF_EXPR (old_fn
);
4340 g
= gimple_build_call (gettm_fn
, 1, old_fn
);
4341 ret
= make_ssa_name (ret
, g
);
4342 gimple_call_set_lhs (g
, ret
);
4344 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
4346 cgraph_create_edge (node
, cgraph_get_create_node (gettm_fn
), g
, 0,
4347 compute_call_stmt_bb_frequency (node
->decl
,
4350 /* Cast return value from tm_gettmclone* into appropriate function
4352 callfn
= create_tmp_var (TREE_TYPE (old_fn
), NULL
);
4353 add_referenced_var (callfn
);
4354 g2
= gimple_build_assign (callfn
,
4355 fold_build1 (NOP_EXPR
, TREE_TYPE (callfn
), ret
));
4356 callfn
= make_ssa_name (callfn
, g2
);
4357 gimple_assign_set_lhs (g2
, callfn
);
4358 gsi_insert_before (gsi
, g2
, GSI_SAME_STMT
);
4360 /* ??? This is a hack to preserve the NOTHROW bit on the call,
4361 which we would have derived from the decl. Failure to save
4362 this bit means we might have to split the basic block. */
4363 if (gimple_call_nothrow_p (stmt
))
4364 gimple_call_set_nothrow (stmt
, true);
4366 gimple_call_set_fn (stmt
, callfn
);
4368 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS
4369 for a call statement. Fix it. */
4371 tree lhs
= gimple_call_lhs (stmt
);
4372 tree rettype
= TREE_TYPE (gimple_call_fntype (stmt
));
4374 && !useless_type_conversion_p (TREE_TYPE (lhs
), rettype
))
4378 temp
= make_rename_temp (rettype
, 0);
4379 gimple_call_set_lhs (stmt
, temp
);
4381 g2
= gimple_build_assign (lhs
,
4382 fold_build1 (VIEW_CONVERT_EXPR
,
4383 TREE_TYPE (lhs
), temp
));
4384 gsi_insert_after (gsi
, g2
, GSI_SAME_STMT
);
4393 /* Helper function for ipa_tm_transform_calls*. Given a call
4394 statement in GSI which resides inside transaction REGION, redirect
4395 the call to either its wrapper function, or its clone. */
4398 ipa_tm_transform_calls_redirect (struct cgraph_node
*node
,
4399 struct tm_region
*region
,
4400 gimple_stmt_iterator
*gsi
,
4401 bool *need_ssa_rename_p
)
4403 gimple stmt
= gsi_stmt (*gsi
);
4404 struct cgraph_node
*new_node
;
4405 struct cgraph_edge
*e
= cgraph_edge (node
, stmt
);
4406 tree fndecl
= gimple_call_fndecl (stmt
);
4408 /* For indirect calls, pass the address through the runtime. */
4411 *need_ssa_rename_p
|=
4412 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
4416 /* Handle some TM builtins. Ordinarily these aren't actually generated
4417 at this point, but handling these functions when written in by the
4418 user makes it easier to build unit tests. */
4419 if (flags_from_decl_or_type (fndecl
) & ECF_TM_BUILTIN
)
4422 /* Fixup recursive calls inside clones. */
4423 /* ??? Why did cgraph_copy_node_for_versioning update the call edges
4424 for recursion but not update the call statements themselves? */
4425 if (e
->caller
== e
->callee
&& decl_is_tm_clone (current_function_decl
))
4427 gimple_call_set_fndecl (stmt
, current_function_decl
);
4431 /* If there is a replacement, use it. */
4432 fndecl
= find_tm_replacement_function (fndecl
);
4435 new_node
= cgraph_get_create_node (fndecl
);
4437 /* ??? Mark all transaction_wrap functions tm_may_enter_irr.
4439 We can't do this earlier in record_tm_replacement because
4440 cgraph_remove_unreachable_nodes is called before we inject
4441 references to the node. Further, we can't do this in some
4442 nice central place in ipa_tm_execute because we don't have
4443 the exact list of wrapper functions that would be used.
4444 Marking more wrappers than necessary results in the creation
4445 of unnecessary cgraph_nodes, which can cause some of the
4446 other IPA passes to crash.
4448 We do need to mark these nodes so that we get the proper
4449 result in expand_call_tm. */
4450 /* ??? This seems broken. How is it that we're marking the
4451 CALLEE as may_enter_irr? Surely we should be marking the
4452 CALLER. Also note that find_tm_replacement_function also
4453 contains mappings into the TM runtime, e.g. memcpy. These
4454 we know won't go irrevocable. */
4455 new_node
->local
.tm_may_enter_irr
= 1;
4459 struct tm_ipa_cg_data
*d
= get_cg_data (e
->callee
);
4460 new_node
= d
->clone
;
4462 /* As we've already skipped pure calls and appropriate builtins,
4463 and we've already marked irrevocable blocks, if we can't come
4464 up with a static replacement, then ask the runtime. */
4465 if (new_node
== NULL
)
4467 *need_ssa_rename_p
|=
4468 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
4469 cgraph_remove_edge (e
);
4473 fndecl
= new_node
->decl
;
4476 cgraph_redirect_edge_callee (e
, new_node
);
4477 gimple_call_set_fndecl (stmt
, fndecl
);
4480 /* Helper function for ipa_tm_transform_calls. For a given BB,
4481 install calls to tm_irrevocable when IRR_BLOCKS are reached,
4482 redirect other calls to the generated transactional clone. */
4485 ipa_tm_transform_calls_1 (struct cgraph_node
*node
, struct tm_region
*region
,
4486 basic_block bb
, bitmap irr_blocks
)
4488 gimple_stmt_iterator gsi
;
4489 bool need_ssa_rename
= false;
4491 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
4493 ipa_tm_insert_irr_call (node
, region
, bb
);
4497 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4499 gimple stmt
= gsi_stmt (gsi
);
4501 if (!is_gimple_call (stmt
))
4503 if (is_tm_pure_call (stmt
))
4506 /* Redirect edges to the appropriate replacement or clone. */
4507 ipa_tm_transform_calls_redirect (node
, region
, &gsi
, &need_ssa_rename
);
4510 return need_ssa_rename
;
4513 /* Walk the CFG for REGION, beginning at BB. Install calls to
4514 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to
4515 the generated transactional clone. */
4518 ipa_tm_transform_calls (struct cgraph_node
*node
, struct tm_region
*region
,
4519 basic_block bb
, bitmap irr_blocks
)
4521 bool need_ssa_rename
= false;
4524 VEC(basic_block
, heap
) *queue
= NULL
;
4525 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4527 VEC_safe_push (basic_block
, heap
, queue
, bb
);
4530 bb
= VEC_pop (basic_block
, queue
);
4533 ipa_tm_transform_calls_1 (node
, region
, bb
, irr_blocks
);
4535 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
4538 if (region
&& bitmap_bit_p (region
->exit_blocks
, bb
->index
))
4541 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4542 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
4544 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
4545 VEC_safe_push (basic_block
, heap
, queue
, e
->dest
);
4548 while (!VEC_empty (basic_block
, queue
));
4550 VEC_free (basic_block
, heap
, queue
);
4551 BITMAP_FREE (visited_blocks
);
4553 return need_ssa_rename
;
4556 /* Transform the calls within the TM regions within NODE. */
4559 ipa_tm_transform_transaction (struct cgraph_node
*node
)
4561 struct tm_ipa_cg_data
*d
= get_cg_data (node
);
4562 struct tm_region
*region
;
4563 bool need_ssa_rename
= false;
4565 current_function_decl
= node
->decl
;
4566 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
4567 calculate_dominance_info (CDI_DOMINATORS
);
4569 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
4571 /* If we're sure to go irrevocable, don't transform anything. */
4572 if (d
->irrevocable_blocks_normal
4573 && bitmap_bit_p (d
->irrevocable_blocks_normal
,
4574 region
->entry_block
->index
))
4576 transaction_subcode_ior (region
, GTMA_DOES_GO_IRREVOCABLE
);
4577 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4582 ipa_tm_transform_calls (node
, region
, region
->entry_block
,
4583 d
->irrevocable_blocks_normal
);
4586 if (need_ssa_rename
)
4587 update_ssa (TODO_update_ssa_only_virtuals
);
4590 current_function_decl
= NULL
;
4593 /* Transform the calls within the transactional clone of NODE. */
4596 ipa_tm_transform_clone (struct cgraph_node
*node
)
4598 struct tm_ipa_cg_data
*d
= get_cg_data (node
);
4599 bool need_ssa_rename
;
4601 /* If this function makes no calls and has no irrevocable blocks,
4602 then there's nothing to do. */
4603 /* ??? Remove non-aborting top-level transactions. */
4604 if (!node
->callees
&& !d
->irrevocable_blocks_clone
)
4607 current_function_decl
= d
->clone
->decl
;
4608 push_cfun (DECL_STRUCT_FUNCTION (current_function_decl
));
4609 calculate_dominance_info (CDI_DOMINATORS
);
4612 ipa_tm_transform_calls (d
->clone
, NULL
, single_succ (ENTRY_BLOCK_PTR
),
4613 d
->irrevocable_blocks_clone
);
4615 if (need_ssa_rename
)
4616 update_ssa (TODO_update_ssa_only_virtuals
);
4619 current_function_decl
= NULL
;
4622 /* Main entry point for the transactional memory IPA pass. */
4625 ipa_tm_execute (void)
4627 cgraph_node_queue tm_callees
= NULL
;
4628 /* List of functions that will go irrevocable. */
4629 cgraph_node_queue irr_worklist
= NULL
;
4631 struct cgraph_node
*node
;
4632 struct tm_ipa_cg_data
*d
;
4633 enum availability a
;
4636 #ifdef ENABLE_CHECKING
4640 bitmap_obstack_initialize (&tm_obstack
);
4642 /* For all local functions marked tm_callable, queue them. */
4643 for (node
= cgraph_nodes
; node
; node
= node
->next
)
4644 if (is_tm_callable (node
->decl
)
4645 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
4647 d
= get_cg_data (node
);
4648 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
4651 /* For all local reachable functions... */
4652 for (node
= cgraph_nodes
; node
; node
= node
->next
)
4653 if (node
->reachable
&& node
->lowered
4654 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
4656 /* ... marked tm_pure, record that fact for the runtime by
4657 indicating that the pure function is its own tm_callable.
4658 No need to do this if the function's address can't be taken. */
4659 if (is_tm_pure (node
->decl
))
4661 if (!node
->local
.local
)
4662 record_tm_clone_pair (node
->decl
, node
->decl
);
4666 current_function_decl
= node
->decl
;
4667 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
4668 calculate_dominance_info (CDI_DOMINATORS
);
4670 tm_region_init (NULL
);
4673 d
= get_cg_data (node
);
4675 /* Scan for calls that are in each transaction. */
4676 ipa_tm_scan_calls_transaction (d
, &tm_callees
);
4678 /* If we saw something that will make us go irrevocable, put it
4679 in the worklist so we can scan the function later
4680 (ipa_tm_scan_irr_function) and mark the irrevocable blocks. */
4681 if (node
->local
.tm_may_enter_irr
)
4683 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
4684 d
->want_irr_scan_normal
= true;
4689 current_function_decl
= NULL
;
4692 /* For every local function on the callee list, scan as if we will be
4693 creating a transactional clone, queueing all new functions we find
4695 for (i
= 0; i
< VEC_length (cgraph_node_p
, tm_callees
); ++i
)
4697 node
= VEC_index (cgraph_node_p
, tm_callees
, i
);
4698 a
= cgraph_function_body_availability (node
);
4699 d
= get_cg_data (node
);
4701 /* If we saw something that will make us go irrevocable, put it
4702 in the worklist so we can scan the function later
4703 (ipa_tm_scan_irr_function) and mark the irrevocable blocks. */
4704 if (node
->local
.tm_may_enter_irr
)
4705 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
4707 /* Some callees cannot be arbitrarily cloned. These will always be
4708 irrevocable. Mark these now, so that we need not scan them. */
4709 if (is_tm_irrevocable (node
->decl
))
4710 ipa_tm_note_irrevocable (node
, &irr_worklist
);
4711 else if (a
<= AVAIL_NOT_AVAILABLE
4712 && !is_tm_safe_or_pure (node
->decl
))
4713 ipa_tm_note_irrevocable (node
, &irr_worklist
);
4714 else if (a
>= AVAIL_OVERWRITABLE
)
4716 if (!tree_versionable_function_p (node
->decl
))
4717 ipa_tm_note_irrevocable (node
, &irr_worklist
);
4718 else if (!d
->is_irrevocable
)
4720 /* If this is an alias, make sure its base is queued as well.
4721 we need not scan the callees now, as the base will do. */
4724 node
= cgraph_get_node (node
->thunk
.alias
);
4725 d
= get_cg_data (node
);
4726 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
4730 /* Add all nodes called by this function into
4731 tm_callees as well. */
4732 ipa_tm_scan_calls_clone (node
, &tm_callees
);
4737 /* Iterate scans until no more work to be done. Prefer not to use
4738 VEC_pop because the worklist tends to follow a breadth-first
4739 search of the callgraph, which should allow convergance with a
4740 minimum number of scans. But we also don't want the worklist
4741 array to grow without bound, so we shift the array up periodically. */
4742 for (i
= 0; i
< VEC_length (cgraph_node_p
, irr_worklist
); ++i
)
4744 if (i
> 256 && i
== VEC_length (cgraph_node_p
, irr_worklist
) / 8)
4746 VEC_block_remove (cgraph_node_p
, irr_worklist
, 0, i
);
4750 node
= VEC_index (cgraph_node_p
, irr_worklist
, i
);
4751 d
= get_cg_data (node
);
4752 d
->in_worklist
= false;
4754 if (d
->want_irr_scan_normal
)
4756 d
->want_irr_scan_normal
= false;
4757 ipa_tm_scan_irr_function (node
, false);
4759 if (d
->in_callee_queue
&& ipa_tm_scan_irr_function (node
, true))
4760 ipa_tm_note_irrevocable (node
, &irr_worklist
);
4763 /* For every function on the callee list, collect the tm_may_enter_irr
4765 VEC_truncate (cgraph_node_p
, irr_worklist
, 0);
4766 for (i
= 0; i
< VEC_length (cgraph_node_p
, tm_callees
); ++i
)
4768 node
= VEC_index (cgraph_node_p
, tm_callees
, i
);
4769 if (ipa_tm_mayenterirr_function (node
))
4771 d
= get_cg_data (node
);
4772 gcc_assert (d
->in_worklist
== false);
4773 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
4777 /* Propagate the tm_may_enter_irr bit to callers until stable. */
4778 for (i
= 0; i
< VEC_length (cgraph_node_p
, irr_worklist
); ++i
)
4780 struct cgraph_node
*caller
;
4781 struct cgraph_edge
*e
;
4782 struct ipa_ref
*ref
;
4785 if (i
> 256 && i
== VEC_length (cgraph_node_p
, irr_worklist
) / 8)
4787 VEC_block_remove (cgraph_node_p
, irr_worklist
, 0, i
);
4791 node
= VEC_index (cgraph_node_p
, irr_worklist
, i
);
4792 d
= get_cg_data (node
);
4793 d
->in_worklist
= false;
4794 node
->local
.tm_may_enter_irr
= true;
4796 /* Propagate back to normal callers. */
4797 for (e
= node
->callers
; e
; e
= e
->next_caller
)
4800 if (!is_tm_safe_or_pure (caller
->decl
)
4801 && !caller
->local
.tm_may_enter_irr
)
4803 d
= get_cg_data (caller
);
4804 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
4808 /* Propagate back to referring aliases as well. */
4809 for (j
= 0; ipa_ref_list_refering_iterate (&node
->ref_list
, j
, ref
); j
++)
4811 caller
= ref
->refering
.cgraph_node
;
4812 if (ref
->use
== IPA_REF_ALIAS
4813 && !caller
->local
.tm_may_enter_irr
)
4815 d
= get_cg_data (caller
);
4816 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
4821 /* Now validate all tm_safe functions, and all atomic regions in
4823 for (node
= cgraph_nodes
; node
; node
= node
->next
)
4824 if (node
->reachable
&& node
->lowered
4825 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
4827 d
= get_cg_data (node
);
4828 if (is_tm_safe (node
->decl
))
4829 ipa_tm_diagnose_tm_safe (node
);
4830 else if (d
->all_tm_regions
)
4831 ipa_tm_diagnose_transaction (node
, d
->all_tm_regions
);
4834 /* Create clones. Do those that are not irrevocable and have a
4835 positive call count. Do those publicly visible functions that
4836 the user directed us to clone. */
4837 for (i
= 0; i
< VEC_length (cgraph_node_p
, tm_callees
); ++i
)
4841 node
= VEC_index (cgraph_node_p
, tm_callees
, i
);
4842 if (node
->same_body_alias
)
4845 a
= cgraph_function_body_availability (node
);
4846 d
= get_cg_data (node
);
4848 if (a
<= AVAIL_NOT_AVAILABLE
)
4849 doit
= is_tm_callable (node
->decl
);
4850 else if (a
<= AVAIL_AVAILABLE
&& is_tm_callable (node
->decl
))
4852 else if (!d
->is_irrevocable
4853 && d
->tm_callers_normal
+ d
->tm_callers_clone
> 0)
4857 ipa_tm_create_version (node
);
4860 /* Redirect calls to the new clones, and insert irrevocable marks. */
4861 for (i
= 0; i
< VEC_length (cgraph_node_p
, tm_callees
); ++i
)
4863 node
= VEC_index (cgraph_node_p
, tm_callees
, i
);
4866 d
= get_cg_data (node
);
4868 ipa_tm_transform_clone (node
);
4871 for (node
= cgraph_nodes
; node
; node
= node
->next
)
4872 if (node
->reachable
&& node
->lowered
4873 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
4875 d
= get_cg_data (node
);
4876 if (d
->all_tm_regions
)
4877 ipa_tm_transform_transaction (node
);
4880 /* Free and clear all data structures. */
4881 VEC_free (cgraph_node_p
, heap
, tm_callees
);
4882 VEC_free (cgraph_node_p
, heap
, irr_worklist
);
4883 bitmap_obstack_release (&tm_obstack
);
4885 for (node
= cgraph_nodes
; node
; node
= node
->next
)
4888 #ifdef ENABLE_CHECKING
4895 struct simple_ipa_opt_pass pass_ipa_tm
=
4901 ipa_tm_execute
, /* execute */
4904 0, /* static_pass_number */
4905 TV_TRANS_MEM
, /* tv_id */
4906 PROP_ssa
| PROP_cfg
, /* properties_required */
4907 0, /* properties_provided */
4908 0, /* properties_destroyed */
4909 0, /* todo_flags_start */
4910 TODO_dump_func
, /* todo_flags_finish */
4914 #include "gt-trans-mem.h"