1 /* Target code for NVPTX.
2 Copyright (C) 2014-2016 Free Software Foundation, Inc.
3 Contributed by Bernd Schmidt <bernds@codesourcery.com>
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published
9 by the Free Software Foundation; either version 3, or (at your
10 option) any later version.
12 GCC is distributed in the hope that it will be useful, but WITHOUT
13 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14 or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
15 License for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
37 #include "diagnostic.h"
39 #include "insn-flags.h"
41 #include "insn-attr.h"
50 #include "tm-constrs.h"
51 #include "langhooks.h"
55 #include "stor-layout.h"
58 #include "gomp-constants.h"
60 #include "internal-fn.h"
61 #include "gimple-iterator.h"
62 #include "stringpool.h"
64 #include "tree-ssa-operands.h"
65 #include "tree-ssanames.h"
67 #include "tree-phinodes.h"
69 #include "fold-const.h"
71 /* This file should be included last. */
72 #include "target-def.h"
74 /* The kind of shuffe instruction. */
75 enum nvptx_shuffle_kind
84 /* The various PTX memory areas an object might reside in. */
96 /* We record the data area in the target symbol flags. */
97 #define SYMBOL_DATA_AREA(SYM) \
98 (nvptx_data_area)((SYMBOL_REF_FLAGS (SYM) >> SYMBOL_FLAG_MACH_DEP_SHIFT) \
100 #define SET_SYMBOL_DATA_AREA(SYM,AREA) \
101 (SYMBOL_REF_FLAGS (SYM) |= (AREA) << SYMBOL_FLAG_MACH_DEP_SHIFT)
103 /* Record the function decls we've written, and the libfuncs and function
104 decls corresponding to them. */
105 static std::stringstream func_decls
;
107 struct declared_libfunc_hasher
: ggc_cache_ptr_hash
<rtx_def
>
109 static hashval_t
hash (rtx x
) { return htab_hash_pointer (x
); }
110 static bool equal (rtx a
, rtx b
) { return a
== b
; }
114 hash_table
<declared_libfunc_hasher
> *declared_libfuncs_htab
;
116 struct tree_hasher
: ggc_cache_ptr_hash
<tree_node
>
118 static hashval_t
hash (tree t
) { return htab_hash_pointer (t
); }
119 static bool equal (tree a
, tree b
) { return a
== b
; }
122 static GTY((cache
)) hash_table
<tree_hasher
> *declared_fndecls_htab
;
123 static GTY((cache
)) hash_table
<tree_hasher
> *needed_fndecls_htab
;
125 /* Buffer needed to broadcast across workers. This is used for both
126 worker-neutering and worker broadcasting. It is shared by all
127 functions emitted. The buffer is placed in shared memory. It'd be
128 nice if PTX supported common blocks, because then this could be
129 shared across TUs (taking the largest size). */
130 static unsigned worker_bcast_size
;
131 static unsigned worker_bcast_align
;
132 static GTY(()) rtx worker_bcast_sym
;
134 /* Buffer needed for worker reductions. This has to be distinct from
135 the worker broadcast array, as both may be live concurrently. */
136 static unsigned worker_red_size
;
137 static unsigned worker_red_align
;
138 static GTY(()) rtx worker_red_sym
;
140 /* Global lock variable, needed for 128bit worker & gang reductions. */
141 static GTY(()) tree global_lock_var
;
143 /* Allocate a new, cleared machine_function structure. */
145 static struct machine_function
*
146 nvptx_init_machine_status (void)
148 struct machine_function
*p
= ggc_cleared_alloc
<machine_function
> ();
149 p
->return_mode
= VOIDmode
;
153 /* Implement TARGET_OPTION_OVERRIDE. */
156 nvptx_option_override (void)
158 init_machine_status
= nvptx_init_machine_status
;
160 /* Set toplevel_reorder, unless explicitly disabled. We need
161 reordering so that we emit necessary assembler decls of
162 undeclared variables. */
163 if (!global_options_set
.x_flag_toplevel_reorder
)
164 flag_toplevel_reorder
= 1;
166 /* Set flag_no_common, unless explicitly disabled. We fake common
167 using .weak, and that's not entirely accurate, so avoid it
169 if (!global_options_set
.x_flag_no_common
)
172 /* Assumes that it will see only hard registers. */
173 flag_var_tracking
= 0;
175 if (nvptx_optimize
< 0)
176 nvptx_optimize
= optimize
> 0;
178 declared_fndecls_htab
= hash_table
<tree_hasher
>::create_ggc (17);
179 needed_fndecls_htab
= hash_table
<tree_hasher
>::create_ggc (17);
180 declared_libfuncs_htab
181 = hash_table
<declared_libfunc_hasher
>::create_ggc (17);
183 worker_bcast_sym
= gen_rtx_SYMBOL_REF (Pmode
, "__worker_bcast");
184 SET_SYMBOL_DATA_AREA (worker_bcast_sym
, DATA_AREA_SHARED
);
185 worker_bcast_align
= GET_MODE_ALIGNMENT (SImode
) / BITS_PER_UNIT
;
187 worker_red_sym
= gen_rtx_SYMBOL_REF (Pmode
, "__worker_red");
188 SET_SYMBOL_DATA_AREA (worker_red_sym
, DATA_AREA_SHARED
);
189 worker_red_align
= GET_MODE_ALIGNMENT (SImode
) / BITS_PER_UNIT
;
192 /* Return a ptx type for MODE. If PROMOTE, then use .u32 for QImode to
193 deal with ptx ideosyncracies. */
196 nvptx_ptx_type_from_mode (machine_mode mode
, bool promote
)
226 /* Encode the PTX data area that DECL (which might not actually be a
227 _DECL) should reside in. */
230 nvptx_encode_section_info (tree decl
, rtx rtl
, int first
)
232 default_encode_section_info (decl
, rtl
, first
);
233 if (first
&& MEM_P (rtl
))
235 nvptx_data_area area
= DATA_AREA_GENERIC
;
237 if (TREE_CONSTANT (decl
))
238 area
= DATA_AREA_CONST
;
239 else if (TREE_CODE (decl
) == VAR_DECL
)
240 /* TODO: This would be a good place to check for a .shared or
241 other section name. */
242 area
= TREE_READONLY (decl
) ? DATA_AREA_CONST
: DATA_AREA_GLOBAL
;
244 SET_SYMBOL_DATA_AREA (XEXP (rtl
, 0), area
);
248 /* Return the PTX name of the data area in which SYM should be
249 placed. The symbol must have already been processed by
250 nvptx_encode_seciton_info, or equivalent. */
253 section_for_sym (rtx sym
)
255 nvptx_data_area area
= SYMBOL_DATA_AREA (sym
);
256 /* Same order as nvptx_data_area enum. */
257 static char const *const areas
[] =
258 {"", ".global", ".shared", ".local", ".const", ".param"};
263 /* Similarly for a decl. */
266 section_for_decl (const_tree decl
)
268 return section_for_sym (XEXP (DECL_RTL (CONST_CAST (tree
, decl
)), 0));
271 /* Check NAME for special function names and redirect them by returning a
272 replacement. This applies to malloc, free and realloc, for which we
273 want to use libgcc wrappers, and call, which triggers a bug in
274 ptxas. We can't use TARGET_MANGLE_DECL_ASSEMBLER_NAME, as that's
275 not active in an offload compiler -- the names are all set by the
276 host-side compiler. */
279 nvptx_name_replacement (const char *name
)
281 if (strcmp (name
, "call") == 0)
282 return "__nvptx_call";
283 if (strcmp (name
, "malloc") == 0)
284 return "__nvptx_malloc";
285 if (strcmp (name
, "free") == 0)
286 return "__nvptx_free";
287 if (strcmp (name
, "realloc") == 0)
288 return "__nvptx_realloc";
292 /* If MODE should be treated as two registers of an inner mode, return
293 that inner mode. Otherwise return VOIDmode. */
296 maybe_split_mode (machine_mode mode
)
298 if (COMPLEX_MODE_P (mode
))
299 return GET_MODE_INNER (mode
);
307 /* Output a register, subreg, or register pair (with optional
308 enclosing braces). */
311 output_reg (FILE *file
, unsigned regno
, machine_mode inner_mode
,
312 int subreg_offset
= -1)
314 if (inner_mode
== VOIDmode
)
316 if (HARD_REGISTER_NUM_P (regno
))
317 fprintf (file
, "%s", reg_names
[regno
]);
319 fprintf (file
, "%%r%d", regno
);
321 else if (subreg_offset
>= 0)
323 output_reg (file
, regno
, VOIDmode
);
324 fprintf (file
, "$%d", subreg_offset
);
328 if (subreg_offset
== -1)
330 output_reg (file
, regno
, inner_mode
, GET_MODE_SIZE (inner_mode
));
332 output_reg (file
, regno
, inner_mode
, 0);
333 if (subreg_offset
== -1)
338 /* Emit forking instructions for MASK. */
341 nvptx_emit_forking (unsigned mask
, bool is_call
)
343 mask
&= (GOMP_DIM_MASK (GOMP_DIM_WORKER
)
344 | GOMP_DIM_MASK (GOMP_DIM_VECTOR
));
347 rtx op
= GEN_INT (mask
| (is_call
<< GOMP_DIM_MAX
));
349 /* Emit fork at all levels. This helps form SESE regions, as
350 it creates a block with a single successor before entering a
351 partitooned region. That is a good candidate for the end of
354 emit_insn (gen_nvptx_fork (op
));
355 emit_insn (gen_nvptx_forked (op
));
359 /* Emit joining instructions for MASK. */
362 nvptx_emit_joining (unsigned mask
, bool is_call
)
364 mask
&= (GOMP_DIM_MASK (GOMP_DIM_WORKER
)
365 | GOMP_DIM_MASK (GOMP_DIM_VECTOR
));
368 rtx op
= GEN_INT (mask
| (is_call
<< GOMP_DIM_MAX
));
370 /* Emit joining for all non-call pars to ensure there's a single
371 predecessor for the block the join insn ends up in. This is
372 needed for skipping entire loops. */
374 emit_insn (gen_nvptx_joining (op
));
375 emit_insn (gen_nvptx_join (op
));
380 /* Determine whether MODE and TYPE (possibly NULL) should be passed or
381 returned in memory. Integer and floating types supported by the
382 machine are passed in registers, everything else is passed in
383 memory. Complex types are split. */
386 pass_in_memory (machine_mode mode
, const_tree type
, bool for_return
)
390 if (AGGREGATE_TYPE_P (type
))
392 if (TREE_CODE (type
) == VECTOR_TYPE
)
396 if (!for_return
&& COMPLEX_MODE_P (mode
))
397 /* Complex types are passed as two underlying args. */
398 mode
= GET_MODE_INNER (mode
);
400 if (GET_MODE_CLASS (mode
) != MODE_INT
401 && GET_MODE_CLASS (mode
) != MODE_FLOAT
)
404 if (GET_MODE_SIZE (mode
) > UNITS_PER_WORD
)
410 /* A non-memory argument of mode MODE is being passed, determine the mode it
411 should be promoted to. This is also used for determining return
415 promote_arg (machine_mode mode
, bool prototyped
)
417 if (!prototyped
&& mode
== SFmode
)
418 /* K&R float promotion for unprototyped functions. */
420 else if (GET_MODE_SIZE (mode
) < GET_MODE_SIZE (SImode
))
426 /* A non-memory return type of MODE is being returned. Determine the
427 mode it should be promoted to. */
430 promote_return (machine_mode mode
)
432 return promote_arg (mode
, true);
435 /* Implement TARGET_FUNCTION_ARG. */
438 nvptx_function_arg (cumulative_args_t
ARG_UNUSED (cum_v
), machine_mode mode
,
439 const_tree
, bool named
)
441 if (mode
== VOIDmode
|| !named
)
444 return gen_reg_rtx (mode
);
447 /* Implement TARGET_FUNCTION_INCOMING_ARG. */
450 nvptx_function_incoming_arg (cumulative_args_t cum_v
, machine_mode mode
,
451 const_tree
, bool named
)
453 CUMULATIVE_ARGS
*cum
= get_cumulative_args (cum_v
);
455 if (mode
== VOIDmode
|| !named
)
458 /* No need to deal with split modes here, the only case that can
459 happen is complex modes and those are dealt with by
460 TARGET_SPLIT_COMPLEX_ARG. */
461 return gen_rtx_UNSPEC (mode
,
462 gen_rtvec (1, GEN_INT (cum
->count
)),
466 /* Implement TARGET_FUNCTION_ARG_ADVANCE. */
469 nvptx_function_arg_advance (cumulative_args_t cum_v
,
470 machine_mode
ARG_UNUSED (mode
),
471 const_tree
ARG_UNUSED (type
),
472 bool ARG_UNUSED (named
))
474 CUMULATIVE_ARGS
*cum
= get_cumulative_args (cum_v
);
479 /* Implement TARGET_FUNCTION_ARG_BOUNDARY.
481 For nvptx This is only used for varadic args. The type has already
482 been promoted and/or converted to invisible reference. */
485 nvptx_function_arg_boundary (machine_mode mode
, const_tree
ARG_UNUSED (type
))
487 return GET_MODE_ALIGNMENT (mode
);
490 /* Handle the TARGET_STRICT_ARGUMENT_NAMING target hook.
492 For nvptx, we know how to handle functions declared as stdarg: by
493 passing an extra pointer to the unnamed arguments. However, the
494 Fortran frontend can produce a different situation, where a
495 function pointer is declared with no arguments, but the actual
496 function and calls to it take more arguments. In that case, we
497 want to ensure the call matches the definition of the function. */
500 nvptx_strict_argument_naming (cumulative_args_t cum_v
)
502 CUMULATIVE_ARGS
*cum
= get_cumulative_args (cum_v
);
504 return cum
->fntype
== NULL_TREE
|| stdarg_p (cum
->fntype
);
507 /* Implement TARGET_LIBCALL_VALUE. */
510 nvptx_libcall_value (machine_mode mode
, const_rtx
)
512 if (!cfun
|| !cfun
->machine
->doing_call
)
513 /* Pretend to return in a hard reg for early uses before pseudos can be
515 return gen_rtx_REG (mode
, NVPTX_RETURN_REGNUM
);
517 return gen_reg_rtx (mode
);
520 /* TARGET_FUNCTION_VALUE implementation. Returns an RTX representing the place
521 where function FUNC returns or receives a value of data type TYPE. */
524 nvptx_function_value (const_tree type
, const_tree
ARG_UNUSED (func
),
527 machine_mode mode
= promote_return (TYPE_MODE (type
));
532 cfun
->machine
->return_mode
= mode
;
533 return gen_rtx_REG (mode
, NVPTX_RETURN_REGNUM
);
536 return nvptx_libcall_value (mode
, NULL_RTX
);
539 /* Implement TARGET_FUNCTION_VALUE_REGNO_P. */
542 nvptx_function_value_regno_p (const unsigned int regno
)
544 return regno
== NVPTX_RETURN_REGNUM
;
547 /* Types with a mode other than those supported by the machine are passed by
548 reference in memory. */
551 nvptx_pass_by_reference (cumulative_args_t
ARG_UNUSED (cum
),
552 machine_mode mode
, const_tree type
,
553 bool ARG_UNUSED (named
))
555 return pass_in_memory (mode
, type
, false);
558 /* Implement TARGET_RETURN_IN_MEMORY. */
561 nvptx_return_in_memory (const_tree type
, const_tree
)
563 return pass_in_memory (TYPE_MODE (type
), type
, true);
566 /* Implement TARGET_PROMOTE_FUNCTION_MODE. */
569 nvptx_promote_function_mode (const_tree type
, machine_mode mode
,
570 int *ARG_UNUSED (punsignedp
),
571 const_tree funtype
, int for_return
)
573 return promote_arg (mode
, for_return
|| !type
|| TYPE_ARG_TYPES (funtype
));
576 /* Helper for write_arg. Emit a single PTX argument of MODE, either
577 in a prototype, or as copy in a function prologue. ARGNO is the
578 index of this argument in the PTX function. FOR_REG is negative,
579 if we're emitting the PTX prototype. It is zero if we're copying
580 to an argument register and it is greater than zero if we're
581 copying to a specific hard register. */
584 write_arg_mode (std::stringstream
&s
, int for_reg
, int argno
,
587 const char *ptx_type
= nvptx_ptx_type_from_mode (mode
, false);
591 /* Writing PTX prototype. */
592 s
<< (argno
? ", " : " (");
593 s
<< ".param" << ptx_type
<< " %in_ar" << argno
;
597 s
<< "\t.reg" << ptx_type
<< " ";
599 s
<< reg_names
[for_reg
];
605 s
<< "\tld.param" << ptx_type
<< " ";
607 s
<< reg_names
[for_reg
];
610 s
<< ", [%in_ar" << argno
<< "];\n";
616 /* Process function parameter TYPE to emit one or more PTX
617 arguments. S, FOR_REG and ARGNO as for write_arg_mode. PROTOTYPED
618 is true, if this is a prototyped function, rather than an old-style
619 C declaration. Returns the next argument number to use.
621 The promotion behavior here must match the regular GCC function
622 parameter marshalling machinery. */
625 write_arg_type (std::stringstream
&s
, int for_reg
, int argno
,
626 tree type
, bool prototyped
)
628 machine_mode mode
= TYPE_MODE (type
);
630 if (mode
== VOIDmode
)
633 if (pass_in_memory (mode
, type
, false))
637 bool split
= TREE_CODE (type
) == COMPLEX_TYPE
;
641 /* Complex types are sent as two separate args. */
642 type
= TREE_TYPE (type
);
643 mode
= TYPE_MODE (type
);
647 mode
= promote_arg (mode
, prototyped
);
649 argno
= write_arg_mode (s
, for_reg
, argno
, mode
);
652 return write_arg_mode (s
, for_reg
, argno
, mode
);
655 /* Emit a PTX return as a prototype or function prologue declaration
659 write_return_mode (std::stringstream
&s
, bool for_proto
, machine_mode mode
)
661 const char *ptx_type
= nvptx_ptx_type_from_mode (mode
, false);
662 const char *pfx
= "\t.reg";
663 const char *sfx
= ";\n";
666 pfx
= "(.param", sfx
= "_out) ";
668 s
<< pfx
<< ptx_type
<< " " << reg_names
[NVPTX_RETURN_REGNUM
] << sfx
;
671 /* Process a function return TYPE to emit a PTX return as a prototype
672 or function prologue declaration. Returns true if return is via an
673 additional pointer parameter. The promotion behavior here must
674 match the regular GCC function return mashalling. */
677 write_return_type (std::stringstream
&s
, bool for_proto
, tree type
)
679 machine_mode mode
= TYPE_MODE (type
);
681 if (mode
== VOIDmode
)
684 bool return_in_mem
= pass_in_memory (mode
, type
, true);
689 return return_in_mem
;
691 /* Named return values can cause us to return a pointer as well
692 as expect an argument for the return location. This is
693 optimization-level specific, so no caller can make use of
694 this data, but more importantly for us, we must ensure it
695 doesn't change the PTX prototype. */
696 mode
= (machine_mode
) cfun
->machine
->return_mode
;
698 if (mode
== VOIDmode
)
699 return return_in_mem
;
701 /* Clear return_mode to inhibit copy of retval to non-existent
703 cfun
->machine
->return_mode
= VOIDmode
;
706 mode
= promote_return (mode
);
708 write_return_mode (s
, for_proto
, mode
);
710 return return_in_mem
;
713 /* Look for attributes in ATTRS that would indicate we must write a function
714 as a .entry kernel rather than a .func. Return true if one is found. */
717 write_as_kernel (tree attrs
)
719 return (lookup_attribute ("kernel", attrs
) != NULL_TREE
720 || lookup_attribute ("omp target entrypoint", attrs
) != NULL_TREE
);
723 /* Emit a linker marker for a function decl or defn. */
726 write_fn_marker (std::stringstream
&s
, bool is_defn
, bool globalize
,
732 s
<< " FUNCTION " << (is_defn
? "DEF: " : "DECL: ");
736 /* Emit a linker marker for a variable decl or defn. */
739 write_var_marker (FILE *file
, bool is_defn
, bool globalize
, const char *name
)
741 fprintf (file
, "\n// BEGIN%s VAR %s: ",
742 globalize
? " GLOBAL" : "",
743 is_defn
? "DEF" : "DECL");
744 assemble_name_raw (file
, name
);
748 /* Write a .func or .kernel declaration or definition along with
749 a helper comment for use by ld. S is the stream to write to, DECL
750 the decl for the function with name NAME. For definitions, emit
751 a declaration too. */
754 write_fn_proto (std::stringstream
&s
, bool is_defn
,
755 const char *name
, const_tree decl
)
758 /* Emit a declaration. The PTX assembler gets upset without it. */
759 name
= write_fn_proto (s
, false, name
, decl
);
762 /* Avoid repeating the name replacement. */
763 name
= nvptx_name_replacement (name
);
768 write_fn_marker (s
, is_defn
, TREE_PUBLIC (decl
), name
);
770 /* PTX declaration. */
771 if (DECL_EXTERNAL (decl
))
773 else if (TREE_PUBLIC (decl
))
774 s
<< (DECL_WEAK (decl
) ? ".weak " : ".visible ");
775 s
<< (write_as_kernel (DECL_ATTRIBUTES (decl
)) ? ".entry " : ".func ");
777 tree fntype
= TREE_TYPE (decl
);
778 tree result_type
= TREE_TYPE (fntype
);
780 /* atomic_compare_exchange_$n builtins have an exceptional calling
782 int not_atomic_weak_arg
= -1;
783 if (DECL_BUILT_IN_CLASS (decl
) == BUILT_IN_NORMAL
)
784 switch (DECL_FUNCTION_CODE (decl
))
786 case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_1
:
787 case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_2
:
788 case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_4
:
789 case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_8
:
790 case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_16
:
791 /* These atomics skip the 'weak' parm in an actual library
792 call. We must skip it in the prototype too. */
793 not_atomic_weak_arg
= 3;
800 /* Declare the result. */
801 bool return_in_mem
= write_return_type (s
, true, result_type
);
807 /* Emit argument list. */
809 argno
= write_arg_type (s
, -1, argno
, ptr_type_node
, true);
812 NULL in TYPE_ARG_TYPES, for old-style functions
813 NULL in DECL_ARGUMENTS, for builtin functions without another
815 So we have to pick the best one we have. */
816 tree args
= TYPE_ARG_TYPES (fntype
);
817 bool prototyped
= true;
820 args
= DECL_ARGUMENTS (decl
);
824 for (; args
; args
= TREE_CHAIN (args
), not_atomic_weak_arg
--)
826 tree type
= prototyped
? TREE_VALUE (args
) : TREE_TYPE (args
);
828 if (not_atomic_weak_arg
)
829 argno
= write_arg_type (s
, -1, argno
, type
, prototyped
);
831 gcc_assert (type
== boolean_type_node
);
834 if (stdarg_p (fntype
))
835 argno
= write_arg_type (s
, -1, argno
, ptr_type_node
, true);
837 if (DECL_STATIC_CHAIN (decl
))
838 argno
= write_arg_type (s
, -1, argno
, ptr_type_node
, true);
840 if (!argno
&& strcmp (name
, "main") == 0)
842 argno
= write_arg_type (s
, -1, argno
, integer_type_node
, true);
843 argno
= write_arg_type (s
, -1, argno
, ptr_type_node
, true);
849 s
<< (is_defn
? "\n" : ";\n");
854 /* Construct a function declaration from a call insn. This can be
855 necessary for two reasons - either we have an indirect call which
856 requires a .callprototype declaration, or we have a libcall
857 generated by emit_library_call for which no decl exists. */
860 write_fn_proto_from_insn (std::stringstream
&s
, const char *name
,
865 s
<< "\t.callprototype ";
870 name
= nvptx_name_replacement (name
);
871 write_fn_marker (s
, false, true, name
);
872 s
<< "\t.extern .func ";
875 if (result
!= NULL_RTX
)
876 write_return_mode (s
, true, GET_MODE (result
));
880 int arg_end
= XVECLEN (pat
, 0);
881 for (int i
= 1; i
< arg_end
; i
++)
883 /* We don't have to deal with mode splitting & promotion here,
884 as that was already done when generating the call
886 machine_mode mode
= GET_MODE (XEXP (XVECEXP (pat
, 0, i
), 0));
888 write_arg_mode (s
, -1, i
- 1, mode
);
895 /* DECL is an external FUNCTION_DECL, make sure its in the fndecl hash
896 table and and write a ptx prototype. These are emitted at end of
900 nvptx_record_fndecl (tree decl
)
902 tree
*slot
= declared_fndecls_htab
->find_slot (decl
, INSERT
);
906 const char *name
= get_fnname_from_decl (decl
);
907 write_fn_proto (func_decls
, false, name
, decl
);
911 /* Record a libcall or unprototyped external function. CALLEE is the
912 SYMBOL_REF. Insert into the libfunc hash table and emit a ptx
913 declaration for it. */
916 nvptx_record_libfunc (rtx callee
, rtx retval
, rtx pat
)
918 rtx
*slot
= declared_libfuncs_htab
->find_slot (callee
, INSERT
);
923 const char *name
= XSTR (callee
, 0);
924 write_fn_proto_from_insn (func_decls
, name
, retval
, pat
);
928 /* DECL is an external FUNCTION_DECL, that we're referencing. If it
929 is prototyped, record it now. Otherwise record it as needed at end
930 of compilation, when we might have more information about it. */
933 nvptx_record_needed_fndecl (tree decl
)
935 if (TYPE_ARG_TYPES (TREE_TYPE (decl
)) == NULL_TREE
)
937 tree
*slot
= needed_fndecls_htab
->find_slot (decl
, INSERT
);
942 nvptx_record_fndecl (decl
);
945 /* SYM is a SYMBOL_REF. If it refers to an external function, record
949 nvptx_maybe_record_fnsym (rtx sym
)
951 tree decl
= SYMBOL_REF_DECL (sym
);
953 if (decl
&& TREE_CODE (decl
) == FUNCTION_DECL
&& DECL_EXTERNAL (decl
))
954 nvptx_record_needed_fndecl (decl
);
957 /* Emit a local array to hold some part of a conventional stack frame
958 and initialize REGNO to point to it. If the size is zero, it'll
959 never be valid to dereference, so we can simply initialize to
963 init_frame (FILE *file
, int regno
, unsigned align
, unsigned size
)
966 fprintf (file
, "\t.local .align %d .b8 %s_ar[%u];\n",
967 align
, reg_names
[regno
], size
);
968 fprintf (file
, "\t.reg.u%d %s;\n",
969 POINTER_SIZE
, reg_names
[regno
]);
970 fprintf (file
, (size
? "\tcvta.local.u%d %s, %s_ar;\n"
971 : "\tmov.u%d %s, 0;\n"),
972 POINTER_SIZE
, reg_names
[regno
], reg_names
[regno
]);
975 /* Emit code to initialize the REGNO predicate register to indicate
976 whether we are not lane zero on the NAME axis. */
979 nvptx_init_axis_predicate (FILE *file
, int regno
, const char *name
)
981 fprintf (file
, "\t{\n");
982 fprintf (file
, "\t\t.reg.u32\t%%%s;\n", name
);
983 fprintf (file
, "\t\tmov.u32\t%%%s, %%tid.%s;\n", name
, name
);
984 fprintf (file
, "\t\tsetp.ne.u32\t%%r%d, %%%s, 0;\n", regno
, name
);
985 fprintf (file
, "\t}\n");
988 /* Implement ASM_DECLARE_FUNCTION_NAME. Writes the start of a ptx
989 function, including local var decls and copies from the arguments to
993 nvptx_declare_function_name (FILE *file
, const char *name
, const_tree decl
)
995 tree fntype
= TREE_TYPE (decl
);
996 tree result_type
= TREE_TYPE (fntype
);
999 /* We construct the initial part of the function into a string
1000 stream, in order to share the prototype writing code. */
1001 std::stringstream s
;
1002 write_fn_proto (s
, true, name
, decl
);
1005 bool return_in_mem
= write_return_type (s
, false, result_type
);
1007 argno
= write_arg_type (s
, 0, argno
, ptr_type_node
, true);
1009 /* Declare and initialize incoming arguments. */
1010 tree args
= TYPE_ARG_TYPES (fntype
);
1011 bool prototyped
= true;
1014 args
= DECL_ARGUMENTS (decl
);
1018 for (; args
!= NULL_TREE
; args
= TREE_CHAIN (args
))
1020 tree type
= prototyped
? TREE_VALUE (args
) : TREE_TYPE (args
);
1022 argno
= write_arg_type (s
, 0, argno
, type
, prototyped
);
1025 if (stdarg_p (fntype
))
1026 argno
= write_arg_type (s
, ARG_POINTER_REGNUM
, argno
, ptr_type_node
,
1029 if (DECL_STATIC_CHAIN (decl
) || cfun
->machine
->has_chain
)
1030 write_arg_type (s
, STATIC_CHAIN_REGNUM
,
1031 DECL_STATIC_CHAIN (decl
) ? argno
: -1, ptr_type_node
,
1034 fprintf (file
, "%s", s
.str().c_str());
1036 /* Declare a local var for outgoing varargs. */
1037 if (cfun
->machine
->has_varadic
)
1038 init_frame (file
, STACK_POINTER_REGNUM
,
1039 UNITS_PER_WORD
, crtl
->outgoing_args_size
);
1041 /* Declare a local variable for the frame. Force its size to be
1042 DImode-compatible. */
1043 HOST_WIDE_INT sz
= get_frame_size ();
1044 if (sz
|| cfun
->machine
->has_chain
)
1045 init_frame (file
, FRAME_POINTER_REGNUM
,
1046 crtl
->stack_alignment_needed
/ BITS_PER_UNIT
,
1047 (sz
+ GET_MODE_SIZE (DImode
) - 1)
1048 & ~(HOST_WIDE_INT
)(GET_MODE_SIZE (DImode
) - 1));
1050 /* Declare the pseudos we have as ptx registers. */
1051 int maxregs
= max_reg_num ();
1052 for (int i
= LAST_VIRTUAL_REGISTER
+ 1; i
< maxregs
; i
++)
1054 if (regno_reg_rtx
[i
] != const0_rtx
)
1056 machine_mode mode
= PSEUDO_REGNO_MODE (i
);
1057 machine_mode split
= maybe_split_mode (mode
);
1059 if (split
!= VOIDmode
)
1061 fprintf (file
, "\t.reg%s ", nvptx_ptx_type_from_mode (mode
, true));
1062 output_reg (file
, i
, split
, -2);
1063 fprintf (file
, ";\n");
1067 /* Emit axis predicates. */
1068 if (cfun
->machine
->axis_predicate
[0])
1069 nvptx_init_axis_predicate (file
,
1070 REGNO (cfun
->machine
->axis_predicate
[0]), "y");
1071 if (cfun
->machine
->axis_predicate
[1])
1072 nvptx_init_axis_predicate (file
,
1073 REGNO (cfun
->machine
->axis_predicate
[1]), "x");
1076 /* Output a return instruction. Also copy the return value to its outgoing
1080 nvptx_output_return (void)
1082 machine_mode mode
= (machine_mode
)cfun
->machine
->return_mode
;
1084 if (mode
!= VOIDmode
)
1085 fprintf (asm_out_file
, "\tst.param%s\t[%s_out], %s;\n",
1086 nvptx_ptx_type_from_mode (mode
, false),
1087 reg_names
[NVPTX_RETURN_REGNUM
],
1088 reg_names
[NVPTX_RETURN_REGNUM
]);
1093 /* Terminate a function by writing a closing brace to FILE. */
1096 nvptx_function_end (FILE *file
)
1098 fprintf (file
, "}\n");
1101 /* Decide whether we can make a sibling call to a function. For ptx, we
1105 nvptx_function_ok_for_sibcall (tree
, tree
)
1110 /* Return Dynamic ReAlignment Pointer RTX. For PTX there isn't any. */
1113 nvptx_get_drap_rtx (void)
1118 /* Implement the TARGET_CALL_ARGS hook. Record information about one
1119 argument to the next call. */
1122 nvptx_call_args (rtx arg
, tree fntype
)
1124 if (!cfun
->machine
->doing_call
)
1126 cfun
->machine
->doing_call
= true;
1127 cfun
->machine
->is_varadic
= false;
1128 cfun
->machine
->num_args
= 0;
1130 if (fntype
&& stdarg_p (fntype
))
1132 cfun
->machine
->is_varadic
= true;
1133 cfun
->machine
->has_varadic
= true;
1134 cfun
->machine
->num_args
++;
1138 if (REG_P (arg
) && arg
!= pc_rtx
)
1140 cfun
->machine
->num_args
++;
1141 cfun
->machine
->call_args
= alloc_EXPR_LIST (VOIDmode
, arg
,
1142 cfun
->machine
->call_args
);
1146 /* Implement the corresponding END_CALL_ARGS hook. Clear and free the
1147 information we recorded. */
1150 nvptx_end_call_args (void)
1152 cfun
->machine
->doing_call
= false;
1153 free_EXPR_LIST_list (&cfun
->machine
->call_args
);
1156 /* Emit the sequence for a call to ADDRESS, setting RETVAL. Keep
1157 track of whether calls involving static chains or varargs were seen
1158 in the current function.
1159 For libcalls, maintain a hash table of decls we have seen, and
1160 record a function decl for later when encountering a new one. */
1163 nvptx_expand_call (rtx retval
, rtx address
)
1165 rtx callee
= XEXP (address
, 0);
1166 rtx varargs
= NULL_RTX
;
1167 unsigned parallel
= 0;
1169 if (!call_insn_operand (callee
, Pmode
))
1171 callee
= force_reg (Pmode
, callee
);
1172 address
= change_address (address
, QImode
, callee
);
1175 if (GET_CODE (callee
) == SYMBOL_REF
)
1177 tree decl
= SYMBOL_REF_DECL (callee
);
1178 if (decl
!= NULL_TREE
)
1180 if (DECL_STATIC_CHAIN (decl
))
1181 cfun
->machine
->has_chain
= true;
1183 tree attr
= get_oacc_fn_attrib (decl
);
1186 tree dims
= TREE_VALUE (attr
);
1188 parallel
= GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1;
1189 for (int ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
1191 if (TREE_PURPOSE (dims
)
1192 && !integer_zerop (TREE_PURPOSE (dims
)))
1194 /* Not on this axis. */
1195 parallel
^= GOMP_DIM_MASK (ix
);
1196 dims
= TREE_CHAIN (dims
);
1202 unsigned nargs
= cfun
->machine
->num_args
;
1203 if (cfun
->machine
->is_varadic
)
1205 varargs
= gen_reg_rtx (Pmode
);
1206 emit_move_insn (varargs
, stack_pointer_rtx
);
1209 rtvec vec
= rtvec_alloc (nargs
+ 1);
1210 rtx pat
= gen_rtx_PARALLEL (VOIDmode
, vec
);
1213 rtx call
= gen_rtx_CALL (VOIDmode
, address
, const0_rtx
);
1214 rtx tmp_retval
= retval
;
1217 if (!nvptx_register_operand (retval
, GET_MODE (retval
)))
1218 tmp_retval
= gen_reg_rtx (GET_MODE (retval
));
1219 call
= gen_rtx_SET (tmp_retval
, call
);
1221 XVECEXP (pat
, 0, vec_pos
++) = call
;
1223 /* Construct the call insn, including a USE for each argument pseudo
1224 register. These will be used when printing the insn. */
1225 for (rtx arg
= cfun
->machine
->call_args
; arg
; arg
= XEXP (arg
, 1))
1226 XVECEXP (pat
, 0, vec_pos
++) = gen_rtx_USE (VOIDmode
, XEXP (arg
, 0));
1229 XVECEXP (pat
, 0, vec_pos
++) = gen_rtx_USE (VOIDmode
, varargs
);
1231 gcc_assert (vec_pos
= XVECLEN (pat
, 0));
1233 nvptx_emit_forking (parallel
, true);
1234 emit_call_insn (pat
);
1235 nvptx_emit_joining (parallel
, true);
1237 if (tmp_retval
!= retval
)
1238 emit_move_insn (retval
, tmp_retval
);
1241 /* Emit a comparison COMPARE, and return the new test to be used in the
1245 nvptx_expand_compare (rtx compare
)
1247 rtx pred
= gen_reg_rtx (BImode
);
1248 rtx cmp
= gen_rtx_fmt_ee (GET_CODE (compare
), BImode
,
1249 XEXP (compare
, 0), XEXP (compare
, 1));
1250 emit_insn (gen_rtx_SET (pred
, cmp
));
1251 return gen_rtx_NE (BImode
, pred
, const0_rtx
);
1254 /* Expand the oacc fork & join primitive into ptx-required unspecs. */
1257 nvptx_expand_oacc_fork (unsigned mode
)
1259 nvptx_emit_forking (GOMP_DIM_MASK (mode
), false);
1263 nvptx_expand_oacc_join (unsigned mode
)
1265 nvptx_emit_joining (GOMP_DIM_MASK (mode
), false);
1268 /* Generate instruction(s) to unpack a 64 bit object into 2 32 bit
1272 nvptx_gen_unpack (rtx dst0
, rtx dst1
, rtx src
)
1276 switch (GET_MODE (src
))
1279 res
= gen_unpackdisi2 (dst0
, dst1
, src
);
1282 res
= gen_unpackdfsi2 (dst0
, dst1
, src
);
1284 default: gcc_unreachable ();
1289 /* Generate instruction(s) to pack 2 32 bit objects into a 64 bit
1293 nvptx_gen_pack (rtx dst
, rtx src0
, rtx src1
)
1297 switch (GET_MODE (dst
))
1300 res
= gen_packsidi2 (dst
, src0
, src1
);
1303 res
= gen_packsidf2 (dst
, src0
, src1
);
1305 default: gcc_unreachable ();
1310 /* Generate an instruction or sequence to broadcast register REG
1311 across the vectors of a single warp. */
1314 nvptx_gen_shuffle (rtx dst
, rtx src
, rtx idx
, nvptx_shuffle_kind kind
)
1318 switch (GET_MODE (dst
))
1321 res
= gen_nvptx_shufflesi (dst
, src
, idx
, GEN_INT (kind
));
1324 res
= gen_nvptx_shufflesf (dst
, src
, idx
, GEN_INT (kind
));
1329 rtx tmp0
= gen_reg_rtx (SImode
);
1330 rtx tmp1
= gen_reg_rtx (SImode
);
1333 emit_insn (nvptx_gen_unpack (tmp0
, tmp1
, src
));
1334 emit_insn (nvptx_gen_shuffle (tmp0
, tmp0
, idx
, kind
));
1335 emit_insn (nvptx_gen_shuffle (tmp1
, tmp1
, idx
, kind
));
1336 emit_insn (nvptx_gen_pack (dst
, tmp0
, tmp1
));
1343 rtx tmp
= gen_reg_rtx (SImode
);
1346 emit_insn (gen_sel_truesi (tmp
, src
, GEN_INT (1), const0_rtx
));
1347 emit_insn (nvptx_gen_shuffle (tmp
, tmp
, idx
, kind
));
1348 emit_insn (gen_rtx_SET (dst
, gen_rtx_NE (BImode
, tmp
, const0_rtx
)));
1356 rtx tmp
= gen_reg_rtx (SImode
);
1359 emit_insn (gen_rtx_SET (tmp
, gen_rtx_fmt_e (ZERO_EXTEND
, SImode
, src
)));
1360 emit_insn (nvptx_gen_shuffle (tmp
, tmp
, idx
, kind
));
1361 emit_insn (gen_rtx_SET (dst
, gen_rtx_fmt_e (TRUNCATE
, GET_MODE (dst
),
1374 /* Generate an instruction or sequence to broadcast register REG
1375 across the vectors of a single warp. */
1378 nvptx_gen_vcast (rtx reg
)
1380 return nvptx_gen_shuffle (reg
, reg
, const0_rtx
, SHUFFLE_IDX
);
1383 /* Structure used when generating a worker-level spill or fill. */
1387 rtx base
; /* Register holding base addr of buffer. */
1388 rtx ptr
; /* Iteration var, if needed. */
1389 unsigned offset
; /* Offset into worker buffer. */
1392 /* Direction of the spill/fill and looping setup/teardown indicator. */
1398 PM_loop_begin
= 1 << 2,
1399 PM_loop_end
= 1 << 3,
1401 PM_read_write
= PM_read
| PM_write
1404 /* Generate instruction(s) to spill or fill register REG to/from the
1405 worker broadcast array. PM indicates what is to be done, REP
1406 how many loop iterations will be executed (0 for not a loop). */
1409 nvptx_gen_wcast (rtx reg
, propagate_mask pm
, unsigned rep
, wcast_data_t
*data
)
1412 machine_mode mode
= GET_MODE (reg
);
1418 rtx tmp
= gen_reg_rtx (SImode
);
1422 emit_insn (gen_sel_truesi (tmp
, reg
, GEN_INT (1), const0_rtx
));
1423 emit_insn (nvptx_gen_wcast (tmp
, pm
, rep
, data
));
1425 emit_insn (gen_rtx_SET (reg
, gen_rtx_NE (BImode
, tmp
, const0_rtx
)));
1433 rtx addr
= data
->ptr
;
1437 unsigned align
= GET_MODE_ALIGNMENT (mode
) / BITS_PER_UNIT
;
1439 if (align
> worker_bcast_align
)
1440 worker_bcast_align
= align
;
1441 data
->offset
= (data
->offset
+ align
- 1) & ~(align
- 1);
1444 addr
= gen_rtx_PLUS (Pmode
, addr
, GEN_INT (data
->offset
));
1447 addr
= gen_rtx_MEM (mode
, addr
);
1449 res
= gen_rtx_SET (addr
, reg
);
1450 else if (pm
== PM_write
)
1451 res
= gen_rtx_SET (reg
, addr
);
1457 /* We're using a ptr, increment it. */
1461 emit_insn (gen_adddi3 (data
->ptr
, data
->ptr
,
1462 GEN_INT (GET_MODE_SIZE (GET_MODE (reg
)))));
1468 data
->offset
+= rep
* GET_MODE_SIZE (GET_MODE (reg
));
1475 /* Returns true if X is a valid address for use in a memory reference. */
1478 nvptx_legitimate_address_p (machine_mode
, rtx x
, bool)
1480 enum rtx_code code
= GET_CODE (x
);
1488 if (REG_P (XEXP (x
, 0)) && CONST_INT_P (XEXP (x
, 1)))
1502 /* Machinery to output constant initializers. When beginning an
1503 initializer, we decide on a fragment size (which is visible in ptx
1504 in the type used), and then all initializer data is buffered until
1505 a fragment is filled and ready to be written out. */
1509 unsigned HOST_WIDE_INT mask
; /* Mask for storing fragment. */
1510 unsigned HOST_WIDE_INT val
; /* Current fragment value. */
1511 unsigned HOST_WIDE_INT remaining
; /* Remaining bytes to be written
1513 unsigned size
; /* Fragment size to accumulate. */
1514 unsigned offset
; /* Offset within current fragment. */
1515 bool started
; /* Whether we've output any initializer. */
1518 /* The current fragment is full, write it out. SYM may provide a
1519 symbolic reference we should output, in which case the fragment
1520 value is the addend. */
1523 output_init_frag (rtx sym
)
1525 fprintf (asm_out_file
, init_frag
.started
? ", " : " = { ");
1526 unsigned HOST_WIDE_INT val
= init_frag
.val
;
1528 init_frag
.started
= true;
1530 init_frag
.offset
= 0;
1531 init_frag
.remaining
--;
1535 fprintf (asm_out_file
, "generic(");
1536 output_address (VOIDmode
, sym
);
1537 fprintf (asm_out_file
, val
? ") + " : ")");
1541 fprintf (asm_out_file
, HOST_WIDE_INT_PRINT_DEC
, val
);
1544 /* Add value VAL of size SIZE to the data we're emitting, and keep
1545 writing out chunks as they fill up. */
1548 nvptx_assemble_value (unsigned HOST_WIDE_INT val
, unsigned size
)
1550 val
&= ((unsigned HOST_WIDE_INT
)2 << (size
* BITS_PER_UNIT
- 1)) - 1;
1552 for (unsigned part
= 0; size
; size
-= part
)
1554 val
>>= part
* BITS_PER_UNIT
;
1555 part
= init_frag
.size
- init_frag
.offset
;
1559 unsigned HOST_WIDE_INT partial
1560 = val
<< (init_frag
.offset
* BITS_PER_UNIT
);
1561 init_frag
.val
|= partial
& init_frag
.mask
;
1562 init_frag
.offset
+= part
;
1564 if (init_frag
.offset
== init_frag
.size
)
1565 output_init_frag (NULL
);
1569 /* Target hook for assembling integer object X of size SIZE. */
1572 nvptx_assemble_integer (rtx x
, unsigned int size
, int ARG_UNUSED (aligned_p
))
1574 HOST_WIDE_INT val
= 0;
1576 switch (GET_CODE (x
))
1579 /* Let the generic machinery figure it out, usually for a
1584 nvptx_assemble_value (INTVAL (x
), size
);
1589 gcc_assert (GET_CODE (x
) == PLUS
);
1590 val
= INTVAL (XEXP (x
, 1));
1592 gcc_assert (GET_CODE (x
) == SYMBOL_REF
);
1596 gcc_assert (size
== init_frag
.size
);
1597 if (init_frag
.offset
)
1598 sorry ("cannot emit unaligned pointers in ptx assembly");
1600 nvptx_maybe_record_fnsym (x
);
1601 init_frag
.val
= val
;
1602 output_init_frag (x
);
1609 /* Output SIZE zero bytes. We ignore the FILE argument since the
1610 functions we're calling to perform the output just use
1614 nvptx_output_skip (FILE *, unsigned HOST_WIDE_INT size
)
1616 /* Finish the current fragment, if it's started. */
1617 if (init_frag
.offset
)
1619 unsigned part
= init_frag
.size
- init_frag
.offset
;
1621 part
= (unsigned) size
;
1623 nvptx_assemble_value (0, part
);
1626 /* If this skip doesn't terminate the initializer, write as many
1627 remaining pieces as possible directly. */
1628 if (size
< init_frag
.remaining
* init_frag
.size
)
1630 while (size
>= init_frag
.size
)
1632 size
-= init_frag
.size
;
1633 output_init_frag (NULL_RTX
);
1636 nvptx_assemble_value (0, size
);
1640 /* Output a string STR with length SIZE. As in nvptx_output_skip we
1641 ignore the FILE arg. */
1644 nvptx_output_ascii (FILE *, const char *str
, unsigned HOST_WIDE_INT size
)
1646 for (unsigned HOST_WIDE_INT i
= 0; i
< size
; i
++)
1647 nvptx_assemble_value (str
[i
], 1);
1650 /* Emit a PTX variable decl and prepare for emission of its
1651 initializer. NAME is the symbol name and SETION the PTX data
1652 area. The type is TYPE, object size SIZE and alignment is ALIGN.
1653 The caller has already emitted any indentation and linkage
1654 specifier. It is responsible for any initializer, terminating ;
1655 and newline. SIZE is in bytes, ALIGN is in bits -- confusingly
1656 this is the opposite way round that PTX wants them! */
1659 nvptx_assemble_decl_begin (FILE *file
, const char *name
, const char *section
,
1660 const_tree type
, HOST_WIDE_INT size
, unsigned align
)
1662 while (TREE_CODE (type
) == ARRAY_TYPE
)
1663 type
= TREE_TYPE (type
);
1665 if (TREE_CODE (type
) == VECTOR_TYPE
1666 || TREE_CODE (type
) == COMPLEX_TYPE
)
1667 /* Neither vector nor complex types can contain the other. */
1668 type
= TREE_TYPE (type
);
1670 unsigned elt_size
= int_size_in_bytes (type
);
1672 /* Largest mode we're prepared to accept. For BLKmode types we
1673 don't know if it'll contain pointer constants, so have to choose
1674 pointer size, otherwise we can choose DImode. */
1675 machine_mode elt_mode
= TYPE_MODE (type
) == BLKmode
? Pmode
: DImode
;
1677 elt_size
|= GET_MODE_SIZE (elt_mode
);
1678 elt_size
&= -elt_size
; /* Extract LSB set. */
1680 init_frag
.size
= elt_size
;
1681 /* Avoid undefined shift behavior by using '2'. */
1682 init_frag
.mask
= ((unsigned HOST_WIDE_INT
)2
1683 << (elt_size
* BITS_PER_UNIT
- 1)) - 1;
1685 init_frag
.offset
= 0;
1686 init_frag
.started
= false;
1687 /* Size might not be a multiple of elt size, if there's an
1688 initialized trailing struct array with smaller type than
1690 init_frag
.remaining
= (size
+ elt_size
- 1) / elt_size
;
1692 fprintf (file
, "%s .align %d .u%d ",
1693 section
, align
/ BITS_PER_UNIT
,
1694 elt_size
* BITS_PER_UNIT
);
1695 assemble_name (file
, name
);
1698 /* We make everything an array, to simplify any initialization
1700 fprintf (file
, "[" HOST_WIDE_INT_PRINT_DEC
"]", init_frag
.remaining
);
1703 /* Called when the initializer for a decl has been completely output through
1704 combinations of the three functions above. */
1707 nvptx_assemble_decl_end (void)
1709 if (init_frag
.offset
)
1710 /* This can happen with a packed struct with trailing array member. */
1711 nvptx_assemble_value (0, init_frag
.size
- init_frag
.offset
);
1712 fprintf (asm_out_file
, init_frag
.started
? " };\n" : ";\n");
1715 /* Output an uninitialized common or file-scope variable. */
1718 nvptx_output_aligned_decl (FILE *file
, const char *name
,
1719 const_tree decl
, HOST_WIDE_INT size
, unsigned align
)
1721 write_var_marker (file
, true, TREE_PUBLIC (decl
), name
);
1723 /* If this is public, it is common. The nearest thing we have to
1725 fprintf (file
, "\t%s", TREE_PUBLIC (decl
) ? ".weak " : "");
1727 nvptx_assemble_decl_begin (file
, name
, section_for_decl (decl
),
1728 TREE_TYPE (decl
), size
, align
);
1729 nvptx_assemble_decl_end ();
1732 /* Implement TARGET_ASM_DECLARE_CONSTANT_NAME. Begin the process of
1733 writing a constant variable EXP with NAME and SIZE and its
1734 initializer to FILE. */
1737 nvptx_asm_declare_constant_name (FILE *file
, const char *name
,
1738 const_tree exp
, HOST_WIDE_INT obj_size
)
1740 write_var_marker (file
, true, false, name
);
1742 fprintf (file
, "\t");
1744 tree type
= TREE_TYPE (exp
);
1745 nvptx_assemble_decl_begin (file
, name
, ".const", type
, obj_size
,
1749 /* Implement the ASM_DECLARE_OBJECT_NAME macro. Used to start writing
1750 a variable DECL with NAME to FILE. */
1753 nvptx_declare_object_name (FILE *file
, const char *name
, const_tree decl
)
1755 write_var_marker (file
, true, TREE_PUBLIC (decl
), name
);
1757 fprintf (file
, "\t%s", (!TREE_PUBLIC (decl
) ? ""
1758 : DECL_WEAK (decl
) ? ".weak " : ".visible "));
1760 tree type
= TREE_TYPE (decl
);
1761 HOST_WIDE_INT obj_size
= tree_to_shwi (DECL_SIZE_UNIT (decl
));
1762 nvptx_assemble_decl_begin (file
, name
, section_for_decl (decl
),
1763 type
, obj_size
, DECL_ALIGN (decl
));
1766 /* Implement TARGET_ASM_GLOBALIZE_LABEL by doing nothing. */
1769 nvptx_globalize_label (FILE *, const char *)
1773 /* Implement TARGET_ASM_ASSEMBLE_UNDEFINED_DECL. Write an extern
1774 declaration only for variable DECL with NAME to FILE. */
1777 nvptx_assemble_undefined_decl (FILE *file
, const char *name
, const_tree decl
)
1779 /* The middle end can place constant pool decls into the varpool as
1780 undefined. Until that is fixed, catch the problem here. */
1781 if (DECL_IN_CONSTANT_POOL (decl
))
1784 /* We support weak defintions, and hence have the right
1785 ASM_WEAKEN_DECL definition. Diagnose the problem here. */
1786 if (DECL_WEAK (decl
))
1787 error_at (DECL_SOURCE_LOCATION (decl
),
1788 "PTX does not support weak declarations"
1789 " (only weak definitions)");
1790 write_var_marker (file
, false, TREE_PUBLIC (decl
), name
);
1792 fprintf (file
, "\t.extern ");
1793 tree size
= DECL_SIZE_UNIT (decl
);
1794 nvptx_assemble_decl_begin (file
, name
, section_for_decl (decl
),
1795 TREE_TYPE (decl
), size
? tree_to_shwi (size
) : 0,
1797 nvptx_assemble_decl_end ();
1800 /* Output a pattern for a move instruction. */
1803 nvptx_output_mov_insn (rtx dst
, rtx src
)
1805 machine_mode dst_mode
= GET_MODE (dst
);
1806 machine_mode dst_inner
= (GET_CODE (dst
) == SUBREG
1807 ? GET_MODE (XEXP (dst
, 0)) : dst_mode
);
1808 machine_mode src_inner
= (GET_CODE (src
) == SUBREG
1809 ? GET_MODE (XEXP (src
, 0)) : dst_mode
);
1812 if (GET_CODE (sym
) == CONST
)
1813 sym
= XEXP (XEXP (sym
, 0), 0);
1814 if (SYMBOL_REF_P (sym
))
1816 if (SYMBOL_DATA_AREA (sym
) != DATA_AREA_GENERIC
)
1817 return "%.\tcvta%D1%t0\t%0, %1;";
1818 nvptx_maybe_record_fnsym (sym
);
1821 if (src_inner
== dst_inner
)
1822 return "%.\tmov%t0\t%0, %1;";
1824 if (CONSTANT_P (src
))
1825 return (GET_MODE_CLASS (dst_inner
) == MODE_INT
1826 && GET_MODE_CLASS (src_inner
) != MODE_FLOAT
1827 ? "%.\tmov%t0\t%0, %1;" : "%.\tmov.b%T0\t%0, %1;");
1829 if (GET_MODE_SIZE (dst_inner
) == GET_MODE_SIZE (src_inner
))
1830 return "%.\tmov.b%T0\t%0, %1;";
1832 return "%.\tcvt%t0%t1\t%0, %1;";
1835 /* Output INSN, which is a call to CALLEE with result RESULT. For ptx, this
1836 involves writing .param declarations and in/out copies into them. For
1837 indirect calls, also write the .callprototype. */
1840 nvptx_output_call_insn (rtx_insn
*insn
, rtx result
, rtx callee
)
1844 bool needs_tgt
= register_operand (callee
, Pmode
);
1845 rtx pat
= PATTERN (insn
);
1846 int arg_end
= XVECLEN (pat
, 0);
1847 tree decl
= NULL_TREE
;
1849 fprintf (asm_out_file
, "\t{\n");
1851 fprintf (asm_out_file
, "\t\t.param%s %s_in;\n",
1852 nvptx_ptx_type_from_mode (GET_MODE (result
), false),
1853 reg_names
[NVPTX_RETURN_REGNUM
]);
1855 /* Ensure we have a ptx declaration in the output if necessary. */
1856 if (GET_CODE (callee
) == SYMBOL_REF
)
1858 decl
= SYMBOL_REF_DECL (callee
);
1860 || (DECL_EXTERNAL (decl
) && !TYPE_ARG_TYPES (TREE_TYPE (decl
))))
1861 nvptx_record_libfunc (callee
, result
, pat
);
1862 else if (DECL_EXTERNAL (decl
))
1863 nvptx_record_fndecl (decl
);
1868 ASM_GENERATE_INTERNAL_LABEL (buf
, "LCT", labelno
);
1870 ASM_OUTPUT_LABEL (asm_out_file
, buf
);
1871 std::stringstream s
;
1872 write_fn_proto_from_insn (s
, NULL
, result
, pat
);
1873 fputs (s
.str().c_str(), asm_out_file
);
1876 for (int argno
= 1; argno
< arg_end
; argno
++)
1878 rtx t
= XEXP (XVECEXP (pat
, 0, argno
), 0);
1879 machine_mode mode
= GET_MODE (t
);
1880 const char *ptx_type
= nvptx_ptx_type_from_mode (mode
, false);
1882 /* Mode splitting has already been done. */
1883 fprintf (asm_out_file
, "\t\t.param%s %%out_arg%d;\n"
1884 "\t\tst.param%s [%%out_arg%d], ",
1885 ptx_type
, argno
, ptx_type
, argno
);
1886 output_reg (asm_out_file
, REGNO (t
), VOIDmode
);
1887 fprintf (asm_out_file
, ";\n");
1890 fprintf (asm_out_file
, "\t\tcall ");
1891 if (result
!= NULL_RTX
)
1892 fprintf (asm_out_file
, "(%s_in), ", reg_names
[NVPTX_RETURN_REGNUM
]);
1896 const char *name
= get_fnname_from_decl (decl
);
1897 name
= nvptx_name_replacement (name
);
1898 assemble_name (asm_out_file
, name
);
1901 output_address (VOIDmode
, callee
);
1903 const char *open
= "(";
1904 for (int argno
= 1; argno
< arg_end
; argno
++)
1906 fprintf (asm_out_file
, ", %s%%out_arg%d", open
, argno
);
1909 if (decl
&& DECL_STATIC_CHAIN (decl
))
1911 fprintf (asm_out_file
, ", %s%s", open
, reg_names
[STATIC_CHAIN_REGNUM
]);
1915 fprintf (asm_out_file
, ")");
1919 fprintf (asm_out_file
, ", ");
1920 assemble_name (asm_out_file
, buf
);
1922 fprintf (asm_out_file
, ";\n");
1924 if (find_reg_note (insn
, REG_NORETURN
, NULL
))
1925 /* No return functions confuse the PTX JIT, as it doesn't realize
1926 the flow control barrier they imply. It can seg fault if it
1927 encounters what looks like an unexitable loop. Emit a trailing
1928 trap, which it does grok. */
1929 fprintf (asm_out_file
, "\t\ttrap; // (noreturn)\n");
1933 static char rval
[sizeof ("\tld.param%%t0\t%%0, [%%%s_in];\n\t}") + 8];
1936 /* We must escape the '%' that starts RETURN_REGNUM. */
1937 sprintf (rval
, "\tld.param%%t0\t%%0, [%%%s_in];\n\t}",
1938 reg_names
[NVPTX_RETURN_REGNUM
]);
1945 /* Implement TARGET_PRINT_OPERAND_PUNCT_VALID_P. */
1948 nvptx_print_operand_punct_valid_p (unsigned char c
)
1950 return c
== '.' || c
== '#';
1953 static void nvptx_print_operand (FILE *, rtx
, int);
1955 /* Subroutine of nvptx_print_operand; used to print a memory reference X to FILE. */
1958 nvptx_print_address_operand (FILE *file
, rtx x
, machine_mode
)
1961 if (GET_CODE (x
) == CONST
)
1963 switch (GET_CODE (x
))
1967 output_address (VOIDmode
, XEXP (x
, 0));
1968 fprintf (file
, "+");
1969 output_address (VOIDmode
, off
);
1974 output_addr_const (file
, x
);
1978 gcc_assert (GET_CODE (x
) != MEM
);
1979 nvptx_print_operand (file
, x
, 0);
1984 /* Write assembly language output for the address ADDR to FILE. */
1987 nvptx_print_operand_address (FILE *file
, machine_mode mode
, rtx addr
)
1989 nvptx_print_address_operand (file
, addr
, mode
);
1992 /* Print an operand, X, to FILE, with an optional modifier in CODE.
1995 . -- print the predicate for the instruction or an emptry string for an
1997 # -- print a rounding mode for the instruction
1999 A -- print a data area for a MEM
2000 c -- print an opcode suffix for a comparison operator, including a type code
2001 D -- print a data area for a MEM operand
2002 S -- print a shuffle kind specified by CONST_INT
2003 t -- print a type opcode suffix, promoting QImode to 32 bits
2004 T -- print a type size in bits
2005 u -- print a type opcode suffix without promotions. */
2008 nvptx_print_operand (FILE *file
, rtx x
, int code
)
2012 x
= current_insn_predicate
;
2015 unsigned int regno
= REGNO (XEXP (x
, 0));
2017 if (GET_CODE (x
) == EQ
)
2019 fputs (reg_names
[regno
], file
);
2024 else if (code
== '#')
2026 fputs (".rn", file
);
2030 enum rtx_code x_code
= GET_CODE (x
);
2031 machine_mode mode
= GET_MODE (x
);
2040 if (GET_CODE (x
) == CONST
)
2042 if (GET_CODE (x
) == PLUS
)
2045 if (GET_CODE (x
) == SYMBOL_REF
)
2046 fputs (section_for_sym (x
), file
);
2051 if (x_code
== SUBREG
)
2053 mode
= GET_MODE (SUBREG_REG (x
));
2056 else if (COMPLEX_MODE_P (mode
))
2057 mode
= GET_MODE_INNER (mode
);
2059 fprintf (file
, "%s", nvptx_ptx_type_from_mode (mode
, code
== 't'));
2064 nvptx_shuffle_kind kind
= (nvptx_shuffle_kind
) UINTVAL (x
);
2065 /* Same order as nvptx_shuffle_kind. */
2066 static const char *const kinds
[] =
2067 {".up", ".down", ".bfly", ".idx"};
2068 fputs (kinds
[kind
], file
);
2073 fprintf (file
, "%d", GET_MODE_BITSIZE (mode
));
2077 fprintf (file
, "@");
2081 fprintf (file
, "@!");
2085 mode
= GET_MODE (XEXP (x
, 0));
2089 fputs (".eq", file
);
2092 if (FLOAT_MODE_P (mode
))
2093 fputs (".neu", file
);
2095 fputs (".ne", file
);
2099 fputs (".le", file
);
2103 fputs (".ge", file
);
2107 fputs (".lt", file
);
2111 fputs (".gt", file
);
2114 fputs (".ne", file
);
2117 fputs (".equ", file
);
2120 fputs (".leu", file
);
2123 fputs (".geu", file
);
2126 fputs (".ltu", file
);
2129 fputs (".gtu", file
);
2132 fputs (".nan", file
);
2135 fputs (".num", file
);
2140 if (FLOAT_MODE_P (mode
)
2141 || x_code
== EQ
|| x_code
== NE
2142 || x_code
== GEU
|| x_code
== GTU
2143 || x_code
== LEU
|| x_code
== LTU
)
2144 fputs (nvptx_ptx_type_from_mode (mode
, true), file
);
2146 fprintf (file
, ".s%d", GET_MODE_BITSIZE (mode
));
2154 rtx inner_x
= SUBREG_REG (x
);
2155 machine_mode inner_mode
= GET_MODE (inner_x
);
2156 machine_mode split
= maybe_split_mode (inner_mode
);
2158 if (split
!= VOIDmode
2159 && (GET_MODE_SIZE (inner_mode
) == GET_MODE_SIZE (mode
)))
2160 output_reg (file
, REGNO (inner_x
), split
);
2162 output_reg (file
, REGNO (inner_x
), split
, SUBREG_BYTE (x
));
2167 output_reg (file
, REGNO (x
), maybe_split_mode (mode
));
2172 nvptx_print_address_operand (file
, XEXP (x
, 0), mode
);
2177 output_addr_const (file
, x
);
2183 /* We could use output_addr_const, but that can print things like
2184 "x-8", which breaks ptxas. Need to ensure it is output as
2186 nvptx_print_address_operand (file
, x
, VOIDmode
);
2191 real_to_target (vals
, CONST_DOUBLE_REAL_VALUE (x
), mode
);
2192 vals
[0] &= 0xffffffff;
2193 vals
[1] &= 0xffffffff;
2195 fprintf (file
, "0f%08lx", vals
[0]);
2197 fprintf (file
, "0d%08lx%08lx", vals
[1], vals
[0]);
2201 output_addr_const (file
, x
);
2206 /* Record replacement regs used to deal with subreg operands. */
2209 rtx replacement
[MAX_RECOG_OPERANDS
];
2215 /* Allocate or reuse a replacement in R and return the rtx. */
2218 get_replacement (struct reg_replace
*r
)
2220 if (r
->n_allocated
== r
->n_in_use
)
2221 r
->replacement
[r
->n_allocated
++] = gen_reg_rtx (r
->mode
);
2222 return r
->replacement
[r
->n_in_use
++];
2225 /* Clean up subreg operands. In ptx assembly, everything is typed, and
2226 the presence of subregs would break the rules for most instructions.
2227 Replace them with a suitable new register of the right size, plus
2228 conversion copyin/copyout instructions. */
2231 nvptx_reorg_subreg (void)
2233 struct reg_replace qiregs
, hiregs
, siregs
, diregs
;
2234 rtx_insn
*insn
, *next
;
2236 qiregs
.n_allocated
= 0;
2237 hiregs
.n_allocated
= 0;
2238 siregs
.n_allocated
= 0;
2239 diregs
.n_allocated
= 0;
2240 qiregs
.mode
= QImode
;
2241 hiregs
.mode
= HImode
;
2242 siregs
.mode
= SImode
;
2243 diregs
.mode
= DImode
;
2245 for (insn
= get_insns (); insn
; insn
= next
)
2247 next
= NEXT_INSN (insn
);
2248 if (!NONDEBUG_INSN_P (insn
)
2249 || asm_noperands (PATTERN (insn
)) >= 0
2250 || GET_CODE (PATTERN (insn
)) == USE
2251 || GET_CODE (PATTERN (insn
)) == CLOBBER
)
2254 qiregs
.n_in_use
= 0;
2255 hiregs
.n_in_use
= 0;
2256 siregs
.n_in_use
= 0;
2257 diregs
.n_in_use
= 0;
2258 extract_insn (insn
);
2259 enum attr_subregs_ok s_ok
= get_attr_subregs_ok (insn
);
2261 for (int i
= 0; i
< recog_data
.n_operands
; i
++)
2263 rtx op
= recog_data
.operand
[i
];
2264 if (GET_CODE (op
) != SUBREG
)
2267 rtx inner
= SUBREG_REG (op
);
2269 machine_mode outer_mode
= GET_MODE (op
);
2270 machine_mode inner_mode
= GET_MODE (inner
);
2273 && (GET_MODE_PRECISION (inner_mode
)
2274 >= GET_MODE_PRECISION (outer_mode
)))
2276 gcc_assert (SCALAR_INT_MODE_P (outer_mode
));
2277 struct reg_replace
*r
= (outer_mode
== QImode
? &qiregs
2278 : outer_mode
== HImode
? &hiregs
2279 : outer_mode
== SImode
? &siregs
2281 rtx new_reg
= get_replacement (r
);
2283 if (recog_data
.operand_type
[i
] != OP_OUT
)
2286 if (GET_MODE_PRECISION (inner_mode
)
2287 < GET_MODE_PRECISION (outer_mode
))
2292 rtx pat
= gen_rtx_SET (new_reg
,
2293 gen_rtx_fmt_e (code
, outer_mode
, inner
));
2294 emit_insn_before (pat
, insn
);
2297 if (recog_data
.operand_type
[i
] != OP_IN
)
2300 if (GET_MODE_PRECISION (inner_mode
)
2301 < GET_MODE_PRECISION (outer_mode
))
2306 rtx pat
= gen_rtx_SET (inner
,
2307 gen_rtx_fmt_e (code
, inner_mode
, new_reg
));
2308 emit_insn_after (pat
, insn
);
2310 validate_change (insn
, recog_data
.operand_loc
[i
], new_reg
, false);
2315 /* Loop structure of the function. The entire function is described as
2320 /* Parent parallel. */
2323 /* Next sibling parallel. */
2326 /* First child parallel. */
2329 /* Partitioning mask of the parallel. */
2332 /* Partitioning used within inner parallels. */
2333 unsigned inner_mask
;
2335 /* Location of parallel forked and join. The forked is the first
2336 block in the parallel and the join is the first block after of
2338 basic_block forked_block
;
2339 basic_block join_block
;
2341 rtx_insn
*forked_insn
;
2342 rtx_insn
*join_insn
;
2344 rtx_insn
*fork_insn
;
2345 rtx_insn
*joining_insn
;
2347 /* Basic blocks in this parallel, but not in child parallels. The
2348 FORKED and JOINING blocks are in the partition. The FORK and JOIN
2350 auto_vec
<basic_block
> blocks
;
2353 parallel (parallel
*parent
, unsigned mode
);
2357 /* Constructor links the new parallel into it's parent's chain of
2360 parallel::parallel (parallel
*parent_
, unsigned mask_
)
2361 :parent (parent_
), next (0), inner (0), mask (mask_
), inner_mask (0)
2363 forked_block
= join_block
= 0;
2364 forked_insn
= join_insn
= 0;
2365 fork_insn
= joining_insn
= 0;
2369 next
= parent
->inner
;
2370 parent
->inner
= this;
2374 parallel::~parallel ()
2380 /* Map of basic blocks to insns */
2381 typedef hash_map
<basic_block
, rtx_insn
*> bb_insn_map_t
;
2383 /* A tuple of an insn of interest and the BB in which it resides. */
2384 typedef std::pair
<rtx_insn
*, basic_block
> insn_bb_t
;
2385 typedef auto_vec
<insn_bb_t
> insn_bb_vec_t
;
2387 /* Split basic blocks such that each forked and join unspecs are at
2388 the start of their basic blocks. Thus afterwards each block will
2389 have a single partitioning mode. We also do the same for return
2390 insns, as they are executed by every thread. Return the
2391 partitioning mode of the function as a whole. Populate MAP with
2392 head and tail blocks. We also clear the BB visited flag, which is
2393 used when finding partitions. */
2396 nvptx_split_blocks (bb_insn_map_t
*map
)
2398 insn_bb_vec_t worklist
;
2402 /* Locate all the reorg instructions of interest. */
2403 FOR_ALL_BB_FN (block
, cfun
)
2405 bool seen_insn
= false;
2407 /* Clear visited flag, for use by parallel locator */
2408 block
->flags
&= ~BB_VISITED
;
2410 FOR_BB_INSNS (block
, insn
)
2414 switch (recog_memoized (insn
))
2419 case CODE_FOR_nvptx_forked
:
2420 case CODE_FOR_nvptx_join
:
2423 case CODE_FOR_return
:
2424 /* We also need to split just before return insns, as
2425 that insn needs executing by all threads, but the
2426 block it is in probably does not. */
2431 /* We've found an instruction that must be at the start of
2432 a block, but isn't. Add it to the worklist. */
2433 worklist
.safe_push (insn_bb_t (insn
, block
));
2435 /* It was already the first instruction. Just add it to
2437 map
->get_or_insert (block
) = insn
;
2442 /* Split blocks on the worklist. */
2445 basic_block remap
= 0;
2446 for (ix
= 0; worklist
.iterate (ix
, &elt
); ix
++)
2448 if (remap
!= elt
->second
)
2450 block
= elt
->second
;
2454 /* Split block before insn. The insn is in the new block */
2455 edge e
= split_block (block
, PREV_INSN (elt
->first
));
2458 map
->get_or_insert (block
) = elt
->first
;
2462 /* BLOCK is a basic block containing a head or tail instruction.
2463 Locate the associated prehead or pretail instruction, which must be
2464 in the single predecessor block. */
2467 nvptx_discover_pre (basic_block block
, int expected
)
2469 gcc_assert (block
->preds
->length () == 1);
2470 basic_block pre_block
= (*block
->preds
)[0]->src
;
2473 for (pre_insn
= BB_END (pre_block
); !INSN_P (pre_insn
);
2474 pre_insn
= PREV_INSN (pre_insn
))
2475 gcc_assert (pre_insn
!= BB_HEAD (pre_block
));
2477 gcc_assert (recog_memoized (pre_insn
) == expected
);
2481 /* Dump this parallel and all its inner parallels. */
2484 nvptx_dump_pars (parallel
*par
, unsigned depth
)
2486 fprintf (dump_file
, "%u: mask %d head=%d, tail=%d\n",
2488 par
->forked_block
? par
->forked_block
->index
: -1,
2489 par
->join_block
? par
->join_block
->index
: -1);
2491 fprintf (dump_file
, " blocks:");
2494 for (unsigned ix
= 0; par
->blocks
.iterate (ix
, &block
); ix
++)
2495 fprintf (dump_file
, " %d", block
->index
);
2496 fprintf (dump_file
, "\n");
2498 nvptx_dump_pars (par
->inner
, depth
+ 1);
2501 nvptx_dump_pars (par
->next
, depth
);
2504 /* If BLOCK contains a fork/join marker, process it to create or
2505 terminate a loop structure. Add this block to the current loop,
2506 and then walk successor blocks. */
2509 nvptx_find_par (bb_insn_map_t
*map
, parallel
*par
, basic_block block
)
2511 if (block
->flags
& BB_VISITED
)
2513 block
->flags
|= BB_VISITED
;
2515 if (rtx_insn
**endp
= map
->get (block
))
2517 rtx_insn
*end
= *endp
;
2519 /* This is a block head or tail, or return instruction. */
2520 switch (recog_memoized (end
))
2522 case CODE_FOR_return
:
2523 /* Return instructions are in their own block, and we
2524 don't need to do anything more. */
2527 case CODE_FOR_nvptx_forked
:
2528 /* Loop head, create a new inner loop and add it into
2529 our parent's child list. */
2531 unsigned mask
= UINTVAL (XVECEXP (PATTERN (end
), 0, 0));
2534 par
= new parallel (par
, mask
);
2535 par
->forked_block
= block
;
2536 par
->forked_insn
= end
;
2537 if (!(mask
& GOMP_DIM_MASK (GOMP_DIM_MAX
))
2538 && (mask
& GOMP_DIM_MASK (GOMP_DIM_WORKER
)))
2540 = nvptx_discover_pre (block
, CODE_FOR_nvptx_fork
);
2544 case CODE_FOR_nvptx_join
:
2545 /* A loop tail. Finish the current loop and return to
2548 unsigned mask
= UINTVAL (XVECEXP (PATTERN (end
), 0, 0));
2550 gcc_assert (par
->mask
== mask
);
2551 par
->join_block
= block
;
2552 par
->join_insn
= end
;
2553 if (!(mask
& GOMP_DIM_MASK (GOMP_DIM_MAX
))
2554 && (mask
& GOMP_DIM_MASK (GOMP_DIM_WORKER
)))
2556 = nvptx_discover_pre (block
, CODE_FOR_nvptx_joining
);
2567 /* Add this block onto the current loop's list of blocks. */
2568 par
->blocks
.safe_push (block
);
2570 /* This must be the entry block. Create a NULL parallel. */
2571 par
= new parallel (0, 0);
2573 /* Walk successor blocks. */
2577 FOR_EACH_EDGE (e
, ei
, block
->succs
)
2578 nvptx_find_par (map
, par
, e
->dest
);
2583 /* DFS walk the CFG looking for fork & join markers. Construct
2584 loop structures as we go. MAP is a mapping of basic blocks
2585 to head & tail markers, discovered when splitting blocks. This
2586 speeds up the discovery. We rely on the BB visited flag having
2587 been cleared when splitting blocks. */
2590 nvptx_discover_pars (bb_insn_map_t
*map
)
2594 /* Mark exit blocks as visited. */
2595 block
= EXIT_BLOCK_PTR_FOR_FN (cfun
);
2596 block
->flags
|= BB_VISITED
;
2598 /* And entry block as not. */
2599 block
= ENTRY_BLOCK_PTR_FOR_FN (cfun
);
2600 block
->flags
&= ~BB_VISITED
;
2602 parallel
*par
= nvptx_find_par (map
, 0, block
);
2606 fprintf (dump_file
, "\nLoops\n");
2607 nvptx_dump_pars (par
, 0);
2608 fprintf (dump_file
, "\n");
2614 /* Analyse a group of BBs within a partitioned region and create N
2615 Single-Entry-Single-Exit regions. Some of those regions will be
2616 trivial ones consisting of a single BB. The blocks of a
2617 partitioned region might form a set of disjoint graphs -- because
2618 the region encloses a differently partitoned sub region.
2620 We use the linear time algorithm described in 'Finding Regions Fast:
2621 Single Entry Single Exit and control Regions in Linear Time'
2622 Johnson, Pearson & Pingali. That algorithm deals with complete
2623 CFGs, where a back edge is inserted from END to START, and thus the
2624 problem becomes one of finding equivalent loops.
2626 In this case we have a partial CFG. We complete it by redirecting
2627 any incoming edge to the graph to be from an arbitrary external BB,
2628 and similarly redirecting any outgoing edge to be to that BB.
2629 Thus we end up with a closed graph.
2631 The algorithm works by building a spanning tree of an undirected
2632 graph and keeping track of back edges from nodes further from the
2633 root in the tree to nodes nearer to the root in the tree. In the
2634 description below, the root is up and the tree grows downwards.
2636 We avoid having to deal with degenerate back-edges to the same
2637 block, by splitting each BB into 3 -- one for input edges, one for
2638 the node itself and one for the output edges. Such back edges are
2639 referred to as 'Brackets'. Cycle equivalent nodes will have the
2640 same set of brackets.
2642 Determining bracket equivalency is done by maintaining a list of
2643 brackets in such a manner that the list length and final bracket
2644 uniquely identify the set.
2646 We use coloring to mark all BBs with cycle equivalency with the
2647 same color. This is the output of the 'Finding Regions Fast'
2648 algorithm. Notice it doesn't actually find the set of nodes within
2649 a particular region, just unorderd sets of nodes that are the
2650 entries and exits of SESE regions.
2652 After determining cycle equivalency, we need to find the minimal
2653 set of SESE regions. Do this with a DFS coloring walk of the
2654 complete graph. We're either 'looking' or 'coloring'. When
2655 looking, and we're in the subgraph, we start coloring the color of
2656 the current node, and remember that node as the start of the
2657 current color's SESE region. Every time we go to a new node, we
2658 decrement the count of nodes with thet color. If it reaches zero,
2659 we remember that node as the end of the current color's SESE region
2660 and return to 'looking'. Otherwise we color the node the current
2663 This way we end up with coloring the inside of non-trivial SESE
2664 regions with the color of that region. */
2666 /* A pair of BBs. We use this to represent SESE regions. */
2667 typedef std::pair
<basic_block
, basic_block
> bb_pair_t
;
2668 typedef auto_vec
<bb_pair_t
> bb_pair_vec_t
;
2670 /* A node in the undirected CFG. The discriminator SECOND indicates just
2671 above or just below the BB idicated by FIRST. */
2672 typedef std::pair
<basic_block
, int> pseudo_node_t
;
2674 /* A bracket indicates an edge towards the root of the spanning tree of the
2675 undirected graph. Each bracket has a color, determined
2676 from the currrent set of brackets. */
2679 pseudo_node_t back
; /* Back target */
2681 /* Current color and size of set. */
2685 bracket (pseudo_node_t back_
)
2686 : back (back_
), color (~0u), size (~0u)
2690 unsigned get_color (auto_vec
<unsigned> &color_counts
, unsigned length
)
2695 color
= color_counts
.length ();
2696 color_counts
.quick_push (0);
2698 color_counts
[color
]++;
2703 typedef auto_vec
<bracket
> bracket_vec_t
;
2705 /* Basic block info for finding SESE regions. */
2709 int node
; /* Node number in spanning tree. */
2710 int parent
; /* Parent node number. */
2712 /* The algorithm splits each node A into Ai, A', Ao. The incoming
2713 edges arrive at pseudo-node Ai and the outgoing edges leave at
2714 pseudo-node Ao. We have to remember which way we arrived at a
2715 particular node when generating the spanning tree. dir > 0 means
2716 we arrived at Ai, dir < 0 means we arrived at Ao. */
2719 /* Lowest numbered pseudo-node reached via a backedge from thsis
2720 node, or any descendant. */
2723 int color
; /* Cycle-equivalence color */
2725 /* Stack of brackets for this node. */
2726 bracket_vec_t brackets
;
2728 bb_sese (unsigned node_
, unsigned p
, int dir_
)
2729 :node (node_
), parent (p
), dir (dir_
)
2734 /* Push a bracket ending at BACK. */
2735 void push (const pseudo_node_t
&back
)
2738 fprintf (dump_file
, "Pushing backedge %d:%+d\n",
2739 back
.first
? back
.first
->index
: 0, back
.second
);
2740 brackets
.safe_push (bracket (back
));
2743 void append (bb_sese
*child
);
2744 void remove (const pseudo_node_t
&);
2746 /* Set node's color. */
2747 void set_color (auto_vec
<unsigned> &color_counts
)
2749 color
= brackets
.last ().get_color (color_counts
, brackets
.length ());
2753 bb_sese::~bb_sese ()
2757 /* Destructively append CHILD's brackets. */
2760 bb_sese::append (bb_sese
*child
)
2762 if (int len
= child
->brackets
.length ())
2768 for (ix
= 0; ix
< len
; ix
++)
2770 const pseudo_node_t
&pseudo
= child
->brackets
[ix
].back
;
2771 fprintf (dump_file
, "Appending (%d)'s backedge %d:%+d\n",
2772 child
->node
, pseudo
.first
? pseudo
.first
->index
: 0,
2776 if (!brackets
.length ())
2777 std::swap (brackets
, child
->brackets
);
2780 brackets
.reserve (len
);
2781 for (ix
= 0; ix
< len
; ix
++)
2782 brackets
.quick_push (child
->brackets
[ix
]);
2787 /* Remove brackets that terminate at PSEUDO. */
2790 bb_sese::remove (const pseudo_node_t
&pseudo
)
2792 unsigned removed
= 0;
2793 int len
= brackets
.length ();
2795 for (int ix
= 0; ix
< len
; ix
++)
2797 if (brackets
[ix
].back
== pseudo
)
2800 fprintf (dump_file
, "Removing backedge %d:%+d\n",
2801 pseudo
.first
? pseudo
.first
->index
: 0, pseudo
.second
);
2805 brackets
[ix
-removed
] = brackets
[ix
];
2811 /* Accessors for BB's aux pointer. */
2812 #define BB_SET_SESE(B, S) ((B)->aux = (S))
2813 #define BB_GET_SESE(B) ((bb_sese *)(B)->aux)
2815 /* DFS walk creating SESE data structures. Only cover nodes with
2816 BB_VISITED set. Append discovered blocks to LIST. We number in
2817 increments of 3 so that the above and below pseudo nodes can be
2818 implicitly numbered too. */
2821 nvptx_sese_number (int n
, int p
, int dir
, basic_block b
,
2822 auto_vec
<basic_block
> *list
)
2824 if (BB_GET_SESE (b
))
2828 fprintf (dump_file
, "Block %d(%d), parent (%d), orientation %+d\n",
2829 b
->index
, n
, p
, dir
);
2831 BB_SET_SESE (b
, new bb_sese (n
, p
, dir
));
2835 list
->quick_push (b
);
2837 /* First walk the nodes on the 'other side' of this node, then walk
2838 the nodes on the same side. */
2839 for (unsigned ix
= 2; ix
; ix
--)
2841 vec
<edge
, va_gc
> *edges
= dir
> 0 ? b
->succs
: b
->preds
;
2842 size_t offset
= (dir
> 0 ? offsetof (edge_def
, dest
)
2843 : offsetof (edge_def
, src
));
2847 FOR_EACH_EDGE (e
, ei
, edges
)
2849 basic_block target
= *(basic_block
*)((char *)e
+ offset
);
2851 if (target
->flags
& BB_VISITED
)
2852 n
= nvptx_sese_number (n
, p
, dir
, target
, list
);
2859 /* Process pseudo node above (DIR < 0) or below (DIR > 0) ME.
2860 EDGES are the outgoing edges and OFFSET is the offset to the src
2861 or dst block on the edges. */
2864 nvptx_sese_pseudo (basic_block me
, bb_sese
*sese
, int depth
, int dir
,
2865 vec
<edge
, va_gc
> *edges
, size_t offset
)
2869 int hi_back
= depth
;
2870 pseudo_node_t
node_back (0, depth
);
2871 int hi_child
= depth
;
2872 pseudo_node_t
node_child (0, depth
);
2873 basic_block child
= NULL
;
2874 unsigned num_children
= 0;
2875 int usd
= -dir
* sese
->dir
;
2878 fprintf (dump_file
, "\nProcessing %d(%d) %+d\n",
2879 me
->index
, sese
->node
, dir
);
2883 /* This is the above pseudo-child. It has the BB itself as an
2884 additional child node. */
2885 node_child
= sese
->high
;
2886 hi_child
= node_child
.second
;
2887 if (node_child
.first
)
2888 hi_child
+= BB_GET_SESE (node_child
.first
)->node
;
2892 /* Examine each edge.
2893 - if it is a child (a) append its bracket list and (b) record
2894 whether it is the child with the highest reaching bracket.
2895 - if it is an edge to ancestor, record whether it's the highest
2896 reaching backlink. */
2897 FOR_EACH_EDGE (e
, ei
, edges
)
2899 basic_block target
= *(basic_block
*)((char *)e
+ offset
);
2901 if (bb_sese
*t_sese
= BB_GET_SESE (target
))
2903 if (t_sese
->parent
== sese
->node
&& !(t_sese
->dir
+ usd
))
2905 /* Child node. Append its bracket list. */
2907 sese
->append (t_sese
);
2909 /* Compare it's hi value. */
2910 int t_hi
= t_sese
->high
.second
;
2912 if (basic_block child_hi_block
= t_sese
->high
.first
)
2913 t_hi
+= BB_GET_SESE (child_hi_block
)->node
;
2915 if (hi_child
> t_hi
)
2918 node_child
= t_sese
->high
;
2922 else if (t_sese
->node
< sese
->node
+ dir
2923 && !(dir
< 0 && sese
->parent
== t_sese
->node
))
2925 /* Non-parental ancestor node -- a backlink. */
2926 int d
= usd
* t_sese
->dir
;
2927 int back
= t_sese
->node
+ d
;
2932 node_back
= pseudo_node_t (target
, d
);
2937 { /* Fallen off graph, backlink to entry node. */
2939 node_back
= pseudo_node_t (0, 0);
2943 /* Remove any brackets that terminate at this pseudo node. */
2944 sese
->remove (pseudo_node_t (me
, dir
));
2946 /* Now push any backlinks from this pseudo node. */
2947 FOR_EACH_EDGE (e
, ei
, edges
)
2949 basic_block target
= *(basic_block
*)((char *)e
+ offset
);
2950 if (bb_sese
*t_sese
= BB_GET_SESE (target
))
2952 if (t_sese
->node
< sese
->node
+ dir
2953 && !(dir
< 0 && sese
->parent
== t_sese
->node
))
2954 /* Non-parental ancestor node - backedge from me. */
2955 sese
->push (pseudo_node_t (target
, usd
* t_sese
->dir
));
2959 /* back edge to entry node */
2960 sese
->push (pseudo_node_t (0, 0));
2964 /* If this node leads directly or indirectly to a no-return region of
2965 the graph, then fake a backedge to entry node. */
2966 if (!sese
->brackets
.length () || !edges
|| !edges
->length ())
2969 node_back
= pseudo_node_t (0, 0);
2970 sese
->push (node_back
);
2973 /* Record the highest reaching backedge from us or a descendant. */
2974 sese
->high
= hi_back
< hi_child
? node_back
: node_child
;
2976 if (num_children
> 1)
2978 /* There is more than one child -- this is a Y shaped piece of
2979 spanning tree. We have to insert a fake backedge from this
2980 node to the highest ancestor reached by not-the-highest
2981 reaching child. Note that there may be multiple children
2982 with backedges to the same highest node. That's ok and we
2983 insert the edge to that highest node. */
2985 if (dir
< 0 && child
)
2987 node_child
= sese
->high
;
2988 hi_child
= node_child
.second
;
2989 if (node_child
.first
)
2990 hi_child
+= BB_GET_SESE (node_child
.first
)->node
;
2993 FOR_EACH_EDGE (e
, ei
, edges
)
2995 basic_block target
= *(basic_block
*)((char *)e
+ offset
);
2997 if (target
== child
)
2998 /* Ignore the highest child. */
3001 bb_sese
*t_sese
= BB_GET_SESE (target
);
3004 if (t_sese
->parent
!= sese
->node
)
3008 /* Compare its hi value. */
3009 int t_hi
= t_sese
->high
.second
;
3011 if (basic_block child_hi_block
= t_sese
->high
.first
)
3012 t_hi
+= BB_GET_SESE (child_hi_block
)->node
;
3014 if (hi_child
> t_hi
)
3017 node_child
= t_sese
->high
;
3021 sese
->push (node_child
);
3026 /* DFS walk of BB graph. Color node BLOCK according to COLORING then
3027 proceed to successors. Set SESE entry and exit nodes of
3031 nvptx_sese_color (auto_vec
<unsigned> &color_counts
, bb_pair_vec_t
®ions
,
3032 basic_block block
, int coloring
)
3034 bb_sese
*sese
= BB_GET_SESE (block
);
3036 if (block
->flags
& BB_VISITED
)
3038 /* If we've already encountered this block, either we must not
3039 be coloring, or it must have been colored the current color. */
3040 gcc_assert (coloring
< 0 || (sese
&& coloring
== sese
->color
));
3044 block
->flags
|= BB_VISITED
;
3050 /* Start coloring a region. */
3051 regions
[sese
->color
].first
= block
;
3052 coloring
= sese
->color
;
3055 if (!--color_counts
[sese
->color
] && sese
->color
== coloring
)
3057 /* Found final block of SESE region. */
3058 regions
[sese
->color
].second
= block
;
3062 /* Color the node, so we can assert on revisiting the node
3063 that the graph is indeed SESE. */
3064 sese
->color
= coloring
;
3067 /* Fallen off the subgraph, we cannot be coloring. */
3068 gcc_assert (coloring
< 0);
3070 /* Walk each successor block. */
3071 if (block
->succs
&& block
->succs
->length ())
3076 FOR_EACH_EDGE (e
, ei
, block
->succs
)
3077 nvptx_sese_color (color_counts
, regions
, e
->dest
, coloring
);
3080 gcc_assert (coloring
< 0);
3083 /* Find minimal set of SESE regions covering BLOCKS. REGIONS might
3084 end up with NULL entries in it. */
3087 nvptx_find_sese (auto_vec
<basic_block
> &blocks
, bb_pair_vec_t
®ions
)
3092 /* First clear each BB of the whole function. */
3093 FOR_EACH_BB_FN (block
, cfun
)
3095 block
->flags
&= ~BB_VISITED
;
3096 BB_SET_SESE (block
, 0);
3098 block
= EXIT_BLOCK_PTR_FOR_FN (cfun
);
3099 block
->flags
&= ~BB_VISITED
;
3100 BB_SET_SESE (block
, 0);
3101 block
= ENTRY_BLOCK_PTR_FOR_FN (cfun
);
3102 block
->flags
&= ~BB_VISITED
;
3103 BB_SET_SESE (block
, 0);
3105 /* Mark blocks in the function that are in this graph. */
3106 for (ix
= 0; blocks
.iterate (ix
, &block
); ix
++)
3107 block
->flags
|= BB_VISITED
;
3109 /* Counts of nodes assigned to each color. There cannot be more
3110 colors than blocks (and hopefully there will be fewer). */
3111 auto_vec
<unsigned> color_counts
;
3112 color_counts
.reserve (blocks
.length ());
3114 /* Worklist of nodes in the spanning tree. Again, there cannot be
3115 more nodes in the tree than blocks (there will be fewer if the
3116 CFG of blocks is disjoint). */
3117 auto_vec
<basic_block
> spanlist
;
3118 spanlist
.reserve (blocks
.length ());
3120 /* Make sure every block has its cycle class determined. */
3121 for (ix
= 0; blocks
.iterate (ix
, &block
); ix
++)
3123 if (BB_GET_SESE (block
))
3124 /* We already met this block in an earlier graph solve. */
3128 fprintf (dump_file
, "Searching graph starting at %d\n", block
->index
);
3130 /* Number the nodes reachable from block initial DFS order. */
3131 int depth
= nvptx_sese_number (2, 0, +1, block
, &spanlist
);
3133 /* Now walk in reverse DFS order to find cycle equivalents. */
3134 while (spanlist
.length ())
3136 block
= spanlist
.pop ();
3137 bb_sese
*sese
= BB_GET_SESE (block
);
3139 /* Do the pseudo node below. */
3140 nvptx_sese_pseudo (block
, sese
, depth
, +1,
3141 sese
->dir
> 0 ? block
->succs
: block
->preds
,
3142 (sese
->dir
> 0 ? offsetof (edge_def
, dest
)
3143 : offsetof (edge_def
, src
)));
3144 sese
->set_color (color_counts
);
3145 /* Do the pseudo node above. */
3146 nvptx_sese_pseudo (block
, sese
, depth
, -1,
3147 sese
->dir
< 0 ? block
->succs
: block
->preds
,
3148 (sese
->dir
< 0 ? offsetof (edge_def
, dest
)
3149 : offsetof (edge_def
, src
)));
3152 fprintf (dump_file
, "\n");
3158 const char *comma
= "";
3160 fprintf (dump_file
, "Found %d cycle equivalents\n",
3161 color_counts
.length ());
3162 for (ix
= 0; color_counts
.iterate (ix
, &count
); ix
++)
3164 fprintf (dump_file
, "%s%d[%d]={", comma
, ix
, count
);
3167 for (unsigned jx
= 0; blocks
.iterate (jx
, &block
); jx
++)
3168 if (BB_GET_SESE (block
)->color
== ix
)
3170 block
->flags
|= BB_VISITED
;
3171 fprintf (dump_file
, "%s%d", comma
, block
->index
);
3174 fprintf (dump_file
, "}");
3177 fprintf (dump_file
, "\n");
3180 /* Now we've colored every block in the subgraph. We now need to
3181 determine the minimal set of SESE regions that cover that
3182 subgraph. Do this with a DFS walk of the complete function.
3183 During the walk we're either 'looking' or 'coloring'. When we
3184 reach the last node of a particular color, we stop coloring and
3185 return to looking. */
3187 /* There cannot be more SESE regions than colors. */
3188 regions
.reserve (color_counts
.length ());
3189 for (ix
= color_counts
.length (); ix
--;)
3190 regions
.quick_push (bb_pair_t (0, 0));
3192 for (ix
= 0; blocks
.iterate (ix
, &block
); ix
++)
3193 block
->flags
&= ~BB_VISITED
;
3195 nvptx_sese_color (color_counts
, regions
, ENTRY_BLOCK_PTR_FOR_FN (cfun
), -1);
3199 const char *comma
= "";
3200 int len
= regions
.length ();
3202 fprintf (dump_file
, "SESE regions:");
3203 for (ix
= 0; ix
!= len
; ix
++)
3205 basic_block from
= regions
[ix
].first
;
3206 basic_block to
= regions
[ix
].second
;
3210 fprintf (dump_file
, "%s %d{%d", comma
, ix
, from
->index
);
3212 fprintf (dump_file
, "->%d", to
->index
);
3214 int color
= BB_GET_SESE (from
)->color
;
3216 /* Print the blocks within the region (excluding ends). */
3217 FOR_EACH_BB_FN (block
, cfun
)
3219 bb_sese
*sese
= BB_GET_SESE (block
);
3221 if (sese
&& sese
->color
== color
3222 && block
!= from
&& block
!= to
)
3223 fprintf (dump_file
, ".%d", block
->index
);
3225 fprintf (dump_file
, "}");
3229 fprintf (dump_file
, "\n\n");
3232 for (ix
= 0; blocks
.iterate (ix
, &block
); ix
++)
3233 delete BB_GET_SESE (block
);
3239 /* Propagate live state at the start of a partitioned region. BLOCK
3240 provides the live register information, and might not contain
3241 INSN. Propagation is inserted just after INSN. RW indicates whether
3242 we are reading and/or writing state. This
3243 separation is needed for worker-level proppagation where we
3244 essentially do a spill & fill. FN is the underlying worker
3245 function to generate the propagation instructions for single
3246 register. DATA is user data.
3248 We propagate the live register set and the entire frame. We could
3249 do better by (a) propagating just the live set that is used within
3250 the partitioned regions and (b) only propagating stack entries that
3251 are used. The latter might be quite hard to determine. */
3253 typedef rtx (*propagator_fn
) (rtx
, propagate_mask
, unsigned, void *);
3256 nvptx_propagate (basic_block block
, rtx_insn
*insn
, propagate_mask rw
,
3257 propagator_fn fn
, void *data
)
3259 bitmap live
= DF_LIVE_IN (block
);
3260 bitmap_iterator iterator
;
3263 /* Copy the frame array. */
3264 HOST_WIDE_INT fs
= get_frame_size ();
3267 rtx tmp
= gen_reg_rtx (DImode
);
3269 rtx ptr
= gen_reg_rtx (Pmode
);
3270 rtx pred
= NULL_RTX
;
3271 rtx_code_label
*label
= NULL
;
3273 /* The frame size might not be DImode compatible, but the frame
3274 array's declaration will be. So it's ok to round up here. */
3275 fs
= (fs
+ GET_MODE_SIZE (DImode
) - 1) / GET_MODE_SIZE (DImode
);
3276 /* Detect single iteration loop. */
3281 emit_insn (gen_rtx_SET (ptr
, frame_pointer_rtx
));
3284 idx
= gen_reg_rtx (SImode
);
3285 pred
= gen_reg_rtx (BImode
);
3286 label
= gen_label_rtx ();
3288 emit_insn (gen_rtx_SET (idx
, GEN_INT (fs
)));
3289 /* Allow worker function to initialize anything needed. */
3290 rtx init
= fn (tmp
, PM_loop_begin
, fs
, data
);
3294 LABEL_NUSES (label
)++;
3295 emit_insn (gen_addsi3 (idx
, idx
, GEN_INT (-1)));
3298 emit_insn (gen_rtx_SET (tmp
, gen_rtx_MEM (DImode
, ptr
)));
3299 emit_insn (fn (tmp
, rw
, fs
, data
));
3301 emit_insn (gen_rtx_SET (gen_rtx_MEM (DImode
, ptr
), tmp
));
3304 emit_insn (gen_rtx_SET (pred
, gen_rtx_NE (BImode
, idx
, const0_rtx
)));
3305 emit_insn (gen_adddi3 (ptr
, ptr
, GEN_INT (GET_MODE_SIZE (DImode
))));
3306 emit_insn (gen_br_true_uni (pred
, label
));
3307 rtx fini
= fn (tmp
, PM_loop_end
, fs
, data
);
3310 emit_insn (gen_rtx_CLOBBER (GET_MODE (idx
), idx
));
3312 emit_insn (gen_rtx_CLOBBER (GET_MODE (tmp
), tmp
));
3313 emit_insn (gen_rtx_CLOBBER (GET_MODE (ptr
), ptr
));
3314 rtx cpy
= get_insns ();
3316 insn
= emit_insn_after (cpy
, insn
);
3319 /* Copy live registers. */
3320 EXECUTE_IF_SET_IN_BITMAP (live
, 0, ix
, iterator
)
3322 rtx reg
= regno_reg_rtx
[ix
];
3324 if (REGNO (reg
) >= FIRST_PSEUDO_REGISTER
)
3326 rtx bcast
= fn (reg
, rw
, 0, data
);
3328 insn
= emit_insn_after (bcast
, insn
);
3333 /* Worker for nvptx_vpropagate. */
3336 vprop_gen (rtx reg
, propagate_mask pm
,
3337 unsigned ARG_UNUSED (count
), void *ARG_UNUSED (data
))
3339 if (!(pm
& PM_read_write
))
3342 return nvptx_gen_vcast (reg
);
3345 /* Propagate state that is live at start of BLOCK across the vectors
3346 of a single warp. Propagation is inserted just after INSN. */
3349 nvptx_vpropagate (basic_block block
, rtx_insn
*insn
)
3351 nvptx_propagate (block
, insn
, PM_read_write
, vprop_gen
, 0);
3354 /* Worker for nvptx_wpropagate. */
3357 wprop_gen (rtx reg
, propagate_mask pm
, unsigned rep
, void *data_
)
3359 wcast_data_t
*data
= (wcast_data_t
*)data_
;
3361 if (pm
& PM_loop_begin
)
3363 /* Starting a loop, initialize pointer. */
3364 unsigned align
= GET_MODE_ALIGNMENT (GET_MODE (reg
)) / BITS_PER_UNIT
;
3366 if (align
> worker_bcast_align
)
3367 worker_bcast_align
= align
;
3368 data
->offset
= (data
->offset
+ align
- 1) & ~(align
- 1);
3370 data
->ptr
= gen_reg_rtx (Pmode
);
3372 return gen_adddi3 (data
->ptr
, data
->base
, GEN_INT (data
->offset
));
3374 else if (pm
& PM_loop_end
)
3376 rtx clobber
= gen_rtx_CLOBBER (GET_MODE (data
->ptr
), data
->ptr
);
3377 data
->ptr
= NULL_RTX
;
3381 return nvptx_gen_wcast (reg
, pm
, rep
, data
);
3384 /* Spill or fill live state that is live at start of BLOCK. PRE_P
3385 indicates if this is just before partitioned mode (do spill), or
3386 just after it starts (do fill). Sequence is inserted just after
3390 nvptx_wpropagate (bool pre_p
, basic_block block
, rtx_insn
*insn
)
3394 data
.base
= gen_reg_rtx (Pmode
);
3396 data
.ptr
= NULL_RTX
;
3398 nvptx_propagate (block
, insn
, pre_p
? PM_read
: PM_write
, wprop_gen
, &data
);
3401 /* Stuff was emitted, initialize the base pointer now. */
3402 rtx init
= gen_rtx_SET (data
.base
, worker_bcast_sym
);
3403 emit_insn_after (init
, insn
);
3405 if (worker_bcast_size
< data
.offset
)
3406 worker_bcast_size
= data
.offset
;
3410 /* Emit a worker-level synchronization barrier. We use different
3411 markers for before and after synchronizations. */
3414 nvptx_wsync (bool after
)
3416 return gen_nvptx_barsync (GEN_INT (after
));
3419 /* Single neutering according to MASK. FROM is the incoming block and
3420 TO is the outgoing block. These may be the same block. Insert at
3423 if (tid.<axis>) goto end.
3425 and insert before ending branch of TO (if there is such an insn):
3428 <possibly-broadcast-cond>
3431 We currently only use differnt FROM and TO when skipping an entire
3432 loop. We could do more if we detected superblocks. */
3435 nvptx_single (unsigned mask
, basic_block from
, basic_block to
)
3437 rtx_insn
*head
= BB_HEAD (from
);
3438 rtx_insn
*tail
= BB_END (to
);
3439 unsigned skip_mask
= mask
;
3441 /* Find first insn of from block */
3442 while (head
!= BB_END (from
) && !INSN_P (head
))
3443 head
= NEXT_INSN (head
);
3445 /* Find last insn of to block */
3446 rtx_insn
*limit
= from
== to
? head
: BB_HEAD (to
);
3447 while (tail
!= limit
&& !INSN_P (tail
) && !LABEL_P (tail
))
3448 tail
= PREV_INSN (tail
);
3450 /* Detect if tail is a branch. */
3451 rtx tail_branch
= NULL_RTX
;
3452 rtx cond_branch
= NULL_RTX
;
3453 if (tail
&& INSN_P (tail
))
3455 tail_branch
= PATTERN (tail
);
3456 if (GET_CODE (tail_branch
) != SET
|| SET_DEST (tail_branch
) != pc_rtx
)
3457 tail_branch
= NULL_RTX
;
3460 cond_branch
= SET_SRC (tail_branch
);
3461 if (GET_CODE (cond_branch
) != IF_THEN_ELSE
)
3462 cond_branch
= NULL_RTX
;
3468 /* If this is empty, do nothing. */
3469 if (!head
|| !INSN_P (head
))
3472 /* If this is a dummy insn, do nothing. */
3473 switch (recog_memoized (head
))
3477 case CODE_FOR_nvptx_fork
:
3478 case CODE_FOR_nvptx_forked
:
3479 case CODE_FOR_nvptx_joining
:
3480 case CODE_FOR_nvptx_join
:
3486 /* If we're only doing vector single, there's no need to
3487 emit skip code because we'll not insert anything. */
3488 if (!(mask
& GOMP_DIM_MASK (GOMP_DIM_VECTOR
)))
3491 else if (tail_branch
)
3492 /* Block with only unconditional branch. Nothing to do. */
3496 /* Insert the vector test inside the worker test. */
3498 rtx_insn
*before
= tail
;
3499 for (mode
= GOMP_DIM_WORKER
; mode
<= GOMP_DIM_VECTOR
; mode
++)
3500 if (GOMP_DIM_MASK (mode
) & skip_mask
)
3502 rtx_code_label
*label
= gen_label_rtx ();
3503 rtx pred
= cfun
->machine
->axis_predicate
[mode
- GOMP_DIM_WORKER
];
3507 pred
= gen_reg_rtx (BImode
);
3508 cfun
->machine
->axis_predicate
[mode
- GOMP_DIM_WORKER
] = pred
;
3512 if (mode
== GOMP_DIM_VECTOR
)
3513 br
= gen_br_true (pred
, label
);
3515 br
= gen_br_true_uni (pred
, label
);
3516 emit_insn_before (br
, head
);
3518 LABEL_NUSES (label
)++;
3520 before
= emit_label_before (label
, before
);
3522 emit_label_after (label
, tail
);
3525 /* Now deal with propagating the branch condition. */
3528 rtx pvar
= XEXP (XEXP (cond_branch
, 0), 0);
3530 if (GOMP_DIM_MASK (GOMP_DIM_VECTOR
) == mask
)
3532 /* Vector mode only, do a shuffle. */
3533 emit_insn_before (nvptx_gen_vcast (pvar
), tail
);
3537 /* Includes worker mode, do spill & fill. By construction
3538 we should never have worker mode only. */
3541 data
.base
= worker_bcast_sym
;
3544 if (worker_bcast_size
< GET_MODE_SIZE (SImode
))
3545 worker_bcast_size
= GET_MODE_SIZE (SImode
);
3548 emit_insn_before (nvptx_gen_wcast (pvar
, PM_read
, 0, &data
),
3550 /* Barrier so other workers can see the write. */
3551 emit_insn_before (nvptx_wsync (false), tail
);
3553 emit_insn_before (nvptx_gen_wcast (pvar
, PM_write
, 0, &data
), tail
);
3554 /* This barrier is needed to avoid worker zero clobbering
3555 the broadcast buffer before all the other workers have
3556 had a chance to read this instance of it. */
3557 emit_insn_before (nvptx_wsync (true), tail
);
3560 extract_insn (tail
);
3561 rtx unsp
= gen_rtx_UNSPEC (BImode
, gen_rtvec (1, pvar
),
3563 validate_change (tail
, recog_data
.operand_loc
[0], unsp
, false);
3567 /* PAR is a parallel that is being skipped in its entirety according to
3568 MASK. Treat this as skipping a superblock starting at forked
3569 and ending at joining. */
3572 nvptx_skip_par (unsigned mask
, parallel
*par
)
3574 basic_block tail
= par
->join_block
;
3575 gcc_assert (tail
->preds
->length () == 1);
3577 basic_block pre_tail
= (*tail
->preds
)[0]->src
;
3578 gcc_assert (pre_tail
->succs
->length () == 1);
3580 nvptx_single (mask
, par
->forked_block
, pre_tail
);
3583 /* If PAR has a single inner parallel and PAR itself only contains
3584 empty entry and exit blocks, swallow the inner PAR. */
3587 nvptx_optimize_inner (parallel
*par
)
3589 parallel
*inner
= par
->inner
;
3591 /* We mustn't be the outer dummy par. */
3595 /* We must have a single inner par. */
3596 if (!inner
|| inner
->next
)
3599 /* We must only contain 2 blocks ourselves -- the head and tail of
3601 if (par
->blocks
.length () != 2)
3604 /* We must be disjoint partitioning. As we only have vector and
3605 worker partitioning, this is sufficient to guarantee the pars
3606 have adjacent partitioning. */
3607 if ((par
->mask
& inner
->mask
) & (GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1))
3608 /* This indicates malformed code generation. */
3611 /* The outer forked insn should be immediately followed by the inner
3613 rtx_insn
*forked
= par
->forked_insn
;
3614 rtx_insn
*fork
= BB_END (par
->forked_block
);
3616 if (NEXT_INSN (forked
) != fork
)
3618 gcc_checking_assert (recog_memoized (fork
) == CODE_FOR_nvptx_fork
);
3620 /* The outer joining insn must immediately follow the inner join
3622 rtx_insn
*joining
= par
->joining_insn
;
3623 rtx_insn
*join
= inner
->join_insn
;
3624 if (NEXT_INSN (join
) != joining
)
3627 /* Preconditions met. Swallow the inner par. */
3629 fprintf (dump_file
, "Merging loop %x [%d,%d] into %x [%d,%d]\n",
3630 inner
->mask
, inner
->forked_block
->index
,
3631 inner
->join_block
->index
,
3632 par
->mask
, par
->forked_block
->index
, par
->join_block
->index
);
3634 par
->mask
|= inner
->mask
& (GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1);
3636 par
->blocks
.reserve (inner
->blocks
.length ());
3637 while (inner
->blocks
.length ())
3638 par
->blocks
.quick_push (inner
->blocks
.pop ());
3640 par
->inner
= inner
->inner
;
3641 inner
->inner
= NULL
;
3646 /* Process the parallel PAR and all its contained
3647 parallels. We do everything but the neutering. Return mask of
3648 partitioned modes used within this parallel. */
3651 nvptx_process_pars (parallel
*par
)
3654 nvptx_optimize_inner (par
);
3656 unsigned inner_mask
= par
->mask
;
3658 /* Do the inner parallels first. */
3661 par
->inner_mask
= nvptx_process_pars (par
->inner
);
3662 inner_mask
|= par
->inner_mask
;
3665 if (par
->mask
& GOMP_DIM_MASK (GOMP_DIM_MAX
))
3666 /* No propagation needed for a call. */;
3667 else if (par
->mask
& GOMP_DIM_MASK (GOMP_DIM_WORKER
))
3669 nvptx_wpropagate (false, par
->forked_block
, par
->forked_insn
);
3670 nvptx_wpropagate (true, par
->forked_block
, par
->fork_insn
);
3671 /* Insert begin and end synchronizations. */
3672 emit_insn_after (nvptx_wsync (false), par
->forked_insn
);
3673 emit_insn_before (nvptx_wsync (true), par
->joining_insn
);
3675 else if (par
->mask
& GOMP_DIM_MASK (GOMP_DIM_VECTOR
))
3676 nvptx_vpropagate (par
->forked_block
, par
->forked_insn
);
3678 /* Now do siblings. */
3680 inner_mask
|= nvptx_process_pars (par
->next
);
3684 /* Neuter the parallel described by PAR. We recurse in depth-first
3685 order. MODES are the partitioning of the execution and OUTER is
3686 the partitioning of the parallels we are contained in. */
3689 nvptx_neuter_pars (parallel
*par
, unsigned modes
, unsigned outer
)
3691 unsigned me
= (par
->mask
3692 & (GOMP_DIM_MASK (GOMP_DIM_WORKER
)
3693 | GOMP_DIM_MASK (GOMP_DIM_VECTOR
)));
3694 unsigned skip_mask
= 0, neuter_mask
= 0;
3697 nvptx_neuter_pars (par
->inner
, modes
, outer
| me
);
3699 for (unsigned mode
= GOMP_DIM_WORKER
; mode
<= GOMP_DIM_VECTOR
; mode
++)
3701 if ((outer
| me
) & GOMP_DIM_MASK (mode
))
3702 {} /* Mode is partitioned: no neutering. */
3703 else if (!(modes
& GOMP_DIM_MASK (mode
)))
3704 {} /* Mode is not used: nothing to do. */
3705 else if (par
->inner_mask
& GOMP_DIM_MASK (mode
)
3706 || !par
->forked_insn
)
3707 /* Partitioned in inner parallels, or we're not a partitioned
3708 at all: neuter individual blocks. */
3709 neuter_mask
|= GOMP_DIM_MASK (mode
);
3710 else if (!par
->parent
|| !par
->parent
->forked_insn
3711 || par
->parent
->inner_mask
& GOMP_DIM_MASK (mode
))
3712 /* Parent isn't a parallel or contains this paralleling: skip
3713 parallel at this level. */
3714 skip_mask
|= GOMP_DIM_MASK (mode
);
3716 {} /* Parent will skip this parallel itself. */
3725 /* Neuter whole SESE regions. */
3726 bb_pair_vec_t regions
;
3728 nvptx_find_sese (par
->blocks
, regions
);
3729 len
= regions
.length ();
3730 for (ix
= 0; ix
!= len
; ix
++)
3732 basic_block from
= regions
[ix
].first
;
3733 basic_block to
= regions
[ix
].second
;
3736 nvptx_single (neuter_mask
, from
, to
);
3743 /* Neuter each BB individually. */
3744 len
= par
->blocks
.length ();
3745 for (ix
= 0; ix
!= len
; ix
++)
3747 basic_block block
= par
->blocks
[ix
];
3749 nvptx_single (neuter_mask
, block
, block
);
3755 nvptx_skip_par (skip_mask
, par
);
3758 nvptx_neuter_pars (par
->next
, modes
, outer
);
3761 /* PTX-specific reorganization
3762 - Split blocks at fork and join instructions
3763 - Compute live registers
3764 - Mark now-unused registers, so function begin doesn't declare
3766 - Insert state propagation when entering partitioned mode
3767 - Insert neutering instructions when in single mode
3768 - Replace subregs with suitable sequences.
3774 /* We are freeing block_for_insn in the toplev to keep compatibility
3775 with old MDEP_REORGS that are not CFG based. Recompute it now. */
3776 compute_bb_for_insn ();
3778 thread_prologue_and_epilogue_insns ();
3780 /* Split blocks and record interesting unspecs. */
3781 bb_insn_map_t bb_insn_map
;
3783 nvptx_split_blocks (&bb_insn_map
);
3785 /* Compute live regs */
3786 df_clear_flags (DF_LR_RUN_DCE
);
3787 df_set_flags (DF_NO_INSN_RESCAN
| DF_NO_HARD_REGS
);
3788 df_live_add_problem ();
3789 df_live_set_all_dirty ();
3791 regstat_init_n_sets_and_refs ();
3794 df_dump (dump_file
);
3796 /* Mark unused regs as unused. */
3797 int max_regs
= max_reg_num ();
3798 for (int i
= LAST_VIRTUAL_REGISTER
+ 1; i
< max_regs
; i
++)
3799 if (REG_N_SETS (i
) == 0 && REG_N_REFS (i
) == 0)
3800 regno_reg_rtx
[i
] = const0_rtx
;
3802 /* Determine launch dimensions of the function. If it is not an
3803 offloaded function (i.e. this is a regular compiler), the
3804 function has no neutering. */
3805 tree attr
= get_oacc_fn_attrib (current_function_decl
);
3808 /* If we determined this mask before RTL expansion, we could
3809 elide emission of some levels of forks and joins. */
3811 tree dims
= TREE_VALUE (attr
);
3814 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++, dims
= TREE_CHAIN (dims
))
3816 int size
= TREE_INT_CST_LOW (TREE_VALUE (dims
));
3817 tree allowed
= TREE_PURPOSE (dims
);
3819 if (size
!= 1 && !(allowed
&& integer_zerop (allowed
)))
3820 mask
|= GOMP_DIM_MASK (ix
);
3822 /* If there is worker neutering, there must be vector
3823 neutering. Otherwise the hardware will fail. */
3824 gcc_assert (!(mask
& GOMP_DIM_MASK (GOMP_DIM_WORKER
))
3825 || (mask
& GOMP_DIM_MASK (GOMP_DIM_VECTOR
)));
3827 /* Discover & process partitioned regions. */
3828 parallel
*pars
= nvptx_discover_pars (&bb_insn_map
);
3829 nvptx_process_pars (pars
);
3830 nvptx_neuter_pars (pars
, mask
, 0);
3834 /* Replace subregs. */
3835 nvptx_reorg_subreg ();
3837 regstat_free_n_sets_and_refs ();
3839 df_finish_pass (true);
3842 /* Handle a "kernel" attribute; arguments as in
3843 struct attribute_spec.handler. */
3846 nvptx_handle_kernel_attribute (tree
*node
, tree name
, tree
ARG_UNUSED (args
),
3847 int ARG_UNUSED (flags
), bool *no_add_attrs
)
3851 if (TREE_CODE (decl
) != FUNCTION_DECL
)
3853 error ("%qE attribute only applies to functions", name
);
3854 *no_add_attrs
= true;
3856 else if (!VOID_TYPE_P (TREE_TYPE (TREE_TYPE (decl
))))
3858 error ("%qE attribute requires a void return type", name
);
3859 *no_add_attrs
= true;
3865 /* Table of valid machine attributes. */
3866 static const struct attribute_spec nvptx_attribute_table
[] =
3868 /* { name, min_len, max_len, decl_req, type_req, fn_type_req, handler,
3869 affects_type_identity } */
3870 { "kernel", 0, 0, true, false, false, nvptx_handle_kernel_attribute
, false },
3871 { NULL
, 0, 0, false, false, false, NULL
, false }
3874 /* Limit vector alignments to BIGGEST_ALIGNMENT. */
3876 static HOST_WIDE_INT
3877 nvptx_vector_alignment (const_tree type
)
3879 HOST_WIDE_INT align
= tree_to_shwi (TYPE_SIZE (type
));
3881 return MIN (align
, BIGGEST_ALIGNMENT
);
3884 /* Indicate that INSN cannot be duplicated. */
3887 nvptx_cannot_copy_insn_p (rtx_insn
*insn
)
3889 switch (recog_memoized (insn
))
3891 case CODE_FOR_nvptx_shufflesi
:
3892 case CODE_FOR_nvptx_shufflesf
:
3893 case CODE_FOR_nvptx_barsync
:
3894 case CODE_FOR_nvptx_fork
:
3895 case CODE_FOR_nvptx_forked
:
3896 case CODE_FOR_nvptx_joining
:
3897 case CODE_FOR_nvptx_join
:
3904 /* Section anchors do not work. Initialization for flag_section_anchor
3905 probes the existence of the anchoring target hooks and prevents
3906 anchoring if they don't exist. However, we may be being used with
3907 a host-side compiler that does support anchoring, and hence see
3908 the anchor flag set (as it's not recalculated). So provide an
3909 implementation denying anchoring. */
3912 nvptx_use_anchors_for_symbol_p (const_rtx
ARG_UNUSED (a
))
3917 /* Record a symbol for mkoffload to enter into the mapping table. */
3920 nvptx_record_offload_symbol (tree decl
)
3922 switch (TREE_CODE (decl
))
3925 fprintf (asm_out_file
, "//:VAR_MAP \"%s\"\n",
3926 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl
)));
3931 tree attr
= get_oacc_fn_attrib (decl
);
3932 tree dims
= TREE_VALUE (attr
);
3935 fprintf (asm_out_file
, "//:FUNC_MAP \"%s\"",
3936 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl
)));
3938 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++, dims
= TREE_CHAIN (dims
))
3940 int size
= TREE_INT_CST_LOW (TREE_VALUE (dims
));
3942 gcc_assert (!TREE_PURPOSE (dims
));
3943 fprintf (asm_out_file
, ", %#x", size
);
3946 fprintf (asm_out_file
, "\n");
3955 /* Implement TARGET_ASM_FILE_START. Write the kinds of things ptxas expects
3956 at the start of a file. */
3959 nvptx_file_start (void)
3961 fputs ("// BEGIN PREAMBLE\n", asm_out_file
);
3962 fputs ("\t.version\t3.1\n", asm_out_file
);
3963 fputs ("\t.target\tsm_30\n", asm_out_file
);
3964 fprintf (asm_out_file
, "\t.address_size %d\n", GET_MODE_BITSIZE (Pmode
));
3965 fputs ("// END PREAMBLE\n", asm_out_file
);
3968 /* Emit a declaration for a worker-level buffer in .shared memory. */
3971 write_worker_buffer (FILE *file
, rtx sym
, unsigned align
, unsigned size
)
3973 const char *name
= XSTR (sym
, 0);
3975 write_var_marker (file
, true, false, name
);
3976 fprintf (file
, ".shared .align %d .u8 %s[%d];\n",
3980 /* Write out the function declarations we've collected and declare storage
3981 for the broadcast buffer. */
3984 nvptx_file_end (void)
3986 hash_table
<tree_hasher
>::iterator iter
;
3988 FOR_EACH_HASH_TABLE_ELEMENT (*needed_fndecls_htab
, decl
, tree
, iter
)
3989 nvptx_record_fndecl (decl
);
3990 fputs (func_decls
.str().c_str(), asm_out_file
);
3992 if (worker_bcast_size
)
3993 write_worker_buffer (asm_out_file
, worker_bcast_sym
,
3994 worker_bcast_align
, worker_bcast_size
);
3996 if (worker_red_size
)
3997 write_worker_buffer (asm_out_file
, worker_red_sym
,
3998 worker_red_align
, worker_red_size
);
4001 /* Expander for the shuffle builtins. */
4004 nvptx_expand_shuffle (tree exp
, rtx target
, machine_mode mode
, int ignore
)
4009 rtx src
= expand_expr (CALL_EXPR_ARG (exp
, 0),
4010 NULL_RTX
, mode
, EXPAND_NORMAL
);
4012 src
= copy_to_mode_reg (mode
, src
);
4014 rtx idx
= expand_expr (CALL_EXPR_ARG (exp
, 1),
4015 NULL_RTX
, SImode
, EXPAND_NORMAL
);
4016 rtx op
= expand_expr (CALL_EXPR_ARG (exp
, 2),
4017 NULL_RTX
, SImode
, EXPAND_NORMAL
);
4019 if (!REG_P (idx
) && GET_CODE (idx
) != CONST_INT
)
4020 idx
= copy_to_mode_reg (SImode
, idx
);
4022 rtx pat
= nvptx_gen_shuffle (target
, src
, idx
,
4023 (nvptx_shuffle_kind
) INTVAL (op
));
4030 /* Worker reduction address expander. */
4033 nvptx_expand_worker_addr (tree exp
, rtx target
,
4034 machine_mode
ARG_UNUSED (mode
), int ignore
)
4039 unsigned align
= TREE_INT_CST_LOW (CALL_EXPR_ARG (exp
, 2));
4040 if (align
> worker_red_align
)
4041 worker_red_align
= align
;
4043 unsigned offset
= TREE_INT_CST_LOW (CALL_EXPR_ARG (exp
, 0));
4044 unsigned size
= TREE_INT_CST_LOW (CALL_EXPR_ARG (exp
, 1));
4045 if (size
+ offset
> worker_red_size
)
4046 worker_red_size
= size
+ offset
;
4048 rtx addr
= worker_red_sym
;
4051 addr
= gen_rtx_PLUS (Pmode
, addr
, GEN_INT (offset
));
4052 addr
= gen_rtx_CONST (Pmode
, addr
);
4055 emit_move_insn (target
, addr
);
4060 /* Expand the CMP_SWAP PTX builtins. We have our own versions that do
4061 not require taking the address of any object, other than the memory
4062 cell being operated on. */
4065 nvptx_expand_cmp_swap (tree exp
, rtx target
,
4066 machine_mode
ARG_UNUSED (m
), int ARG_UNUSED (ignore
))
4068 machine_mode mode
= TYPE_MODE (TREE_TYPE (exp
));
4071 target
= gen_reg_rtx (mode
);
4073 rtx mem
= expand_expr (CALL_EXPR_ARG (exp
, 0),
4074 NULL_RTX
, Pmode
, EXPAND_NORMAL
);
4075 rtx cmp
= expand_expr (CALL_EXPR_ARG (exp
, 1),
4076 NULL_RTX
, mode
, EXPAND_NORMAL
);
4077 rtx src
= expand_expr (CALL_EXPR_ARG (exp
, 2),
4078 NULL_RTX
, mode
, EXPAND_NORMAL
);
4081 mem
= gen_rtx_MEM (mode
, mem
);
4083 cmp
= copy_to_mode_reg (mode
, cmp
);
4085 src
= copy_to_mode_reg (mode
, src
);
4088 pat
= gen_atomic_compare_and_swapsi_1 (target
, mem
, cmp
, src
, const0_rtx
);
4090 pat
= gen_atomic_compare_and_swapdi_1 (target
, mem
, cmp
, src
, const0_rtx
);
4098 /* Codes for all the NVPTX builtins. */
4101 NVPTX_BUILTIN_SHUFFLE
,
4102 NVPTX_BUILTIN_SHUFFLELL
,
4103 NVPTX_BUILTIN_WORKER_ADDR
,
4104 NVPTX_BUILTIN_CMP_SWAP
,
4105 NVPTX_BUILTIN_CMP_SWAPLL
,
4109 static GTY(()) tree nvptx_builtin_decls
[NVPTX_BUILTIN_MAX
];
4111 /* Return the NVPTX builtin for CODE. */
4114 nvptx_builtin_decl (unsigned code
, bool ARG_UNUSED (initialize_p
))
4116 if (code
>= NVPTX_BUILTIN_MAX
)
4117 return error_mark_node
;
4119 return nvptx_builtin_decls
[code
];
4122 /* Set up all builtin functions for this target. */
4125 nvptx_init_builtins (void)
4127 #define DEF(ID, NAME, T) \
4128 (nvptx_builtin_decls[NVPTX_BUILTIN_ ## ID] \
4129 = add_builtin_function ("__builtin_nvptx_" NAME, \
4130 build_function_type_list T, \
4131 NVPTX_BUILTIN_ ## ID, BUILT_IN_MD, NULL, NULL))
4133 #define UINT unsigned_type_node
4134 #define LLUINT long_long_unsigned_type_node
4135 #define PTRVOID ptr_type_node
4137 DEF (SHUFFLE
, "shuffle", (UINT
, UINT
, UINT
, UINT
, NULL_TREE
));
4138 DEF (SHUFFLELL
, "shufflell", (LLUINT
, LLUINT
, UINT
, UINT
, NULL_TREE
));
4139 DEF (WORKER_ADDR
, "worker_addr",
4140 (PTRVOID
, ST
, UINT
, UINT
, NULL_TREE
));
4141 DEF (CMP_SWAP
, "cmp_swap", (UINT
, PTRVOID
, UINT
, UINT
, NULL_TREE
));
4142 DEF (CMP_SWAPLL
, "cmp_swapll", (LLUINT
, PTRVOID
, LLUINT
, LLUINT
, NULL_TREE
));
4151 /* Expand an expression EXP that calls a built-in function,
4152 with result going to TARGET if that's convenient
4153 (and in mode MODE if that's convenient).
4154 SUBTARGET may be used as the target for computing one of EXP's operands.
4155 IGNORE is nonzero if the value is to be ignored. */
4158 nvptx_expand_builtin (tree exp
, rtx target
, rtx
ARG_UNUSED (subtarget
),
4159 machine_mode mode
, int ignore
)
4161 tree fndecl
= TREE_OPERAND (CALL_EXPR_FN (exp
), 0);
4162 switch (DECL_FUNCTION_CODE (fndecl
))
4164 case NVPTX_BUILTIN_SHUFFLE
:
4165 case NVPTX_BUILTIN_SHUFFLELL
:
4166 return nvptx_expand_shuffle (exp
, target
, mode
, ignore
);
4168 case NVPTX_BUILTIN_WORKER_ADDR
:
4169 return nvptx_expand_worker_addr (exp
, target
, mode
, ignore
);
4171 case NVPTX_BUILTIN_CMP_SWAP
:
4172 case NVPTX_BUILTIN_CMP_SWAPLL
:
4173 return nvptx_expand_cmp_swap (exp
, target
, mode
, ignore
);
4175 default: gcc_unreachable ();
4179 /* Define dimension sizes for known hardware. */
4180 #define PTX_VECTOR_LENGTH 32
4181 #define PTX_WORKER_LENGTH 32
4182 #define PTX_GANG_DEFAULT 32
4184 /* Validate compute dimensions of an OpenACC offload or routine, fill
4185 in non-unity defaults. FN_LEVEL indicates the level at which a
4186 routine might spawn a loop. It is negative for non-routines. If
4187 DECL is null, we are validating the default dimensions. */
4190 nvptx_goacc_validate_dims (tree decl
, int dims
[], int fn_level
)
4192 bool changed
= false;
4194 /* The vector size must be 32, unless this is a SEQ routine. */
4195 if (fn_level
<= GOMP_DIM_VECTOR
&& fn_level
>= -1
4196 && dims
[GOMP_DIM_VECTOR
] >= 0
4197 && dims
[GOMP_DIM_VECTOR
] != PTX_VECTOR_LENGTH
)
4199 if (fn_level
< 0 && dims
[GOMP_DIM_VECTOR
] >= 0)
4200 warning_at (decl
? DECL_SOURCE_LOCATION (decl
) : UNKNOWN_LOCATION
, 0,
4201 dims
[GOMP_DIM_VECTOR
]
4202 ? "using vector_length (%d), ignoring %d"
4203 : "using vector_length (%d), ignoring runtime setting",
4204 PTX_VECTOR_LENGTH
, dims
[GOMP_DIM_VECTOR
]);
4205 dims
[GOMP_DIM_VECTOR
] = PTX_VECTOR_LENGTH
;
4209 /* Check the num workers is not too large. */
4210 if (dims
[GOMP_DIM_WORKER
] > PTX_WORKER_LENGTH
)
4212 warning_at (decl
? DECL_SOURCE_LOCATION (decl
) : UNKNOWN_LOCATION
, 0,
4213 "using num_workers (%d), ignoring %d",
4214 PTX_WORKER_LENGTH
, dims
[GOMP_DIM_WORKER
]);
4215 dims
[GOMP_DIM_WORKER
] = PTX_WORKER_LENGTH
;
4221 dims
[GOMP_DIM_VECTOR
] = PTX_VECTOR_LENGTH
;
4222 if (dims
[GOMP_DIM_WORKER
] < 0)
4223 dims
[GOMP_DIM_WORKER
] = PTX_WORKER_LENGTH
;
4224 if (dims
[GOMP_DIM_GANG
] < 0)
4225 dims
[GOMP_DIM_GANG
] = PTX_GANG_DEFAULT
;
4232 /* Return maximum dimension size, or zero for unbounded. */
4235 nvptx_dim_limit (int axis
)
4239 case GOMP_DIM_WORKER
:
4240 return PTX_WORKER_LENGTH
;
4242 case GOMP_DIM_VECTOR
:
4243 return PTX_VECTOR_LENGTH
;
4251 /* Determine whether fork & joins are needed. */
4254 nvptx_goacc_fork_join (gcall
*call
, const int dims
[],
4255 bool ARG_UNUSED (is_fork
))
4257 tree arg
= gimple_call_arg (call
, 2);
4258 unsigned axis
= TREE_INT_CST_LOW (arg
);
4260 /* We only care about worker and vector partitioning. */
4261 if (axis
< GOMP_DIM_WORKER
)
4264 /* If the size is 1, there's no partitioning. */
4265 if (dims
[axis
] == 1)
4271 /* Generate a PTX builtin function call that returns the address in
4272 the worker reduction buffer at OFFSET. TYPE is the type of the
4273 data at that location. */
4276 nvptx_get_worker_red_addr (tree type
, tree offset
)
4278 machine_mode mode
= TYPE_MODE (type
);
4279 tree fndecl
= nvptx_builtin_decl (NVPTX_BUILTIN_WORKER_ADDR
, true);
4280 tree size
= build_int_cst (unsigned_type_node
, GET_MODE_SIZE (mode
));
4281 tree align
= build_int_cst (unsigned_type_node
,
4282 GET_MODE_ALIGNMENT (mode
) / BITS_PER_UNIT
);
4283 tree call
= build_call_expr (fndecl
, 3, offset
, size
, align
);
4285 return fold_convert (build_pointer_type (type
), call
);
4288 /* Emit a SHFL.DOWN using index SHFL of VAR into DEST_VAR. This function
4289 will cast the variable if necessary. */
4292 nvptx_generate_vector_shuffle (location_t loc
,
4293 tree dest_var
, tree var
, unsigned shift
,
4296 unsigned fn
= NVPTX_BUILTIN_SHUFFLE
;
4297 tree_code code
= NOP_EXPR
;
4298 tree arg_type
= unsigned_type_node
;
4299 tree var_type
= TREE_TYPE (var
);
4300 tree dest_type
= var_type
;
4302 if (TREE_CODE (var_type
) == COMPLEX_TYPE
)
4303 var_type
= TREE_TYPE (var_type
);
4305 if (TREE_CODE (var_type
) == REAL_TYPE
)
4306 code
= VIEW_CONVERT_EXPR
;
4308 if (TYPE_SIZE (var_type
)
4309 == TYPE_SIZE (long_long_unsigned_type_node
))
4311 fn
= NVPTX_BUILTIN_SHUFFLELL
;
4312 arg_type
= long_long_unsigned_type_node
;
4315 tree call
= nvptx_builtin_decl (fn
, true);
4316 tree bits
= build_int_cst (unsigned_type_node
, shift
);
4317 tree kind
= build_int_cst (unsigned_type_node
, SHUFFLE_DOWN
);
4320 if (var_type
!= dest_type
)
4322 /* Do real and imaginary parts separately. */
4323 tree real
= fold_build1 (REALPART_EXPR
, var_type
, var
);
4324 real
= fold_build1 (code
, arg_type
, real
);
4325 real
= build_call_expr_loc (loc
, call
, 3, real
, bits
, kind
);
4326 real
= fold_build1 (code
, var_type
, real
);
4328 tree imag
= fold_build1 (IMAGPART_EXPR
, var_type
, var
);
4329 imag
= fold_build1 (code
, arg_type
, imag
);
4330 imag
= build_call_expr_loc (loc
, call
, 3, imag
, bits
, kind
);
4331 imag
= fold_build1 (code
, var_type
, imag
);
4333 expr
= fold_build2 (COMPLEX_EXPR
, dest_type
, real
, imag
);
4337 expr
= fold_build1 (code
, arg_type
, var
);
4338 expr
= build_call_expr_loc (loc
, call
, 3, expr
, bits
, kind
);
4339 expr
= fold_build1 (code
, dest_type
, expr
);
4342 gimplify_assign (dest_var
, expr
, seq
);
4345 /* Lazily generate the global lock var decl and return its address. */
4348 nvptx_global_lock_addr ()
4350 tree v
= global_lock_var
;
4354 tree name
= get_identifier ("__reduction_lock");
4355 tree type
= build_qualified_type (unsigned_type_node
,
4356 TYPE_QUAL_VOLATILE
);
4357 v
= build_decl (BUILTINS_LOCATION
, VAR_DECL
, name
, type
);
4358 global_lock_var
= v
;
4359 DECL_ARTIFICIAL (v
) = 1;
4360 DECL_EXTERNAL (v
) = 1;
4361 TREE_STATIC (v
) = 1;
4362 TREE_PUBLIC (v
) = 1;
4364 mark_addressable (v
);
4365 mark_decl_referenced (v
);
4368 return build_fold_addr_expr (v
);
4371 /* Insert code to locklessly update *PTR with *PTR OP VAR just before
4372 GSI. We use a lockless scheme for nearly all case, which looks
4374 actual = initval(OP);
4377 write = guess OP myval;
4378 actual = cmp&swap (ptr, guess, write)
4379 } while (actual bit-different-to guess);
4382 This relies on a cmp&swap instruction, which is available for 32-
4383 and 64-bit types. Larger types must use a locking scheme. */
4386 nvptx_lockless_update (location_t loc
, gimple_stmt_iterator
*gsi
,
4387 tree ptr
, tree var
, tree_code op
)
4389 unsigned fn
= NVPTX_BUILTIN_CMP_SWAP
;
4390 tree_code code
= NOP_EXPR
;
4391 tree arg_type
= unsigned_type_node
;
4392 tree var_type
= TREE_TYPE (var
);
4394 if (TREE_CODE (var_type
) == COMPLEX_TYPE
4395 || TREE_CODE (var_type
) == REAL_TYPE
)
4396 code
= VIEW_CONVERT_EXPR
;
4398 if (TYPE_SIZE (var_type
) == TYPE_SIZE (long_long_unsigned_type_node
))
4400 arg_type
= long_long_unsigned_type_node
;
4401 fn
= NVPTX_BUILTIN_CMP_SWAPLL
;
4404 tree swap_fn
= nvptx_builtin_decl (fn
, true);
4406 gimple_seq init_seq
= NULL
;
4407 tree init_var
= make_ssa_name (arg_type
);
4408 tree init_expr
= omp_reduction_init_op (loc
, op
, var_type
);
4409 init_expr
= fold_build1 (code
, arg_type
, init_expr
);
4410 gimplify_assign (init_var
, init_expr
, &init_seq
);
4411 gimple
*init_end
= gimple_seq_last (init_seq
);
4413 gsi_insert_seq_before (gsi
, init_seq
, GSI_SAME_STMT
);
4415 /* Split the block just after the init stmts. */
4416 basic_block pre_bb
= gsi_bb (*gsi
);
4417 edge pre_edge
= split_block (pre_bb
, init_end
);
4418 basic_block loop_bb
= pre_edge
->dest
;
4419 pre_bb
= pre_edge
->src
;
4420 /* Reset the iterator. */
4421 *gsi
= gsi_for_stmt (gsi_stmt (*gsi
));
4423 tree expect_var
= make_ssa_name (arg_type
);
4424 tree actual_var
= make_ssa_name (arg_type
);
4425 tree write_var
= make_ssa_name (arg_type
);
4427 /* Build and insert the reduction calculation. */
4428 gimple_seq red_seq
= NULL
;
4429 tree write_expr
= fold_build1 (code
, var_type
, expect_var
);
4430 write_expr
= fold_build2 (op
, var_type
, write_expr
, var
);
4431 write_expr
= fold_build1 (code
, arg_type
, write_expr
);
4432 gimplify_assign (write_var
, write_expr
, &red_seq
);
4434 gsi_insert_seq_before (gsi
, red_seq
, GSI_SAME_STMT
);
4436 /* Build & insert the cmp&swap sequence. */
4437 gimple_seq latch_seq
= NULL
;
4438 tree swap_expr
= build_call_expr_loc (loc
, swap_fn
, 3,
4439 ptr
, expect_var
, write_var
);
4440 gimplify_assign (actual_var
, swap_expr
, &latch_seq
);
4442 gcond
*cond
= gimple_build_cond (EQ_EXPR
, actual_var
, expect_var
,
4443 NULL_TREE
, NULL_TREE
);
4444 gimple_seq_add_stmt (&latch_seq
, cond
);
4446 gimple
*latch_end
= gimple_seq_last (latch_seq
);
4447 gsi_insert_seq_before (gsi
, latch_seq
, GSI_SAME_STMT
);
4449 /* Split the block just after the latch stmts. */
4450 edge post_edge
= split_block (loop_bb
, latch_end
);
4451 basic_block post_bb
= post_edge
->dest
;
4452 loop_bb
= post_edge
->src
;
4453 *gsi
= gsi_for_stmt (gsi_stmt (*gsi
));
4455 post_edge
->flags
^= EDGE_TRUE_VALUE
| EDGE_FALLTHRU
;
4456 edge loop_edge
= make_edge (loop_bb
, loop_bb
, EDGE_FALSE_VALUE
);
4457 set_immediate_dominator (CDI_DOMINATORS
, loop_bb
, pre_bb
);
4458 set_immediate_dominator (CDI_DOMINATORS
, post_bb
, loop_bb
);
4460 gphi
*phi
= create_phi_node (expect_var
, loop_bb
);
4461 add_phi_arg (phi
, init_var
, pre_edge
, loc
);
4462 add_phi_arg (phi
, actual_var
, loop_edge
, loc
);
4464 loop
*loop
= alloc_loop ();
4465 loop
->header
= loop_bb
;
4466 loop
->latch
= loop_bb
;
4467 add_loop (loop
, loop_bb
->loop_father
);
4469 return fold_build1 (code
, var_type
, write_var
);
4472 /* Insert code to lockfully update *PTR with *PTR OP VAR just before
4473 GSI. This is necessary for types larger than 64 bits, where there
4474 is no cmp&swap instruction to implement a lockless scheme. We use
4475 a lock variable in global memory.
4477 while (cmp&swap (&lock_var, 0, 1))
4480 accum = accum OP var;
4482 cmp&swap (&lock_var, 1, 0);
4485 A lock in global memory is necessary to force execution engine
4486 descheduling and avoid resource starvation that can occur if the
4487 lock is in .shared memory. */
4490 nvptx_lockfull_update (location_t loc
, gimple_stmt_iterator
*gsi
,
4491 tree ptr
, tree var
, tree_code op
)
4493 tree var_type
= TREE_TYPE (var
);
4494 tree swap_fn
= nvptx_builtin_decl (NVPTX_BUILTIN_CMP_SWAP
, true);
4495 tree uns_unlocked
= build_int_cst (unsigned_type_node
, 0);
4496 tree uns_locked
= build_int_cst (unsigned_type_node
, 1);
4498 /* Split the block just before the gsi. Insert a gimple nop to make
4500 gimple
*nop
= gimple_build_nop ();
4501 gsi_insert_before (gsi
, nop
, GSI_SAME_STMT
);
4502 basic_block entry_bb
= gsi_bb (*gsi
);
4503 edge entry_edge
= split_block (entry_bb
, nop
);
4504 basic_block lock_bb
= entry_edge
->dest
;
4505 /* Reset the iterator. */
4506 *gsi
= gsi_for_stmt (gsi_stmt (*gsi
));
4508 /* Build and insert the locking sequence. */
4509 gimple_seq lock_seq
= NULL
;
4510 tree lock_var
= make_ssa_name (unsigned_type_node
);
4511 tree lock_expr
= nvptx_global_lock_addr ();
4512 lock_expr
= build_call_expr_loc (loc
, swap_fn
, 3, lock_expr
,
4513 uns_unlocked
, uns_locked
);
4514 gimplify_assign (lock_var
, lock_expr
, &lock_seq
);
4515 gcond
*cond
= gimple_build_cond (EQ_EXPR
, lock_var
, uns_unlocked
,
4516 NULL_TREE
, NULL_TREE
);
4517 gimple_seq_add_stmt (&lock_seq
, cond
);
4518 gimple
*lock_end
= gimple_seq_last (lock_seq
);
4519 gsi_insert_seq_before (gsi
, lock_seq
, GSI_SAME_STMT
);
4521 /* Split the block just after the lock sequence. */
4522 edge locked_edge
= split_block (lock_bb
, lock_end
);
4523 basic_block update_bb
= locked_edge
->dest
;
4524 lock_bb
= locked_edge
->src
;
4525 *gsi
= gsi_for_stmt (gsi_stmt (*gsi
));
4527 /* Create the lock loop ... */
4528 locked_edge
->flags
^= EDGE_TRUE_VALUE
| EDGE_FALLTHRU
;
4529 make_edge (lock_bb
, lock_bb
, EDGE_FALSE_VALUE
);
4530 set_immediate_dominator (CDI_DOMINATORS
, lock_bb
, entry_bb
);
4531 set_immediate_dominator (CDI_DOMINATORS
, update_bb
, lock_bb
);
4533 /* ... and the loop structure. */
4534 loop
*lock_loop
= alloc_loop ();
4535 lock_loop
->header
= lock_bb
;
4536 lock_loop
->latch
= lock_bb
;
4537 lock_loop
->nb_iterations_estimate
= 1;
4538 lock_loop
->any_estimate
= true;
4539 add_loop (lock_loop
, entry_bb
->loop_father
);
4541 /* Build and insert the reduction calculation. */
4542 gimple_seq red_seq
= NULL
;
4543 tree acc_in
= make_ssa_name (var_type
);
4544 tree ref_in
= build_simple_mem_ref (ptr
);
4545 TREE_THIS_VOLATILE (ref_in
) = 1;
4546 gimplify_assign (acc_in
, ref_in
, &red_seq
);
4548 tree acc_out
= make_ssa_name (var_type
);
4549 tree update_expr
= fold_build2 (op
, var_type
, ref_in
, var
);
4550 gimplify_assign (acc_out
, update_expr
, &red_seq
);
4552 tree ref_out
= build_simple_mem_ref (ptr
);
4553 TREE_THIS_VOLATILE (ref_out
) = 1;
4554 gimplify_assign (ref_out
, acc_out
, &red_seq
);
4556 gsi_insert_seq_before (gsi
, red_seq
, GSI_SAME_STMT
);
4558 /* Build & insert the unlock sequence. */
4559 gimple_seq unlock_seq
= NULL
;
4560 tree unlock_expr
= nvptx_global_lock_addr ();
4561 unlock_expr
= build_call_expr_loc (loc
, swap_fn
, 3, unlock_expr
,
4562 uns_locked
, uns_unlocked
);
4563 gimplify_and_add (unlock_expr
, &unlock_seq
);
4564 gsi_insert_seq_before (gsi
, unlock_seq
, GSI_SAME_STMT
);
4569 /* Emit a sequence to update a reduction accumlator at *PTR with the
4570 value held in VAR using operator OP. Return the updated value.
4572 TODO: optimize for atomic ops and indepedent complex ops. */
4575 nvptx_reduction_update (location_t loc
, gimple_stmt_iterator
*gsi
,
4576 tree ptr
, tree var
, tree_code op
)
4578 tree type
= TREE_TYPE (var
);
4579 tree size
= TYPE_SIZE (type
);
4581 if (size
== TYPE_SIZE (unsigned_type_node
)
4582 || size
== TYPE_SIZE (long_long_unsigned_type_node
))
4583 return nvptx_lockless_update (loc
, gsi
, ptr
, var
, op
);
4585 return nvptx_lockfull_update (loc
, gsi
, ptr
, var
, op
);
4588 /* NVPTX implementation of GOACC_REDUCTION_SETUP. */
4591 nvptx_goacc_reduction_setup (gcall
*call
)
4593 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
4594 tree lhs
= gimple_call_lhs (call
);
4595 tree var
= gimple_call_arg (call
, 2);
4596 int level
= TREE_INT_CST_LOW (gimple_call_arg (call
, 3));
4597 gimple_seq seq
= NULL
;
4599 push_gimplify_context (true);
4601 if (level
!= GOMP_DIM_GANG
)
4603 /* Copy the receiver object. */
4604 tree ref_to_res
= gimple_call_arg (call
, 1);
4606 if (!integer_zerop (ref_to_res
))
4607 var
= build_simple_mem_ref (ref_to_res
);
4610 if (level
== GOMP_DIM_WORKER
)
4612 /* Store incoming value to worker reduction buffer. */
4613 tree offset
= gimple_call_arg (call
, 5);
4614 tree call
= nvptx_get_worker_red_addr (TREE_TYPE (var
), offset
);
4615 tree ptr
= make_ssa_name (TREE_TYPE (call
));
4617 gimplify_assign (ptr
, call
, &seq
);
4618 tree ref
= build_simple_mem_ref (ptr
);
4619 TREE_THIS_VOLATILE (ref
) = 1;
4620 gimplify_assign (ref
, var
, &seq
);
4624 gimplify_assign (lhs
, var
, &seq
);
4626 pop_gimplify_context (NULL
);
4627 gsi_replace_with_seq (&gsi
, seq
, true);
4630 /* NVPTX implementation of GOACC_REDUCTION_INIT. */
4633 nvptx_goacc_reduction_init (gcall
*call
)
4635 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
4636 tree lhs
= gimple_call_lhs (call
);
4637 tree var
= gimple_call_arg (call
, 2);
4638 int level
= TREE_INT_CST_LOW (gimple_call_arg (call
, 3));
4639 enum tree_code rcode
4640 = (enum tree_code
)TREE_INT_CST_LOW (gimple_call_arg (call
, 4));
4641 tree init
= omp_reduction_init_op (gimple_location (call
), rcode
,
4643 gimple_seq seq
= NULL
;
4645 push_gimplify_context (true);
4647 if (level
== GOMP_DIM_VECTOR
)
4649 /* Initialize vector-non-zeroes to INIT_VAL (OP). */
4650 tree tid
= make_ssa_name (integer_type_node
);
4651 tree dim_vector
= gimple_call_arg (call
, 3);
4652 gimple
*tid_call
= gimple_build_call_internal (IFN_GOACC_DIM_POS
, 1,
4654 gimple
*cond_stmt
= gimple_build_cond (NE_EXPR
, tid
, integer_zero_node
,
4655 NULL_TREE
, NULL_TREE
);
4657 gimple_call_set_lhs (tid_call
, tid
);
4658 gimple_seq_add_stmt (&seq
, tid_call
);
4659 gimple_seq_add_stmt (&seq
, cond_stmt
);
4661 /* Split the block just after the call. */
4662 edge init_edge
= split_block (gsi_bb (gsi
), call
);
4663 basic_block init_bb
= init_edge
->dest
;
4664 basic_block call_bb
= init_edge
->src
;
4666 /* Fixup flags from call_bb to init_bb. */
4667 init_edge
->flags
^= EDGE_FALLTHRU
| EDGE_TRUE_VALUE
;
4669 /* Set the initialization stmts. */
4670 gimple_seq init_seq
= NULL
;
4671 tree init_var
= make_ssa_name (TREE_TYPE (var
));
4672 gimplify_assign (init_var
, init
, &init_seq
);
4673 gsi
= gsi_start_bb (init_bb
);
4674 gsi_insert_seq_before (&gsi
, init_seq
, GSI_SAME_STMT
);
4676 /* Split block just after the init stmt. */
4678 edge inited_edge
= split_block (gsi_bb (gsi
), gsi_stmt (gsi
));
4679 basic_block dst_bb
= inited_edge
->dest
;
4681 /* Create false edge from call_bb to dst_bb. */
4682 edge nop_edge
= make_edge (call_bb
, dst_bb
, EDGE_FALSE_VALUE
);
4684 /* Create phi node in dst block. */
4685 gphi
*phi
= create_phi_node (lhs
, dst_bb
);
4686 add_phi_arg (phi
, init_var
, inited_edge
, gimple_location (call
));
4687 add_phi_arg (phi
, var
, nop_edge
, gimple_location (call
));
4689 /* Reset dominator of dst bb. */
4690 set_immediate_dominator (CDI_DOMINATORS
, dst_bb
, call_bb
);
4692 /* Reset the gsi. */
4693 gsi
= gsi_for_stmt (call
);
4697 if (level
== GOMP_DIM_GANG
)
4699 /* If there's no receiver object, propagate the incoming VAR. */
4700 tree ref_to_res
= gimple_call_arg (call
, 1);
4701 if (integer_zerop (ref_to_res
))
4705 gimplify_assign (lhs
, init
, &seq
);
4708 pop_gimplify_context (NULL
);
4709 gsi_replace_with_seq (&gsi
, seq
, true);
4712 /* NVPTX implementation of GOACC_REDUCTION_FINI. */
4715 nvptx_goacc_reduction_fini (gcall
*call
)
4717 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
4718 tree lhs
= gimple_call_lhs (call
);
4719 tree ref_to_res
= gimple_call_arg (call
, 1);
4720 tree var
= gimple_call_arg (call
, 2);
4721 int level
= TREE_INT_CST_LOW (gimple_call_arg (call
, 3));
4723 = (enum tree_code
)TREE_INT_CST_LOW (gimple_call_arg (call
, 4));
4724 gimple_seq seq
= NULL
;
4725 tree r
= NULL_TREE
;;
4727 push_gimplify_context (true);
4729 if (level
== GOMP_DIM_VECTOR
)
4731 /* Emit binary shuffle tree. TODO. Emit this as an actual loop,
4732 but that requires a method of emitting a unified jump at the
4734 for (int shfl
= PTX_VECTOR_LENGTH
/ 2; shfl
> 0; shfl
= shfl
>> 1)
4736 tree other_var
= make_ssa_name (TREE_TYPE (var
));
4737 nvptx_generate_vector_shuffle (gimple_location (call
),
4738 other_var
, var
, shfl
, &seq
);
4740 r
= make_ssa_name (TREE_TYPE (var
));
4741 gimplify_assign (r
, fold_build2 (op
, TREE_TYPE (var
),
4742 var
, other_var
), &seq
);
4748 tree accum
= NULL_TREE
;
4750 if (level
== GOMP_DIM_WORKER
)
4752 /* Get reduction buffer address. */
4753 tree offset
= gimple_call_arg (call
, 5);
4754 tree call
= nvptx_get_worker_red_addr (TREE_TYPE (var
), offset
);
4755 tree ptr
= make_ssa_name (TREE_TYPE (call
));
4757 gimplify_assign (ptr
, call
, &seq
);
4760 else if (integer_zerop (ref_to_res
))
4767 /* UPDATE the accumulator. */
4768 gsi_insert_seq_before (&gsi
, seq
, GSI_SAME_STMT
);
4770 r
= nvptx_reduction_update (gimple_location (call
), &gsi
,
4776 gimplify_assign (lhs
, r
, &seq
);
4777 pop_gimplify_context (NULL
);
4779 gsi_replace_with_seq (&gsi
, seq
, true);
4782 /* NVPTX implementation of GOACC_REDUCTION_TEARDOWN. */
4785 nvptx_goacc_reduction_teardown (gcall
*call
)
4787 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
4788 tree lhs
= gimple_call_lhs (call
);
4789 tree var
= gimple_call_arg (call
, 2);
4790 int level
= TREE_INT_CST_LOW (gimple_call_arg (call
, 3));
4791 gimple_seq seq
= NULL
;
4793 push_gimplify_context (true);
4794 if (level
== GOMP_DIM_WORKER
)
4796 /* Read the worker reduction buffer. */
4797 tree offset
= gimple_call_arg (call
, 5);
4798 tree call
= nvptx_get_worker_red_addr(TREE_TYPE (var
), offset
);
4799 tree ptr
= make_ssa_name (TREE_TYPE (call
));
4801 gimplify_assign (ptr
, call
, &seq
);
4802 var
= build_simple_mem_ref (ptr
);
4803 TREE_THIS_VOLATILE (var
) = 1;
4806 if (level
!= GOMP_DIM_GANG
)
4808 /* Write to the receiver object. */
4809 tree ref_to_res
= gimple_call_arg (call
, 1);
4811 if (!integer_zerop (ref_to_res
))
4812 gimplify_assign (build_simple_mem_ref (ref_to_res
), var
, &seq
);
4816 gimplify_assign (lhs
, var
, &seq
);
4818 pop_gimplify_context (NULL
);
4820 gsi_replace_with_seq (&gsi
, seq
, true);
4823 /* NVPTX reduction expander. */
4826 nvptx_goacc_reduction (gcall
*call
)
4828 unsigned code
= (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
4832 case IFN_GOACC_REDUCTION_SETUP
:
4833 nvptx_goacc_reduction_setup (call
);
4836 case IFN_GOACC_REDUCTION_INIT
:
4837 nvptx_goacc_reduction_init (call
);
4840 case IFN_GOACC_REDUCTION_FINI
:
4841 nvptx_goacc_reduction_fini (call
);
4844 case IFN_GOACC_REDUCTION_TEARDOWN
:
4845 nvptx_goacc_reduction_teardown (call
);
4853 #undef TARGET_OPTION_OVERRIDE
4854 #define TARGET_OPTION_OVERRIDE nvptx_option_override
4856 #undef TARGET_ATTRIBUTE_TABLE
4857 #define TARGET_ATTRIBUTE_TABLE nvptx_attribute_table
4860 #define TARGET_LRA_P hook_bool_void_false
4862 #undef TARGET_LEGITIMATE_ADDRESS_P
4863 #define TARGET_LEGITIMATE_ADDRESS_P nvptx_legitimate_address_p
4865 #undef TARGET_PROMOTE_FUNCTION_MODE
4866 #define TARGET_PROMOTE_FUNCTION_MODE nvptx_promote_function_mode
4868 #undef TARGET_FUNCTION_ARG
4869 #define TARGET_FUNCTION_ARG nvptx_function_arg
4870 #undef TARGET_FUNCTION_INCOMING_ARG
4871 #define TARGET_FUNCTION_INCOMING_ARG nvptx_function_incoming_arg
4872 #undef TARGET_FUNCTION_ARG_ADVANCE
4873 #define TARGET_FUNCTION_ARG_ADVANCE nvptx_function_arg_advance
4874 #undef TARGET_FUNCTION_ARG_BOUNDARY
4875 #define TARGET_FUNCTION_ARG_BOUNDARY nvptx_function_arg_boundary
4876 #undef TARGET_PASS_BY_REFERENCE
4877 #define TARGET_PASS_BY_REFERENCE nvptx_pass_by_reference
4878 #undef TARGET_FUNCTION_VALUE_REGNO_P
4879 #define TARGET_FUNCTION_VALUE_REGNO_P nvptx_function_value_regno_p
4880 #undef TARGET_FUNCTION_VALUE
4881 #define TARGET_FUNCTION_VALUE nvptx_function_value
4882 #undef TARGET_LIBCALL_VALUE
4883 #define TARGET_LIBCALL_VALUE nvptx_libcall_value
4884 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
4885 #define TARGET_FUNCTION_OK_FOR_SIBCALL nvptx_function_ok_for_sibcall
4886 #undef TARGET_GET_DRAP_RTX
4887 #define TARGET_GET_DRAP_RTX nvptx_get_drap_rtx
4888 #undef TARGET_SPLIT_COMPLEX_ARG
4889 #define TARGET_SPLIT_COMPLEX_ARG hook_bool_const_tree_true
4890 #undef TARGET_RETURN_IN_MEMORY
4891 #define TARGET_RETURN_IN_MEMORY nvptx_return_in_memory
4892 #undef TARGET_OMIT_STRUCT_RETURN_REG
4893 #define TARGET_OMIT_STRUCT_RETURN_REG true
4894 #undef TARGET_STRICT_ARGUMENT_NAMING
4895 #define TARGET_STRICT_ARGUMENT_NAMING nvptx_strict_argument_naming
4896 #undef TARGET_CALL_ARGS
4897 #define TARGET_CALL_ARGS nvptx_call_args
4898 #undef TARGET_END_CALL_ARGS
4899 #define TARGET_END_CALL_ARGS nvptx_end_call_args
4901 #undef TARGET_ASM_FILE_START
4902 #define TARGET_ASM_FILE_START nvptx_file_start
4903 #undef TARGET_ASM_FILE_END
4904 #define TARGET_ASM_FILE_END nvptx_file_end
4905 #undef TARGET_ASM_GLOBALIZE_LABEL
4906 #define TARGET_ASM_GLOBALIZE_LABEL nvptx_globalize_label
4907 #undef TARGET_ASM_ASSEMBLE_UNDEFINED_DECL
4908 #define TARGET_ASM_ASSEMBLE_UNDEFINED_DECL nvptx_assemble_undefined_decl
4909 #undef TARGET_PRINT_OPERAND
4910 #define TARGET_PRINT_OPERAND nvptx_print_operand
4911 #undef TARGET_PRINT_OPERAND_ADDRESS
4912 #define TARGET_PRINT_OPERAND_ADDRESS nvptx_print_operand_address
4913 #undef TARGET_PRINT_OPERAND_PUNCT_VALID_P
4914 #define TARGET_PRINT_OPERAND_PUNCT_VALID_P nvptx_print_operand_punct_valid_p
4915 #undef TARGET_ASM_INTEGER
4916 #define TARGET_ASM_INTEGER nvptx_assemble_integer
4917 #undef TARGET_ASM_DECL_END
4918 #define TARGET_ASM_DECL_END nvptx_assemble_decl_end
4919 #undef TARGET_ASM_DECLARE_CONSTANT_NAME
4920 #define TARGET_ASM_DECLARE_CONSTANT_NAME nvptx_asm_declare_constant_name
4921 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
4922 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P hook_bool_mode_const_rtx_true
4923 #undef TARGET_ASM_NEED_VAR_DECL_BEFORE_USE
4924 #define TARGET_ASM_NEED_VAR_DECL_BEFORE_USE true
4926 #undef TARGET_MACHINE_DEPENDENT_REORG
4927 #define TARGET_MACHINE_DEPENDENT_REORG nvptx_reorg
4928 #undef TARGET_NO_REGISTER_ALLOCATION
4929 #define TARGET_NO_REGISTER_ALLOCATION true
4931 #undef TARGET_ENCODE_SECTION_INFO
4932 #define TARGET_ENCODE_SECTION_INFO nvptx_encode_section_info
4933 #undef TARGET_RECORD_OFFLOAD_SYMBOL
4934 #define TARGET_RECORD_OFFLOAD_SYMBOL nvptx_record_offload_symbol
4936 #undef TARGET_VECTOR_ALIGNMENT
4937 #define TARGET_VECTOR_ALIGNMENT nvptx_vector_alignment
4939 #undef TARGET_CANNOT_COPY_INSN_P
4940 #define TARGET_CANNOT_COPY_INSN_P nvptx_cannot_copy_insn_p
4942 #undef TARGET_USE_ANCHORS_FOR_SYMBOL_P
4943 #define TARGET_USE_ANCHORS_FOR_SYMBOL_P nvptx_use_anchors_for_symbol_p
4945 #undef TARGET_INIT_BUILTINS
4946 #define TARGET_INIT_BUILTINS nvptx_init_builtins
4947 #undef TARGET_EXPAND_BUILTIN
4948 #define TARGET_EXPAND_BUILTIN nvptx_expand_builtin
4949 #undef TARGET_BUILTIN_DECL
4950 #define TARGET_BUILTIN_DECL nvptx_builtin_decl
4952 #undef TARGET_GOACC_VALIDATE_DIMS
4953 #define TARGET_GOACC_VALIDATE_DIMS nvptx_goacc_validate_dims
4955 #undef TARGET_GOACC_DIM_LIMIT
4956 #define TARGET_GOACC_DIM_LIMIT nvptx_dim_limit
4958 #undef TARGET_GOACC_FORK_JOIN
4959 #define TARGET_GOACC_FORK_JOIN nvptx_goacc_fork_join
4961 #undef TARGET_GOACC_REDUCTION
4962 #define TARGET_GOACC_REDUCTION nvptx_goacc_reduction
4964 struct gcc_target targetm
= TARGET_INITIALIZER
;
4966 #include "gt-nvptx.h"